{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ur8xi4C7S06n"
      },
      "outputs": [],
      "source": [
        "# Copyright 2024 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAPoU8Sm5E6e"
      },
      "source": [
        "# Advanced Diarized Transcription and Domain Specific Summarization\n",
        "\n",
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/diarized-transcription-summarization/diarized_transcription_and_summarization.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84f0f73a0f76"
      },
      "source": [
        "| | |\n",
        "|-|-|\n",
        "| Author(s) | [Anant Nawalgaria](https://github.com/anantnawal/), [Patrick Nestler](https://github.com/nestler/)| "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvgnzT1CKxrO"
      },
      "source": [
        "## Overview\n",
        "\n",
        "Many businesses, particularly in the financial sector (but also other industries), struggle with accurately transcribing and summarizing multilingual audio recordings which could be several hours long. This challenge is especially critical for use cases that directly impact customer experience and financial outcomes.\n",
        "\n",
        "Some key obstacles include:\n",
        "\n",
        "- **Hallucinations**: AI models sometimes generate incorrect or nonsensical information.\n",
        "- **Numerical Inaccuracies**: Precise transcription of numbers is crucial in finance, and errors can have serious consequences.\n",
        "- **Speaker Misidentification for multilingual conversations**: Accurately attributing dialogue in multi-speaker settings, especially with more than two participants in multilingual settings, can be complex.\n",
        "- **Summarization Deficiencies**: Generating concise summaries tailored to the specific financial domain is essential for efficient analysis.\n",
        "- **Recordings which are several hours long**: Ensuring diarization and transcription can be both effectively and efficiently performed on long recordings.\n",
        "\n",
        "This notebook demonstrates sample code for a semi-agentic, multimodal [solution developed for Commerzbank](https://cloud.google.com/blog/products/ai-machine-learning/how-commerzbank-is-transforming-financial-advisory-workflows-with-gen-ai).  A more advanced version of this solution is currently deployed in production, delivering substantial productivity gains.\n",
        "\n",
        "### Objectives\n",
        "\n",
        "In this tutorial, you will learn how to do build an advanced diarized transcription and domain/task specific summarization using the multimodal capabilities of Gemini on Vertex AI together with the Gen AI Evaluation Service API in Vertex AI service for Python.\n",
        "You will complete the following tasks:\n",
        "\n",
        "- Install the Vertex AI SDK for Python\n",
        "- Chunk an audio file into segments of pre-specified durations\n",
        "- Use the Gemini on Vertex AI to interact with the audio files\n",
        "  - Gemini 2.0 (`gemini-2.0-flash`) model:\n",
        "    - to use few-shot multimodal prompting combined with specific task specific instructions to perform sequential diarized transcription of all the contiguous audio chunks, using the output generated at each step as input for the next one\n",
        "    - extract task/domain specific facts and figures \n",
        "    - generate multiple task specific summaries from the extracted facts and diarized transcript\n",
        "    - select the best summary for each task, using pointwise and pairwise evaluations using the Gen AI Evaluation Service."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61RBz8LLbxCR"
      },
      "source": [
        "## Get started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "No17Cw5hgx12"
      },
      "source": [
        "### Install Vertex AI SDK and other required packages\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tFy3H3aPgx12"
      },
      "outputs": [],
      "source": [
        "%pip install --upgrade --user --quiet google-cloud-aiplatform[evaluation]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5Xep4W9lq-Z"
      },
      "source": [
        "### Restart runtime\n",
        "\n",
        "To use the newly installed packages in this Jupyter runtime, you must restart the runtime. You can do this by running the cell below, which restarts the current kernel.\n",
        "\n",
        "The restart might take a minute or longer. After it's restarted, continue to the next step."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XRvKdaPDTznN"
      },
      "outputs": [],
      "source": [
        "import IPython\n",
        "\n",
        "app = IPython.Application.instance()\n",
        "app.kernel.do_shutdown(True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbmM4z7FOBpM"
      },
      "source": [
        "<div class=\"alert alert-block alert-warning\">\n",
        "<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
        "</div>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dmWOrTJ3gx13"
      },
      "source": [
        "### Authenticate your notebook environment (Colab only)\n",
        "\n",
        "If you're running this notebook on Google Colab, run the cell below to authenticate your environment."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NyKGtVQjgx13"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    from google.colab import auth\n",
        "\n",
        "    auth.authenticate_user()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DF4l8DTdWgPY"
      },
      "source": [
        "### Set Google Cloud project information and initialize Vertex AI SDK\n",
        "\n",
        "To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
        "\n",
        "Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nqwi-5ufWp_B"
      },
      "outputs": [],
      "source": [
        "# Use the environment variable if the user doesn't provide Project ID.\n",
        "import os\n",
        "\n",
        "import vertexai\n",
        "\n",
        "# fmt: off\n",
        "PROJECT_ID = \"[your-project-id]\"  # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n",
        "# fmt: on\n",
        "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
        "    PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\"))\n",
        "\n",
        "LOCATION = os.environ.get(\"GOOGLE_CLOUD_REGION\", \"us-central1\")\n",
        "\n",
        "vertexai.init(project=PROJECT_ID, location=LOCATION)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5303c05f7aa6"
      },
      "source": [
        "### Import libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "6fc324893334"
      },
      "outputs": [],
      "source": [
        "import functools\n",
        "import uuid\n",
        "from functools import partial\n",
        "\n",
        "import nest_asyncio\n",
        "import pandas as pd\n",
        "from google.cloud import aiplatform, storage\n",
        "from pydub import AudioSegment\n",
        "from pydub.utils import make_chunks\n",
        "from vertexai.evaluation import EvalTask, MetricPromptTemplateExamples\n",
        "from vertexai.generative_models import GenerationConfig, GenerativeModel, Part\n",
        "\n",
        "# fmt: off\n",
        "nest_asyncio.apply()  # @param {type:\"string\", isTemplate: true}\n",
        "# fmt: on"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e43229f3ad4f"
      },
      "source": [
        "## Load Helper Functions\n",
        "In this section we are going to define the various functions involved in developing an advanced diarized transcription and domain/task specific summarization system."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5d4a983bd7f3"
      },
      "source": [
        "### Step 1. Audio chunking.\n",
        "In this step we chunk the audio into smaller chunks of predefined durations, in order to allow transcriptions of large audio files (lasting several hours) which go beyond the size of the output window of the Gemini 2.0 model.  (which was `8192` at the time of publication)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3091633fde85"
      },
      "outputs": [],
      "source": [
        "storage_client = storage.Client()\n",
        "\n",
        "\n",
        "def chunk_audio(\n",
        "    audio_url: str,\n",
        "    output_bucket: str,\n",
        "    output_format: str = \"wav\",\n",
        "    chunk_length: int = 600000,\n",
        ") -> list[str]:\n",
        "    \"\"\"Splits an audio file in GCS into chunks and stores them back in GCS.\n",
        "\n",
        "    Args:\n",
        "        audio_url (str): the GCS url of the audio file to be chunked.\n",
        "        output_bucket (str):the GCS bucket the chunked audio files would be stored.\n",
        "        chunk_length_ms (int): Desired chunk length in milliseconds (default: 10\n",
        "          minutes).\n",
        "        output_format (str): Output audio format (default: wav).\n",
        "    \"\"\"\n",
        "    bucket_name = audio_url.split(\"/\")[2]\n",
        "\n",
        "    input_blob_name = \"/\".join(audio_url.split(\"/\")[3:])\n",
        "\n",
        "    bucket = storage_client.bucket(bucket_name)\n",
        "    blob = bucket.blob(input_blob_name)\n",
        "\n",
        "    # Download the FLAC file to a temporary local file\n",
        "    with open(\"temp_audio.\" + output_format, \"wb\") as temp_file:\n",
        "        storage_client.download_blob_to_file(blob, temp_file)\n",
        "\n",
        "    url_audio_chunks = list()\n",
        "    # Split the audio using pydub\n",
        "    audio = AudioSegment.from_file(\"temp_audio.\" + output_format, format=output_format)\n",
        "    chunks = make_chunks(audio, chunk_length)\n",
        "\n",
        "    bucket = storage_client.bucket(output_bucket)\n",
        "    for i, chunk in enumerate(chunks):\n",
        "        chunk_name_local = f\"{input_blob_name.split('/')[-1].split('.')[0]}_part{i + 1}.{output_format}\"\n",
        "        chunk.export(chunk_name_local, format=output_format)\n",
        "\n",
        "        chunk_name_gcs = f\"{input_blob_name.split('.')[0]}_part{i + 1}.{output_format}\"\n",
        "\n",
        "        # Upload the split chunk back to GCS\n",
        "        blob = bucket.blob(chunk_name_gcs)\n",
        "        blob.upload_from_filename(chunk_name_local)\n",
        "        url_audio_chunks.append(f\"gs://{output_bucket}/{chunk_name_gcs}\")\n",
        "\n",
        "    return url_audio_chunks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d369ed0db89e"
      },
      "source": [
        "### Step 2. Advanced diarized transcription\n",
        "This process focuses on creating a highly accurate transcript from audio data using Gemini 2.0. Here's the breakdown:\n",
        "\n",
        "- Diarization: The audio is broken into chunks and processed sequentially. Gemini 2.0 identifies the speakers in each chunk and attributes their words correctly.\n",
        "- Contextual Processing: To ensure accuracy, the model receives the transcript generated up to the current chunk, along with carefully engineered prompts and a few-shot example. This helps maintain consistency and correctly identify speakers, especially with crucial numerical information.\n",
        "- Cleanup: After the full transcript is created, the individual audio chunks are deleted to save storage space in the cleanup phase.\n",
        "\n",
        "Essentially, this is a careful, step-by-step process designed to produce a structured, accurate transcript that preserves speaker identification and important details like numbers.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a8ffaf698c51"
      },
      "outputs": [],
      "source": [
        "def diarize_transcribe(\n",
        "    url_audio_chunks: list[str],\n",
        "    consider_previous: bool = True,\n",
        "    quick: bool = False,\n",
        "    file_format: str = \"audio/wav\",\n",
        "    path_example_transcription: str = \"gs://\",\n",
        "    path_example_audio: str = \"gs://\",\n",
        ") -> str:\n",
        "    \"\"\"Take the GCS urls of contiguous chunked audio files and the example transcription of audio files and\n",
        "    do advanced diarized transcription\n",
        "    \"\"\"\n",
        "    bucket = storage_client.bucket(path_example_transcription.split(\"/\")[2])\n",
        "    blob = bucket.blob(\"/\".join(path_example_transcription.split(\"/\")[3:]))\n",
        "    ex_output_transcription = blob.download_as_string().decode(\"utf-8\")\n",
        "\n",
        "    generation_model_audio = GenerativeModel(\n",
        "        \"gemini-2.0-flash\".replace(\"pro\", \"flash\" if quick else \"pro\")\n",
        "    )\n",
        "    generation_config_audio = GenerationConfig(\n",
        "        temperature=0.0,\n",
        "        max_output_tokens=8192,\n",
        "        candidate_count=1,\n",
        "    )\n",
        "    prompts_transcription_initial_p1 = \"\"\"\n",
        "      Transcribe the referenced audio file. The file contains a recording of an advisory call between one or more bank advisor(s) and their customer.\n",
        "      Differentiate and diarize the speakers in the call clearly. Do not add time marks.\n",
        "      Ensure correctness and consistency in speaker names while attributing statements during diarization.\n",
        "      Ensure that spoken numbers and numerical figures are transcribed correctly and precisely.\n",
        "      Some calls can also contain segments of conversations just between banking advisors, sure the diarization correctly reflects that.\n",
        "      In case the call contains conversation segments between banking advisors might contain reference to internal systems, make sure this is properly transcribed.\n",
        "      Here is an example of an input audio file and its corresponding transcription.\n",
        "      Example Input Audio File:\n",
        "      \"\"\"\n",
        "    prompts_transcription_initial_p2 = \"\"\"\n",
        "      transcription of the example audio file:\n",
        "      {ex_output_transcription}\n",
        "      Here is the referenced audio file to transcribe keeping the instructions above in mind:\n",
        "      \"\"\"\n",
        "    prompts_transcription_subsequent_p1 = \"\"\"\n",
        "      The given file is a contiguous audio chunk, containing continuation of a recording of an advisory call between one or more bank advisor(s) and their customer.\n",
        "      Differentiate and diarize the speakers in the call clearly. \n",
        "      Transcribe the referenced audio file, continuing on from the given transcription of the previous part , while also keeping speaker names consistent. Do not add time marks.\n",
        "      Ensure correctness and consistency in speaker names while attributing statements during diarization.\n",
        "      Ensure that spoken numbers and numerical figures are transcribed correctly and precisely.\n",
        "      Some calls can also contain segments of conversations just between banking advisors, sure the diarization correctly reflects that.\n",
        "      In case the call contains conversation segments between banking advisors might contain reference to internal systems, make sure this is properly transcribed.\n",
        "      Here is an example of an input audio file and its corresponding transcription.\n",
        "      Example Input Audio File:\n",
        "      \"\"\"\n",
        "    prompts_transcription_subsequent_p2 = \"\"\"\n",
        "      transcription of the example audio file:\n",
        "      {ex_output_transcription}    \n",
        "      Transcription so far of previous audio chunks:\n",
        "      {transcription}\n",
        "      Here is the referenced audio file to transcribe keeping the instructions above in mind:\n",
        "      \"\"\"\n",
        "    generated_transcription = \"\"\n",
        "    for i in range(len(url_audio_chunks)):\n",
        "        if i == 0:\n",
        "            prompt = prompts_transcription_initial_p1\n",
        "            prompt_2 = prompts_transcription_initial_p2.format(\n",
        "                ex_output_transcription=ex_output_transcription\n",
        "            )\n",
        "        else:\n",
        "            prompt = (\n",
        "                prompts_transcription_subsequent_p1\n",
        "                if consider_previous\n",
        "                else prompts_transcription_initial_p1\n",
        "            )\n",
        "            prompt_2 = (\n",
        "                prompts_transcription_subsequent_p2.format(\n",
        "                    ex_output_transcription=ex_output_transcription,\n",
        "                    transcription=generated_transcription,\n",
        "                )\n",
        "                if consider_previous\n",
        "                else prompts_transcription_initial_p2.format(\n",
        "                    ex_output_transcription=ex_output_transcription\n",
        "                )\n",
        "            )\n",
        "        audio_url = url_audio_chunks[i]\n",
        "        audio_file = Part.from_uri(audio_url, mime_type=file_format)\n",
        "        ex_audio_file = Part.from_uri(path_example_audio, mime_type=file_format)\n",
        "        contents = [prompt, ex_audio_file, prompt_2, audio_file]\n",
        "        response = generation_model_audio.generate_content(\n",
        "            contents=contents, generation_config=generation_config_audio\n",
        "        )\n",
        "        generated_transcription += response.text\n",
        "\n",
        "    return generated_transcription"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1b4e256e8259"
      },
      "source": [
        "### Step 3 and 4: Fact extraction & Summary generation\n",
        "Step 3 involves identifying key information related to the specific task/document ( in this example financial advisory document) that needs to be completed. The model is prompted to recognize and extract crucial details such as client names, investment preferences, risk tolerance, and financial goals.\n",
        "\n",
        "Step 4 then focuses on generating concise and accurate summaries for each field within the document. Leveraging the extracted facts from the previous step and employing Zero-shot Chain-of-Thought (CoT) prompting, Gemini creates multiple  summaries tailored to the specific domain and the requirements of each form field. This ensures the generated summaries are not only informative but also comply any internal guidelines requirements.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "82811a63412e"
      },
      "outputs": [],
      "source": [
        "def extract_facts(generated_transcription: str, quick: bool = True) -> str:\n",
        "    \"\"\"Take the generated transcript and extract key domain/task-specific facts for downstream processing\"\"\"\n",
        "    generation_model_facts = GenerativeModel(\n",
        "        \"gemini-2.0-flash\".replace(\"pro\", \"flash\" if quick else \"pro\")\n",
        "    )\n",
        "    generation_config_facts = GenerationConfig(\n",
        "        temperature=0.0,\n",
        "        max_output_tokens=512,\n",
        "    )\n",
        "\n",
        "    prompt = f\"\"\"You are an experienced bank advisor who provides consultations to clients regarding financial products. You conducted a telephone consultation with a client, during which you discussed the client's need for one or more financial products. This consultation was recorded. A consultation can span multiple individual conversations. The transcription or transcriptions of the recording can be found below.\n",
        "    Your task is to extract information from the content of the consultation that you conducted with your client. Combine the content of multiple transcripts. Use the field definition provided below for the extraction of fields. The field definition consists of the field name and a field description for each field.\n",
        "\n",
        "    Reason for Consultation: Describes the reason for the consultation, i.e., the reason why the consultation was conducted.\n",
        "\n",
        "    Details of the Consultation Reason: Details of the client's needs. The following content should be mentioned in particular, if they were discussed in the conversation: financial needs, the scope of a basic transaction, interest rates, and desired conditions.\n",
        "\n",
        "    Current Market Expectation and Market Forecast of the Client (if available): Information on the market expectation and expected changes in the market that the client has in relation to the reason for the consultation.\n",
        "\n",
        "    Investment/Financing Horizon of the Client: Period for which the financial need exists or an investment is to be made.\n",
        "\n",
        "    Priorities and Goals of the Client: Expected benefits for the client and priorities of the client. Also includes any exclusion criteria for certain products mentioned by the client.\n",
        "\n",
        "    Existing Knowledge and Experience of the Client: Existing knowledge and experience of the client in relation to the reason for the consultation.\n",
        "\n",
        "    Risk Profile of the Client: What risks the client is willing to take.\n",
        "\n",
        "    Existing Products: Financial products that the client already uses at Commerzbank or other financial institutions in connection with the reason for the consultation.\n",
        "\n",
        "    Products Discussed: Products that were discussed as candidates for a recommendation to meet the client's needs.\n",
        "\n",
        "    Recommended Product: Financial product or products recommended to the client. Explicit mention of the product name/product designation.\n",
        "\n",
        "    Reason for the Recommended Product: Recommendation of the advisor regarding the use of the product. Suitability of the product for the investment objectives, the investment horizon, and the risk profile of the client.\n",
        "\n",
        "    Functionality of the Proposed Product: Describes the functionality of the proposed product.\n",
        "\n",
        "    Product Advantages: The advantages of the presented product. If several products were discussed, comparison of the advantages of the recommended product to these products.\n",
        "\n",
        "    Risk Disclosure: The risks and disadvantages of the presented product. If several products were discussed, comparison of the risks and disadvantages of the recommended product to these products.\n",
        "\n",
        "    Product-Specific Features: Important additional information on the product and its functionality that was explained to the client. In particular, this includes mandatory disclosures such as bail-in, fee-based advice, default risk, or a negative market value.\n",
        "\n",
        "    Higher Costs of Structured Products Compared to Plain Vanilla Products: Mentioning and naming the higher costs of structured products compared to plain vanilla products.\n",
        "\n",
        "    Initially Negative Market Value for OTC Products: Initially negative market value for structured OTC products, with the exception of purchased options.\n",
        "\n",
        "    Client's Feedback on the Recommendation: Client's feedback on the recommended product and further steps.\n",
        "\n",
        "    Objection Handling: Client's objections and the advisor's response to these objections.\n",
        "\n",
        "    Client Questions: Questions clearly attributable to the client regarding the recommendation made and the answer to these questions.\n",
        "\n",
        "    Agreement to Receive the Documents after the Trade: Client's consent to receive the documents used in connection with the consultation only after the trade or the conclusion of the transaction.\n",
        "\n",
        "    Client's Prior Knowledge of the Recommended Product: Describes the client's prior knowledge in the specific context of the recommended product.\n",
        "\n",
        "    Always consider the following guidelines for extracting information:\n",
        "    Do not add any statements or comments to the content of the recording.\n",
        "    Verify facts contained in the document, especially product names, numerical values, dates, and sums, within the context of the entire transcript.\n",
        "    Ensure that details from the conversation such as product names, sums, and interest rates are included in the extracted information.\n",
        "    Retain English terms, particularly the names of financial products.\n",
        "    Justify your answers with facts from the provided input and refer specifically only to sources from the audio call.\n",
        "    Ensure that statements are correctly attributed to either the bank or the client. Do not include source references or timestamps.\n",
        "    Use the term 'The Client' instead of the client's name.\n",
        "    Provide the information grammatically correct and in German.\n",
        "    Think step-by-step and check whether the results meet the above tasks.\n",
        "    Here is the transcript of the telephone call or the transcripts of the telephone calls. Iterate over the transcript multiple times to increase confidence in extracting all information correctly:\n",
        "    {generated_transcription}\n",
        "    \"\"\"\n",
        "    generated_facts = generation_model_facts.generate_content(\n",
        "        contents=prompt,\n",
        "        generation_config=generation_config_facts,\n",
        "    ).text\n",
        "    return generated_facts\n",
        "\n",
        "\n",
        "def generate_summaries(\n",
        "    context: str, quick: bool = False, num_summaries: int = 3\n",
        ") -> list[str]:\n",
        "    \"\"\"Take the generated transcript extracted facts and generate one or more task and domain specific summaries.\"\"\"\n",
        "    prompt_instr = \"\"\"\n",
        "    You are an experienced bank advisor providing consultations to clients on banking products. Create a 5-sentence summary. Ensure that details from the conversation, like sums and interest rates, are included in the summary.\n",
        "    Your inputs are the transcription of the conversation and some important information extracted from this transcription, provided below. The relevant fields for you to consider are the following:\n",
        "    - Reason for consultation:\n",
        "    - Details of the reason for consultation:\n",
        "    - Existing products:\n",
        "    Do not mention the client's name, refrain from using a salutation.\n",
        "    Formulate the summary as continuous text and use varied sentence beginnings. Formulate the answer as a direct customer address. Start the summary with the phrase 'The reason for our conversation was'. Address the customer in the following sentences with 'you'.\n",
        "    Provide the information grammatically correctly and in English.\n",
        "    Avoid translating product names into English. Make sure that all product names are pronounced as they were mentioned in the conversation.\n",
        "    Avoid abbreviations. Spell out the entire word.\n",
        "    Base your answer on facts from the provided inputs and refer only to sources from the available fields.\n",
        "    Ensure that statements from the client and the advisor are clearly distinguished.\n",
        "    Think step-by-step and check if the results meet the above tasks.\n",
        "    \"\"\"\n",
        "    generation_model_summary = GenerativeModel(\n",
        "        \"gemini-2.0-flash\".replace(\"pro\", \"flash\" if quick else \"pro\")\n",
        "    )\n",
        "    generation_config_summary = GenerationConfig(\n",
        "        temperature=0.0, max_output_tokens=512, candidate_count=num_summaries\n",
        "    )\n",
        "    prompt = f\"\"\"{prompt_instr}\n",
        "    {context}\n",
        "    \"\"\"\n",
        "\n",
        "    generated_summaries = [\n",
        "        candidate.text\n",
        "        for candidate in generation_model_summary.generate_content(\n",
        "            contents=prompt,\n",
        "            generation_config=generation_config_summary,\n",
        "        ).candidates\n",
        "    ]\n",
        "    return prompt_instr, generated_summaries"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "693996feb2ef"
      },
      "source": [
        "### Step 5: Summary optimization by selecting the best one\n",
        "To ensure the highest quality output, the multiple summaries generated for each form field are [evaluated and the best summary for each field is selected](https://cloud.google.com/blog/products/ai-machine-learning/enhancing-llm-quality-and-interpretability-with-the-vertex-gen-ai-evaluation-service?e=48754805) using the Vertex AI Gen AI Evaluation Service. Importantly, the service also provides a human-readable explanation for its selection, enabling sales advisors to understand the reasoning behind the AI's choices and maintain trust in the automated process."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cf93d5f0ce00"
      },
      "outputs": [],
      "source": [
        "experiment_name = \"summarization-quality\"\n",
        "\n",
        "\n",
        "def pointwise_eval(\n",
        "    instruction: str,\n",
        "    context: str,\n",
        "    responses: list[str],\n",
        "    experiment_name: str = experiment_name,\n",
        "    eval_metrics: list[MetricPromptTemplateExamples.Pointwise] = [\n",
        "        MetricPromptTemplateExamples.Pointwise.SUMMARIZATION_QUALITY,\n",
        "        MetricPromptTemplateExamples.Pointwise.GROUNDEDNESS,\n",
        "    ],\n",
        ") -> vertexai.evaluation._base.EvalResult:\n",
        "    \"\"\"Takes the instruction, context and a variable number of corresponding generated responses, and returns the pointwise evaluation metrics\n",
        "    for each of the provided metrics. For this example the metrics are Q & A related, however the full list can be found on the website:\n",
        "    https://cloud.google.com/vertex-ai/generative-ai/docs/models/online-pipeline-services\n",
        "    \"\"\"\n",
        "    instructions = [instruction] * len(responses)\n",
        "\n",
        "    contexts = [context] * len(responses)\n",
        "\n",
        "    eval_dataset = pd.DataFrame(\n",
        "        {\n",
        "            \"instruction\": instructions,\n",
        "            \"context\": contexts,\n",
        "            \"response\": responses,\n",
        "        }\n",
        "    )\n",
        "\n",
        "    eval_task = EvalTask(\n",
        "        dataset=eval_dataset,\n",
        "        metrics=eval_metrics,\n",
        "        experiment=experiment_name,\n",
        "    )\n",
        "    results = eval_task.evaluate(\n",
        "        prompt_template=\"{instruction} \\n {context}\",\n",
        "        experiment_run_name=\"gemini-summ-pointwise-\" + str(uuid.uuid4()),\n",
        "    )\n",
        "\n",
        "    return results\n",
        "\n",
        "\n",
        "def pairwise_greater(\n",
        "    instructions: list,\n",
        "    context: str,\n",
        "    project_id: str,\n",
        "    location: str,\n",
        "    experiment_name: str,\n",
        "    baseline: str,\n",
        "    candidate: str,\n",
        ") -> tuple[str, str]:\n",
        "    \"\"\"Takes Instructions, Context and two different responses.\n",
        "    Returns the response which best matches the instructions/Context for the given\n",
        "    quality metric ( in this case question answering).\n",
        "    More details on the web API and different quality metrics which this function\n",
        "    can be extended to can be found on\n",
        "    https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation\n",
        "    \"\"\"\n",
        "    eval_dataset = pd.DataFrame(\n",
        "        {\n",
        "            \"instruction\": [instructions],\n",
        "            \"context\": [context],\n",
        "            \"response\": [candidate],\n",
        "            \"baseline_model_response\": [baseline],\n",
        "        }\n",
        "    )\n",
        "\n",
        "    eval_task = EvalTask(\n",
        "        dataset=eval_dataset,\n",
        "        metrics=[\n",
        "            MetricPromptTemplateExamples.Pairwise.SUMMARIZATION_QUALITY,\n",
        "        ],\n",
        "        experiment=experiment_name,\n",
        "    )\n",
        "    results = eval_task.evaluate(\n",
        "        prompt_template=\"{instruction} \\n {context}\",\n",
        "        experiment_run_name=\"gemini-summ-pairwise-\" + str(uuid.uuid4()),\n",
        "    )\n",
        "    result = results.metrics_table[\n",
        "        [\n",
        "            \"pairwise_summarization_quality/pairwise_choice\",\n",
        "            \"pairwise_summarization_quality/explanation\",\n",
        "        ]\n",
        "    ].to_dict(\"records\")[0]\n",
        "    choice = (\n",
        "        baseline\n",
        "        if result[\"pairwise_summarization_quality/pairwise_choice\"] == \"BASELINE\"\n",
        "        else candidate\n",
        "    )\n",
        "    return (choice, result[\"pairwise_summarization_quality/explanation\"])\n",
        "\n",
        "\n",
        "def greater(cmp: callable, a: str, b: str) -> int:\n",
        "    \"\"\"A comparison function which takes the comparison function, and two variables as input\n",
        "    and returns the one which is greater according to the logic defined inside the cmp function.\n",
        "    \"\"\"\n",
        "    choice, explanation = cmp(a, b)\n",
        "\n",
        "    if choice == a:\n",
        "        return 1\n",
        "    return -1\n",
        "\n",
        "\n",
        "def select_best_response(instruction, context, responses) -> tuple[str, dict]:\n",
        "    \"\"\"Takes the instruction, context and a variable number of responses as input, and returns the best performing response as well as its associated\n",
        "\n",
        "    human readable pointwise quality metrics for the configured criteria in the\n",
        "    above functions.\n",
        "    The process consists of two steps:\n",
        "    1. Selecting the best response by using Pairwise comparisons between the\n",
        "    responses for the user specified metric ( e.g. Q & A)\n",
        "    2. Doing pointwise evaluation of the best response and returning human\n",
        "    readable quality metrics and explanation along with the best response.\n",
        "    \"\"\"\n",
        "    cmp_f = partial(\n",
        "        pairwise_greater, instruction, context, PROJECT_ID, LOCATION, experiment_name\n",
        "    )\n",
        "    cmp_greater = partial(greater, cmp_f)\n",
        "\n",
        "    pairwise_best_response = max(responses, key=functools.cmp_to_key(cmp_greater))\n",
        "    pointwise_metric = pointwise_eval(\n",
        "        instruction, context, [pairwise_best_response], experiment_name\n",
        "    )\n",
        "    qa_metrics = pointwise_metric.metrics_table[\n",
        "        [\n",
        "            col\n",
        "            for col in pointwise_metric.metrics_table.columns\n",
        "            if (\"summarization\" in col) or (\"groundedness\" in col)\n",
        "        ]\n",
        "    ].to_dict(\"records\")[0]\n",
        "    return pairwise_best_response, qa_metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0fee5aabe694"
      },
      "source": [
        "We put all the steps together in one final step"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2a4e033321ad"
      },
      "source": [
        "## End-to-end execution for diarized transcription and summarization\n",
        "\n",
        "In this step you will run all the steps mentioned in sequence on a toy audio file.\n",
        "- First we do step 1 and 2: to chunk the audio file into smaller contiguous chunks, and then sequentially iterate over them to do advanced multimodal task/domain-specific diarized transcription. \n",
        "- Then step 3 to extract facts is performed."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2cdca4a8b090"
      },
      "outputs": [],
      "source": [
        "use_facts_for_summaries: bool = True\n",
        "# enter the bucket to store temporary chunked audio files\n",
        "output_bucket: str = \"gs://your_output_bucket\"\n",
        "audio_url = (\n",
        "    \"gs://github-repo/use-cases/diarization-transcription-summarization/rec1.wav\"\n",
        ")\n",
        "url_audio_chunks = chunk_audio(audio_url, output_bucket)\n",
        "print(url_audio_chunks)\n",
        "print(\"Finished audio chunking\")\n",
        "generated_transcription = diarize_transcribe(\n",
        "    url_audio_chunks,\n",
        "    path_example_transcription=\"gs://github-repo/use-cases/diarization-transcription-summarization/transcript_example.txt\",\n",
        "    path_example_audio=\"gs://github-repo/use-cases/diarization-transcription-summarization/rec2.wav\",\n",
        ")\n",
        "print(\"Finished transcription\")\n",
        "generated_facts = extract_facts(generated_transcription)\n",
        "print(\"Finished fact generation\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a593abeaf90b"
      },
      "outputs": [],
      "source": [
        "print(generated_transcription)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "04f75bdb19c4"
      },
      "outputs": [],
      "source": [
        "print(generated_facts)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4d3d3797784e"
      },
      "source": [
        "Then step 4 and 5 to generate the optimized summaries based on the transcript and facts generated."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4d7d0ecd83a0"
      },
      "outputs": [],
      "source": [
        "context_summary = (\n",
        "    f\"\"\"Transcript:\n",
        "{generated_transcription}\n",
        "Excerpt from the transcript containing facts:\n",
        "{generated_facts}\n",
        "\"\"\"\n",
        "    if use_facts_for_summaries\n",
        "    else f\"\"\"Transcript:\n",
        "    {generated_transcription}\n",
        "    \"\"\"\n",
        ")\n",
        "\n",
        "summary_prompt, generated_summaries = generate_summaries(context_summary, 3)\n",
        "best_summary = select_best_response(\n",
        "    summary_prompt, context_summary, generated_summaries\n",
        ")\n",
        "print(best_summary)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2fbfe877bcb9"
      },
      "source": [
        "## Cleanup\n",
        "In this step we delete the temporary files and experiments generated during this lab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9ea449f5e979"
      },
      "outputs": [],
      "source": [
        "aiplatform.init(project=PROJECT_ID, location=LOCATION)\n",
        "experiment = aiplatform.Experiment(experiment_name)\n",
        "experiment.delete()\n",
        "for url in url_audio_chunks:\n",
        "    bucket_name = url.replace(\"gs://\", \"\").split(\"/\")[0]\n",
        "    object_name = \"/\".join(url.replace(\"gs://\", \"\").split(\"/\")[1:])\n",
        "\n",
        "    # Get the bucket and delete the object\n",
        "    bucket = storage_client.bucket(bucket_name)\n",
        "    blob = bucket.blob(object_name)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "diarized_transcription_and_summarization.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
