{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ur8xi4C7S06n"
      },
      "outputs": [],
      "source": [
        "# Copyright 2024 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAPoU8Sm5E6e"
      },
      "source": [
        "# Data Augmentation for Text Data using BigQuery DataFrames\n",
        "\n",
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fuse-cases%2Fdata-augmentation%2Fdata_augmentation_for_text.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://console.cloud.google.com/bigquery/import?url=https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/bigquery/v1/32px.svg\" alt=\"BigQuery Studio logo\"><br> Open in BigQuery Studio\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://raw.githubusercontent.com/primer/octicons/refs/heads/main/icons/mark-github-24.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/data-augmentation/data_augmentation_for_text.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>            "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84f0f73a0f76"
      },
      "source": [
        "| | | |\n",
        "|-|-|-|\n",
        "|Author(s) | [Karl Weinmeister](https://github.com/kweinmeister)| [Kaz Sato](https://github.com/kazunori279)|"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tvgnzT1CKxrO"
      },
      "source": [
        "## Overview\n",
        "\n",
        "Data augmentation is a technique used to expand the size and diversity of a training dataset with synthetic data. This is particularly beneficial when dealing with limited datasets, which can lead to overfitting.\n",
        "\n",
        "It can be applied to multiple types of data through techniques including:\n",
        "* Text: Synonym replacement, word shuffling, or adding typos can be used to create new variations of text data.\n",
        "* Images: Flipping, rotating, cropping, scaling, adjusting brightness or contrast, and adding noise can all be used to generate new image variations.\n",
        "* Audio: Time-warping, adding noise, or pitch-shifting can be used to modify audio data.\n",
        "\n",
        "In this notebook, we will demonstrate 3 different techniques for augmenting data on text data: synonym replacement, back-translation, and noise injection.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61RBz8LLbxCR"
      },
      "source": [
        "## Getting Started"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "No17Cw5hgx12"
      },
      "source": [
        "### Install BigQuery DataFrames\n",
        "\n",
        "We will augment our data using Gemini for each of these techniques. Since we'll use a BigQuery public dataset, BigQuery DataFrames provides a natural interface to perform bulk LLM request operations on the original data.\n",
        "\n",
        "Learn more about [BigQuery DataFrames](https://cloud.google.com/python/docs/reference/bigframes/latest)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tFy3H3aPgx12"
      },
      "outputs": [],
      "source": [
        "%pip install --upgrade --user --quiet bigframes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5Xep4W9lq-Z"
      },
      "source": [
        "### Restart runtime (Colab only)\n",
        "\n",
        "To use the newly installed packages, you must restart the runtime on Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XRvKdaPDTznN"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    import IPython\n",
        "\n",
        "    app = IPython.Application.instance()\n",
        "    app.kernel.do_shutdown(True)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SbmM4z7FOBpM"
      },
      "source": [
        "<div class=\"alert alert-block alert-warning\">\n",
        "<b>⚠️ The kernel is going to restart. Please wait until it is finished before continuing to the next step. ⚠️</b>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dmWOrTJ3gx13"
      },
      "source": [
        "### Authenticate your notebook environment (Colab only)\n",
        "\n",
        "Authenticate your environment on Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NyKGtVQjgx13"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    from google.colab import auth\n",
        "\n",
        "    auth.authenticate_user()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ck1oBeRezgRe"
      },
      "source": [
        "### Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XiNwjR-FzfKm"
      },
      "outputs": [],
      "source": [
        "from bigframes.ml.llm import GeminiTextGenerator\n",
        "import bigframes.pandas as bpd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DF4l8DTdWgPY"
      },
      "source": [
        "### Set Google Cloud project information"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "311ef79ecd32"
      },
      "source": [
        "To get started using BigQuery DataFrames, you must have an existing Google Cloud project and [enable the BigQuery API](https://console.cloud.google.com/flows/enableapi?apiid=bigquery.googleapis.com)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Nqwi-5ufWp_B"
      },
      "outputs": [],
      "source": [
        "PROJECT_ID = \"your-project-id\"  # @param {type:\"string\"}\n",
        "LOCATION = \"US\"  # @param {type:\"string\"}\n",
        "\n",
        "# Note: The project option is not required in all environments.\n",
        "# On BigQuery Studio, the project ID is automatically detected.\n",
        "bpd.options.bigquery.project = PROJECT_ID\n",
        "\n",
        "# Note: The location option is not required.\n",
        "# It defaults to the location of the first table or query\n",
        "# passed to read_gbq(). For APIs where a location can't be\n",
        "# auto-detected, the location defaults to the \"US\" location.\n",
        "bpd.options.bigquery.location = LOCATION"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k8JDkJl_GBqe"
      },
      "source": [
        "## Load data\n",
        "\n",
        "We will use the StackOverflow data on BigQuery Public Datasets, limiting to questions with the `python` tag, and accepted answers for answers since 2020-01-01."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LBExdMfy910h"
      },
      "outputs": [],
      "source": [
        "stack_overflow_df = bpd.read_gbq_query(\n",
        "    \"\"\"SELECT\n",
        "           CONCAT(q.title, q.body) AS input_text,\n",
        "           a.body AS output_text\n",
        "       FROM `bigquery-public-data.stackoverflow.posts_questions` q\n",
        "       JOIN `bigquery-public-data.stackoverflow.posts_answers` a\n",
        "         ON q.accepted_answer_id = a.id\n",
        "       WHERE q.accepted_answer_id IS NOT NULL\n",
        "         AND REGEXP_CONTAINS(q.tags, \"python\")\n",
        "         AND a.creation_date >= \"2020-01-01\"\n",
        "       LIMIT 550\n",
        "    \"\"\"\n",
        ")\n",
        "\n",
        "stack_overflow_df.peek()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OjC20FslGH4u"
      },
      "source": [
        "## Synonym Replacement\n",
        "\n",
        "Synonym replacement involves replacing words in a sentence with their synonyms to create variations of the original text. This helps to introduce diversity into the dataset, which can improve the model's ability to generalize with unseen data. For instance, if the original sentence is \"Create easily interpretable topics with Large Language Models\", a synonym replacement approach might replace \"large\" with \"big\" to create \"Create easily interpretable topics with Big Language Models\"."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yDzSdEcwwXuF"
      },
      "outputs": [],
      "source": [
        "# Constants\n",
        "n_sample_dataset_rows = 10  # How many rows to sample for synonym replacement\n",
        "n_replacement_words = 5  # How many words per input text to replace\n",
        "\n",
        "# Sample a number of rows from the original dataframe\n",
        "df = stack_overflow_df.sample(n_sample_dataset_rows, random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91886f35ea7a"
      },
      "source": [
        "Define a Gemini text generator LLM [model](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.ml.llm.GeminiTextGenerator):"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BS100XYmztcN"
      },
      "outputs": [],
      "source": [
        "model = GeminiTextGenerator()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R2bFzvz7GqTF"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the synonym replacement instructions and the input text\n",
        "df[\"synonym_prompt\"] = (\n",
        "    f\"Replace {n_replacement_words} words from the input text with synonyms, \"\n",
        "    + \"keeping the overall meaning as close to the original text as possible.\"\n",
        "    + \"Only provide the synonymized text, with no additional explanation.\"\n",
        "    + \"Preserve the original formatting.\\n\\nInput text: \"\n",
        "    + df[\"input_text\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_with_synonyms\"] = model.predict(\n",
        "    df[\"synonym_prompt\"]\n",
        ").ml_generate_text_llm_result\n",
        "\n",
        "# Compare the original and new columns\n",
        "df.peek()[[\"input_text\", \"input_text_with_synonyms\"]]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YXR5VdHRSu8G"
      },
      "source": [
        "## Back translation\n",
        "\n",
        "Back translation is a data augmentation technique used in NLP to artificially expand your training dataset. It works by translating your text data into another language and then translating it back to the original language. This process injects variations into your data due to the imperfections of machine translation.\n",
        "\n",
        "Here's a breakdown of the steps involved:\n",
        "\n",
        "* Original Text: You start with your original text data that you want to augment.\n",
        "* Translation (Round Trip):  This text is then translated into a different language using a machine translation model.\n",
        "* Back Translation: The translated text is then translated back to the original language."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7fPqRYJUSxYX"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the translation instructions and the input text\n",
        "df[\"translation_prompt\"] = (\n",
        "    \"Translate the input text from English to Spanish. The text may include HTML characters; preserve them in the text.\\n\\nInput text: \"\n",
        "    + df[\"input_text\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_translated\"] = model.predict(\n",
        "    df[\"translation_prompt\"]\n",
        ").ml_generate_text_llm_result\n",
        "\n",
        "# Compare the original and new columns\n",
        "df.peek()[[\"input_text\", \"input_text_translated\"]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WE3M6GZWURlp"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the back-translation instructions and the input text\n",
        "df[\"backtranslation_prompt\"] = (\n",
        "    \"Translate the input text from Spanish to English. The text may include HTML characters; preserve them in the text.\\n\\nInput text: \"\n",
        "    + df[\"input_text_translated\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_backtranslated\"] = model.predict(\n",
        "    df[\"backtranslation_prompt\"]\n",
        ").ml_generate_text_llm_result\n",
        "\n",
        "# Compare the original and new columns\n",
        "df.peek()[[\"input_text\", \"input_text_translated\", \"input_text_backtranslated\"]]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2NxvIuMlWZMJ"
      },
      "source": [
        "## Noise injection\n",
        "\n",
        "Noise injection is the deliberate introduction of random perturbations or alterations into training data. The goal is to make the model more robust and prevent overfitting.\n",
        "\n",
        "Here we will apply a 3-step noise injection approach described in the paper \"[Understanding Back-Translation at Scale](https://arxiv.org/pdf/1808.09381.pdf)\" by Edunov et al. (2018)\n",
        "\n",
        "This process involves:\n",
        "1. Word Deletion: Randomly removing a percentage of words from the input text.\n",
        "1. Word Replacement: Replacing another percentage of words in the text with a filler token.\n",
        "1. Word Rearranging: Randomly shuffling the order of words, with the restriction that words can't be moved more than a specific distance from their original position."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ge19S-d5WmCo"
      },
      "outputs": [],
      "source": [
        "# Constants\n",
        "delete_ratio = 0.1\n",
        "filler_ratio = 0.1\n",
        "filler_token = \"BLANK\"\n",
        "swap_range = 3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3qld1BJgWf6G"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the delete instructions and the input text\n",
        "df[\"deletion_prompt\"] = (\n",
        "    f\"Delete words from the input text with probability {delete_ratio}. Preserve all HTML tags.\\n\\nInput text:\"\n",
        "    + df[\"input_text\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_deleted\"] = model.predict(\n",
        "    df[\"deletion_prompt\"]\n",
        ").ml_generate_text_llm_result\n",
        "\n",
        "# View results\n",
        "df.peek()[[\"input_text\", \"input_text_deleted\"]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tAYu1fHXYA42"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the filler instructions and the input text\n",
        "df[\"filler_prompt\"] = (\n",
        "    f\"Replace words from the input text with probability {filler_ratio} with this filler word {filler_token}. Preserve all HTML tags.\\n\\nInput text:\"\n",
        "    + df[\"input_text_deleted\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_filler\"] = model.predict(df[\"filler_prompt\"]).ml_generate_text_llm_result\n",
        "\n",
        "# View results\n",
        "df.peek()[[\"input_text\", \"input_text_filler\"]]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KsOsLugjYvAZ"
      },
      "outputs": [],
      "source": [
        "# Create a prompt with the swap instructions and the input text\n",
        "df[\"swap_prompt\"] = (\n",
        "    f\"Rearrange words from the input text no more than {swap_range} words apart. Preserve all HTML tags.\\n\\nInput text:\"\n",
        "    + df[\"input_text_filler\"]\n",
        ")\n",
        "\n",
        "# Run batch job and assign to a new column\n",
        "df[\"input_text_swap\"] = model.predict(df[\"swap_prompt\"]).ml_generate_text_llm_result\n",
        "\n",
        "# View results\n",
        "df.peek()[[\"input_text\", \"input_text_swap\"]]"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "data_augmentation_for_text.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
