{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5XNYlDkDLpqU"
      },
      "outputs": [],
      "source": [
        "# Copyright 2024 Google LLC\n",
        "#\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5tR528hOD4Dx"
      },
      "source": [
        "# Query-Level Boosting, Filtering, and Facets for Vertex AI Search Website Datastores\n",
        "\n",
        "<table align=\"left\">\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://art-analytics.appspot.com/r.html?uaid=G-FHXEFWTT4E&utm_source=aRT-vais-building-blocks&utm_medium=aRT-clicks&utm_campaign=vais-building-blocks&destination=vais-building-blocks&url=https%3A%2F%2Fcolab.research.google.com%2Fgithub%2FGoogleCloudPlatform%2Fapplied-ai-engineering-samples%2Fblob%2Fmain%2Fsearch%2Fvais-building-blocks%2Fquery_level_boosting_filtering_and_facets.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://art-analytics.appspot.com/r.html?uaid=G-FHXEFWTT4E&utm_source=aRT-vais-building-blocks&utm_medium=aRT-clicks&utm_campaign=vais-building-blocks&destination=vais-building-blocks&url=https%3A%2F%2Fconsole.cloud.google.com%2Fvertex-ai%2Fcolab%2Fimport%2Fhttps%3A%252F%252Fraw.githubusercontent.com%252FGoogleCloudPlatform%252Fapplied-ai-engineering-samples%252Fmain%252Fsearch%252Fvais-building-blocks%252Fquery_level_boosting_filtering_and_facets.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://art-analytics.appspot.com/r.html?uaid=G-FHXEFWTT4E&utm_source=aRT-vais-building-blocks&utm_medium=aRT-clicks&utm_campaign=vais-building-blocks&destination=vais-building-blocks&url=https%3A%2F%2Fconsole.cloud.google.com%2Fvertex-ai%2Fworkbench%2Fdeploy-notebook%3Fdownload_url%3Dhttps%3A%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fapplied-ai-engineering-samples%2Fmain%2Fsearch%2Fvais-building-blocks%2Fquery_level_boosting_filtering_and_facets.ipynb\">\n",
        "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
        "    </a>\n",
        "  </td>\n",
        "  <td style=\"text-align: center\">\n",
        "    <a href=\"https://github.com/GoogleCloudPlatform/applied-ai-engineering-samples/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\">\n",
        "      <img width=\"32px\" src=\"https://upload.wikimedia.org/wikipedia/commons/9/91/Octicons-mark-github.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
        "    </a>\n",
        "  </td>\n",
        "</table>\n",
        "\n",
        "<div style=\"clear: both;\"></div>\n",
        "\n",
        "<b>Share to:</b>\n",
        "\n",
        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/53/X_logo_2023_original.svg\" alt=\"X logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
        "</a>\n",
        "\n",
        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/search/vais-building-blocks/query_level_boosting_filtering_and_facets.ipynb\" target=\"_blank\">\n",
        "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
        "</a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pkd93iDpEBWx"
      },
      "source": [
        "| | |\n",
        "|----------|-------------|\n",
        "| Author(s)   | Hossein Mansour|\n",
        "| Reviewers(s) | Ismail Najim, Rajesh Thallam|\n",
        "| Last updated | 2024-09-06: The first draft |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yAnTektvEQjb"
      },
      "source": [
        "# Overview\n",
        "\n",
        "In this notebook, we demonstrate how to influence search results and their respective ranking by specifying [filters](https://cloud.google.com/generative-ai-app-builder/docs/filter-website-search#examples-advanced-indexing) and [boost rules](https://cloud.google.com/generative-ai-app-builder/docs/boost-search-results) within the request. Boosting and Filtering are typically used to improve precision and recall as well as removing certain pages from consideration to satisfy a user-specified preference (e.g. limit the results to movies and exclude TV series), or customer-specified preference (e.g. do not show this movie in search results before it's officially released, do not show specific results to users from a specific country, quickly exclude an noncompliant page from search results until it is properly removed from the index). User specified preferences are typically applied via facets in the UI, for that reason, we also cover facets in this notebook.\n",
        "\n",
        "Boosting and Filtering are applied at the data store/index level and as part of the retrieval process. For that reason, customers cannot achieve the same goal by post processing the search results.\n",
        "\n",
        "VAIS website search leverages a sophisticated algorithm and many signals to surface relevant results in the right order (similar to what you get in google.com), as a result it is generally advised to evaluate the results without additional rules and incrementally add custom rules only as needed.\n",
        "\n",
        "Also note that Boosting is one of many ways by which you can influence ranking and retrieval in VAIS. A few examples of alternative routes are:\n",
        "\n",
        "- [User Events](https://cloud.google.com/generative-ai-app-builder/docs/user-events) to implicitly and gradually tune the ranking based on end-user behavior\n",
        "- [Search Tuning](https://cloud.google.com/generative-ai-app-builder/docs/tune-search) to fine-tune the definition of \"relevant\" to your corpus, organization, domain, or preferences\n",
        "- [Custom Embeddings](https://cloud.google.com/generative-ai-app-builder/docs/bring-embeddings) to augment the ranking identified by VAIS\n",
        "- [Synonyms](https://cloud.google.com/generative-ai-app-builder/docs/configure-serving-controls#synonyms) to expand abbreviations and domain-specific terms based on their broadly-understood meaning   \n",
        "\n",
        "While these functionalities are available irrespective of the datastore type (e.g. structured, unstructured, or website), we limit our focus in this notebook to [Advanced Website datastores](https://cloud.google.com/generative-ai-app-builder/docs/about-advanced-features#advanced-website-indexing). Other than a few exceptions (e.g. filtering and boosting based on URL) most of the syntaxes presented here are applicable to other Datastore types as well.\n",
        "\n",
        "Note that there is an alternative way to apply [serving controls](https://cloud.google.com/generative-ai-app-builder/docs/configure-serving-controls) (i.e. Boosting, Filtering, Synonyms, and Redirects) at the Global level, which means they do not need to be provided together with each query. That alternative path is out of the scope for this notebook and will be covered in a separate one.\n",
        "\n",
        "In order to specify boosting and filtering, we need particular attributes for each document which act as the hooks to identify the right target documents and to apply the corresponding modifiers to them. While each page (i.e. document) within the index has some predefined fields (e.g. URL, datePublished, dateModified), it is common to [leverage metadata within the page-source](https://cloud.google.com/generative-ai-app-builder/docs/add-website-metadata) as additional hooks. In order to identify those metadata, we need to update the Datastore Schema which is also covered in this notebook.\n",
        "\n",
        "We will perform the following steps:\n",
        "\n",
        "- [Prerequisite] Creating a Vertex AI Search Website Datastore and Search App via API\n",
        "- Updating the Schema to identify page Metadata\n",
        "- Filtering based on predefined fields\n",
        "- Filtering based on page Metadata\n",
        "- Defining Facets based on page Metadata\n",
        "- Basic boosting\n",
        "- A sample advanced boosting based on user ratings\n",
        "- Clean up\n",
        "\n",
        "\n",
        "Please refer to the [official documentation](https://cloud.google.com/generative-ai-app-builder/docs/create-datastore-ingest) for the definition of Datastores and Apps and their relationships to one another\n",
        "\n",
        "REST API is used throughout this notebook. Please consult the [official documentation](https://cloud.google.com/generative-ai-app-builder/docs/apis) for alternative ways to achieve the same goal, namely Client libraries and RPC.\n",
        "\n",
        "\n",
        "# Vertex AI Search\n",
        "Vertex AI Search (VAIS) is a fully-managed platform, powered by large language models, that lets you build AI-enabled search and recommendation experiences for your public or private websites or mobile applications\n",
        "\n",
        "VAIS can handle a diverse set of data sources including structured, unstructured, and website data, as well as data from third-party applications such as Jira, Salesforce, and Confluence.\n",
        "\n",
        "VAIS also has built-in integration with LLMs which enables you to provide answers to complex questions, grounded in your data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nszpkYyzAqpA"
      },
      "source": [
        "# Using this Notebook\n",
        "If you're running outside of Colab, depending on your environment you may need to install pip packages that are included in the Colab environment by default but are not part of the Python Standard Library. Outside of Colab you'll also notice comments in code cells that look like #@something, these trigger special Colab functionality but don't change the behavior of the notebook.\n",
        "\n",
        "This tutorial uses the following Google Cloud services and resources:\n",
        "\n",
        "- Service Usage API\n",
        "- Discovery Engine API\n",
        "\n",
        "This notebook has been tested in the following environment:\n",
        "\n",
        "- Python version = 3.10.12\n",
        "- google.cloud.storage = 2.8.0\n",
        "- google.auth = 2.27.0"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NZKU9BnbA0xz"
      },
      "source": [
        "## Getting Started\n",
        "\n",
        "The following steps are necessary to run this notebook, no matter what notebook environment you're using.\n",
        "\n",
        "If you're entirely new to Google Cloud, [get started here](https://cloud.google.com/docs/get-started)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HNxgWBHqA5CF"
      },
      "source": [
        "## Google Cloud Project Setup\n",
        "\n",
        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs\n",
        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
        "3. [Enable the Service Usage API](https://console.cloud.google.com/apis/library/serviceusage.googleapis.com)\n",
        "4. [Enable the Cloud Storage API](https://console.cloud.google.com/flows/enableapi?apiid=storage.googleapis.com)\n",
        "5. [Enable the Discovery Engine API for your project](https://console.cloud.google.com/marketplace/product/google/discoveryengine.googleapis.com)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F2vGcA6QA6xG"
      },
      "source": [
        "## Google Cloud Permissions\n",
        "\n",
        "Ideally you should have [Owner role](https://cloud.google.com/iam/docs/understanding-roles) for your project to run this notebook. If that is not an option, you need at least the following [roles](https://cloud.google.com/iam/docs/granting-changing-revoking-access)\n",
        "- **`roles/serviceusage.serviceUsageAdmin`** to enable APIs\n",
        "- **`roles/iam.serviceAccountAdmin`** to modify service agent permissions\n",
        "- **`roles/discoveryengine.admin`** to modify discoveryengine assets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "49x_J4vWOuNg"
      },
      "source": [
        "#Setup Environment"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kMYYfGpyOl5G"
      },
      "source": [
        "## Authentication\n",
        "\n",
        " If you're using Colab, run the code in the next cell. Follow the pop-ups and authenticate with an account that has access to your Google Cloud [project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects).\n",
        "\n",
        "If you're running this notebook somewhere besides Colab, make sure your environment has the right Google Cloud access. If that's a new concept to you, consider looking into [Application Default Credentials for your local environment](https://cloud.google.com/docs/authentication/provide-credentials-adc#local-dev) and [initializing the Google Cloud CLI](https://cloud.google.com/docs/authentication/gcloud). In many cases, running `gcloud auth application-default login` in a shell on the machine running the notebook kernel is sufficient.\n",
        "\n",
        "More authentication options are discussed [here](https://cloud.google.com/docs/authentication)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DZjtfEDG7Sr3"
      },
      "outputs": [],
      "source": [
        "# Colab authentication.\n",
        "import sys\n",
        "\n",
        "if \"google.colab\" in sys.modules:\n",
        "    from google.colab import auth\n",
        "\n",
        "    auth.authenticate_user()\n",
        "    print(\"Authenticated\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kT3Eda7_mlTP"
      },
      "outputs": [],
      "source": [
        "from google.auth import default\n",
        "from google.auth.transport.requests import AuthorizedSession\n",
        "\n",
        "creds, _ = default()\n",
        "authed_session = AuthorizedSession(creds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "otijhCIjOzk-"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DlIp4zv3cdA7"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "import time"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N51y_mPgPHsj"
      },
      "source": [
        "## Configure environment\n",
        "\n",
        "`DATASTORE_ID` and `APP_ID` must match the pattern: [a-z0-9][a-z0-9-_]*\n",
        "\n",
        "The Location of a Datastore is set at the time of creation and it should be called appropriately to query the Datastore. `global` is typically recommended unless you have a particular reason to use a regional Datastore.\n",
        "\n",
        "You can find more information regarding the `Location` of datastores and associated limitations [here](https://cloud.google.com/generative-ai-app-builder/docs/locations#specify_a_multi-region_for_your_data_store).\n",
        "\n",
        "`VAIS_BRANCH` is the branch of VAIS to use.\n",
        "\n",
        "\n",
        "`INCLUDE_URL_PATTERN` is the pattern of a website to be included in the datastore, e.g. \"www.example.com/*\", \"www.example.com/abc/*\".\n",
        "\n",
        "For this particular example we Index books on Google Play Store. To keep the size of the index manageable, we only include books with \"The\" in their title. The corresponding URL pattern to include looks like: \"play.google.com/store/books/details/\\*_The_\\*\"\n",
        "\n",
        "Note that you need to [verify the ownership of a domain](https://cloud.google.com/generative-ai-app-builder/docs/domain-verification) to be able to index it."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hKLBf1GqROW7"
      },
      "outputs": [],
      "source": [
        "PROJECT_ID = \"\"  # @param {type: 'string'}\n",
        "DATASTORE_ID = \"\"  # @param {type: 'string'}\n",
        "APP_ID = \"\"  # @param {type: 'string'}\n",
        "LOCATION = \"global\"  # @param [\"global\", \"us\", \"eu\"]\n",
        "VAIS_BRANCH = \"v1alpha\"  # @param [\"v1\", \"v1beta\", \"v1alpha\"]\n",
        "INCLUDE_URL_PATTERN = \"\"  # @param {type: 'string'}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Akk3C5vK8oG6"
      },
      "source": [
        "# Step 1. [Prerequisite] Create a Website Search Datastore and APP\n",
        "In this section we will programmatically create a VAIS [Advanced Website Datastore and APP](https://cloud.google.com/generative-ai-app-builder/docs/about-advanced-features#advanced-website-indexing). You can achieve the same goal with a [few clicks](https://cloud.google.com/generative-ai-app-builder/docs/website-search-checklist?indexing=advanced) in the UI.\n",
        "\n",
        "If you already have an Advanced Website Datastore available, you can skip this section.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C2hXlewDINDg"
      },
      "source": [
        "## Helper functions to issue basic search on a Datastore or an App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v-XHQIOooshe"
      },
      "outputs": [],
      "source": [
        "def search_by_datastore(project_id: str, location: str, datastore_id: str, query: str):\n",
        "    \"\"\"Searches a datastore using the provided query.\"\"\"\n",
        "    response = authed_session.post(\n",
        "        f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{project_id}/locations/{location}/collections/default_collection/dataStores/{datastore_id}/servingConfigs/default_search:search\",\n",
        "        headers={\n",
        "            \"Content-Type\": \"application/json\",\n",
        "        },\n",
        "        json={\"query\": query, \"pageSize\": 1},\n",
        "    )\n",
        "    return response\n",
        "\n",
        "\n",
        "def search_by_app(project_id: str, location: str, app_id: str, query: str):\n",
        "    \"\"\"Searches an app using the provided query.\"\"\"\n",
        "    response = authed_session.post(\n",
        "        f\"https://discoveryengine.googleapis.com/v1/projects/{project_id}/locations/{location}/collections/default_collection/engines/{app_id}/servingConfigs/default_config:search\",\n",
        "        headers={\n",
        "            \"Content-Type\": \"application/json\",\n",
        "        },\n",
        "        json={\"query\": query, \"pageSize\": 1},\n",
        "    )\n",
        "    return response"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eAigF6KHkMZ2"
      },
      "source": [
        "## Helper functions to check whether or not a Datastore or an App already exist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IO1AxLZckXYK"
      },
      "outputs": [],
      "source": [
        "def datastore_exists(project_id: str, location: str, datastore_id: str) -> bool:\n",
        "    \"\"\"Check if a datastore exists.\"\"\"\n",
        "    response = search_by_datastore(project_id, location, datastore_id, \"test\")\n",
        "    status_code = response.status_code\n",
        "    # A 400 response is expected as the URL pattern needs to be set first\n",
        "    if status_code == 200 or status_code == 400:\n",
        "        return True\n",
        "    if status_code == 404:\n",
        "        return False\n",
        "    raise Exception(f\"Error: {status_code}\")\n",
        "\n",
        "\n",
        "def app_exists(project_id: str, location: str, app_id: str) -> bool:\n",
        "    \"\"\"Check if an App exists.\"\"\"\n",
        "    response = search_by_app(project_id, location, app_id, \"test\")\n",
        "    status_code = response.status_code\n",
        "    if status_code == 200:\n",
        "        return True\n",
        "    if status_code == 404:\n",
        "        return False\n",
        "    raise Exception(f\"Error: {status_code}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DYArgsiAiVfs"
      },
      "source": [
        "## Helper functions to create a Datastore or an App"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_0uAQTKD78_k"
      },
      "outputs": [],
      "source": [
        "def create_website_datastore(\n",
        "    vais_branch: str, project_id: str, location: str, datastore_id: str\n",
        ") -> int:\n",
        "    \"\"\"Create a website datastore\"\"\"\n",
        "    payload = {\n",
        "        \"displayName\": datastore_id,\n",
        "        \"industryVertical\": \"GENERIC\",\n",
        "        \"solutionTypes\": [\"SOLUTION_TYPE_SEARCH\"],\n",
        "        \"contentConfig\": \"PUBLIC_WEBSITE\",\n",
        "    }\n",
        "    header = {\"X-Goog-User-Project\": project_id, \"Content-Type\": \"application/json\"}\n",
        "    es_endpoint = f\"https://discoveryengine.googleapis.com/{vais_branch}/projects/{project_id}/locations/{location}/collections/default_collection/dataStores?dataStoreId={datastore_id}\"\n",
        "    response = authed_session.post(\n",
        "        es_endpoint, data=json.dumps(payload), headers=header\n",
        "    )\n",
        "    if response.status_code == 200:\n",
        "        print(f\"The creation of Datastore {datastore_id} is initiated.\")\n",
        "        print(\"It may take a few minutes for the Datastore to become available\")\n",
        "    else:\n",
        "        print(f\"Failed to create Datastore {datastore_id}\")\n",
        "        print(response.json())\n",
        "    return response.status_code\n",
        "\n",
        "\n",
        "def create_app(\n",
        "    vais_branch: str, project_id: str, location: str, datastore_id: str, app_id: str\n",
        ") -> int:\n",
        "    \"\"\"Create a search app.\"\"\"\n",
        "    payload = {\n",
        "        \"displayName\": app_id,\n",
        "        \"dataStoreIds\": [datastore_id],\n",
        "        \"solutionType\": \"SOLUTION_TYPE_SEARCH\",\n",
        "        \"searchEngineConfig\": {\n",
        "            \"searchTier\": \"SEARCH_TIER_ENTERPRISE\",\n",
        "            \"searchAddOns\": [\"SEARCH_ADD_ON_LLM\"],\n",
        "        },\n",
        "    }\n",
        "    header = {\"X-Goog-User-Project\": project_id, \"Content-Type\": \"application/json\"}\n",
        "    es_endpoint = f\"https://discoveryengine.googleapis.com/{vais_branch}/projects/{project_id}/locations/{location}/collections/default_collection/engines?engineId={app_id}\"\n",
        "    response = authed_session.post(\n",
        "        es_endpoint, data=json.dumps(payload), headers=header\n",
        "    )\n",
        "    if response.status_code == 200:\n",
        "        print(f\"The creation of App {app_id}  is initiated.\")\n",
        "        print(\"It may take a few minutes for the App to become available\")\n",
        "    else:\n",
        "        print(f\"Failed to create App {app_id}\")\n",
        "        print(response.json())\n",
        "    return response.status_code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1hAp5cBnIYxJ"
      },
      "source": [
        "## Create a Datastores with the provided ID if it doesn't exist\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hBUwJxxeAazj"
      },
      "outputs": [],
      "source": [
        "if datastore_exists(PROJECT_ID, LOCATION, DATASTORE_ID):\n",
        "    print(f\"Datastore {DATASTORE_ID} already exists.\")\n",
        "else:\n",
        "    create_website_datastore(VAIS_BRANCH, PROJECT_ID, LOCATION, DATASTORE_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C1d-pd2WLJZI"
      },
      "source": [
        "## [Optional] Check if the Datastore is created successfully\n",
        "\n",
        "\n",
        "The Datastore is polled to track when it becomes available.\n",
        "\n",
        "This may take a few minutes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EZGzOCnTLOwf"
      },
      "outputs": [],
      "source": [
        "while not datastore_exists(PROJECT_ID, LOCATION, DATASTORE_ID):\n",
        "    print(f\"Datastore {DATASTORE_ID} is still being created.\")\n",
        "    time.sleep(30)\n",
        "print(f\"Datastore {DATASTORE_ID} is created successfully.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vSzz2AzmI5kx"
      },
      "source": [
        "## Create an App with the provided ID if it doesn't exist\n",
        "The App will be connected to a Datastore with the ID provided earlier in this notebook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4lp4kPXNm9sE"
      },
      "outputs": [],
      "source": [
        "if app_exists(PROJECT_ID, LOCATION, APP_ID):\n",
        "    print(f\"App {APP_ID} already exists.\")\n",
        "else:\n",
        "    create_app(VAIS_BRANCH, PROJECT_ID, LOCATION, DATASTORE_ID, APP_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fxlTn7dVK-Q2"
      },
      "source": [
        "## [Optional] Check if the App is created successfully\n",
        "\n",
        "\n",
        "The App is polled to track when it becomes available.\n",
        "\n",
        "This may take a few minutes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZuQQ2HCGK4BA"
      },
      "outputs": [],
      "source": [
        "while not app_exists(PROJECT_ID, LOCATION, APP_ID):\n",
        "    print(f\"App {APP_ID} is still being created.\")\n",
        "    time.sleep(30)\n",
        "print(f\"App {APP_ID} is created successfully.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A38IfFRD83UG"
      },
      "source": [
        "## Upgrade an existing Website Datastore to [Advanced Website](https://cloud.google.com/generative-ai-app-builder/docs/about-advanced-features#advanced-website-indexing) DataStore"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BYXR-yQ38vdd"
      },
      "outputs": [],
      "source": [
        "def upgrade_to_advanced(\n",
        "    vais_branch: str, project_id: str, location: str, datastore_id: str\n",
        ") -> int:\n",
        "    \"\"\"Upgrade the website search datastore to advanced\"\"\"\n",
        "    header = {\"X-Goog-User-Project\": project_id}\n",
        "    es_endpoint = f\"https://discoveryengine.googleapis.com/{vais_branch}/projects/{project_id}/locations/{location}/collections/default_collection/dataStores/{datastore_id}/siteSearchEngine:enableAdvancedSiteSearch\"\n",
        "    response = authed_session.post(es_endpoint, headers=header)\n",
        "    if response.status_code == 200:\n",
        "        print(f\"Datastore {datastore_id} upgraded to Advanced Website Search\")\n",
        "    else:\n",
        "        print(f\"Failed to upgrade Datastore {datastore_id}\")\n",
        "        print(response.text())\n",
        "    return response.status_code\n",
        "\n",
        "\n",
        "upgrade_to_advanced(VAIS_BRANCH, PROJECT_ID, LOCATION, DATASTORE_ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NlUq4ADT8975"
      },
      "source": [
        "## Set the URLs to Include/Exclude in the Index\n",
        "\n",
        "You can set up to 500 Include and Exclude URL patterns for Advanced website search Datastores.\n",
        "\n",
        "This function sets a single URL pattern to be included every time it gets executed.\n",
        "\n",
        "The field `type` in the payload is used to indicate if the provided Uri pattern should be included or excluded. Here we only use `INCLUDE`.\n",
        "\n",
        "The `INCLUDE` and `EXCLUDE` URL patterns specified with this function are incremental. You also have options to [Delete](https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1alpha/projects.locations.collections.dataStores.siteSearchEngine.targetSites/delete), [List](https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1alpha/projects.locations.collections.dataStores.siteSearchEngine.targetSites/list), [Batch Create](https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1alpha/projects.locations.collections.dataStores.siteSearchEngine.targetSites/batchCreate), etc  \n",
        "\n",
        "For this example, we index \"play.google.com/store/books/details/\\*_The_\\*\"\n",
        "\n",
        "Note that you need to [verify the ownership of a domain](https://cloud.google.com/generative-ai-app-builder/docs/domain-verification) to be able to index it."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yc2OWvFd9Tvu"
      },
      "outputs": [],
      "source": [
        "def include_url_patterns(\n",
        "    vais_branch: str,\n",
        "    project_id: str,\n",
        "    location: str,\n",
        "    datastore_id: str,\n",
        "    include_url_patterns,\n",
        ") -> int:\n",
        "    \"\"\"Set include and exclude URL patterns for the Datastore\"\"\"\n",
        "    payload = {\n",
        "        \"providedUriPattern\": include_url_patterns,\n",
        "        \"type\": \"INCLUDE\",\n",
        "    }\n",
        "    header = {\"X-Goog-User-Project\": project_id, \"Content-Type\": \"application/json\"}\n",
        "    es_endpoint = f\"https://discoveryengine.googleapis.com/{vais_branch}/projects/{project_id}/locations/{location}/dataStores/{datastore_id}/siteSearchEngine/targetSites\"\n",
        "    response = authed_session.post(\n",
        "        es_endpoint, data=json.dumps(payload), headers=header\n",
        "    )\n",
        "    if response.status_code == 200:\n",
        "        print(\"URL patterns successfully set\")\n",
        "        print(\n",
        "            \"Depending on the size of your domain, the initial indexing may take from minutes to hours\"\n",
        "        )\n",
        "    else:\n",
        "        print(f\"Failed to set URL patterns for the Datastore {datastore_id}\")\n",
        "        print(response.text())\n",
        "    return response.status_code\n",
        "\n",
        "\n",
        "include_url_patterns(\n",
        "    VAIS_BRANCH, PROJECT_ID, LOCATION, DATASTORE_ID, INCLUDE_URL_PATTERN\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rSAbsrg8Pkc2"
      },
      "source": [
        "# Step 2. Update the Schema to include page Metadata"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dc-gaZ6rP6mC"
      },
      "source": [
        "## Set the Schema\n",
        "\n",
        "In this example, we use [Books on Google Play Store](https://play.google.com/store/books) as the source for the datastore.\n",
        "\n",
        "In addition to the properties of the pages which are available by default (e.g. Date Published, Date Modified, URL, and Title), we are also interest in a few other properties such as Rating Count, Average Rating, Price, and Date Published (i.e. the actual publication date of the book, not the page of Play Store). VAIS extracts this additional information for each URL within the datastore upon appropriately [updating the Schema](https://cloud.google.com/generative-ai-app-builder/docs/provide-schema).\n",
        "\n",
        "At the time of creating this notebook, VAIS supports three types of Metadata within the page source: [Meta Tags](https://cloud.google.com/generative-ai-app-builder/docs/add-website-metadata#example-meta-tags), [PageMap](https://cloud.google.com/generative-ai-app-builder/docs/add-website-metadata#example-pagemaps), and [Schema.org](https://cloud.google.com/generative-ai-app-builder/docs/add-website-metadata#example-schema-org)\n",
        "\n",
        "For this example we extract the Description field from Meta tags and the other fields from Schema.org. You are encouraged to check the page source for a [sample app](https://play.google.com/store/books/details/Margaret_Atwood_The_Testaments?id=P6F7DwAAQBAJ) to see how these fields are defined in our target domain.\n",
        "\n",
        "As mentioned above, you can only index a website you own, as a result the metadata defined on your Datastore will be different with the ones defined in this example.\n",
        "\n",
        "Updating the Schema will trigger an update to the index which may take a few hours to complete.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TDpAyVAUbxXM"
      },
      "outputs": [],
      "source": [
        "header = {\"X-Goog-User-Project\": PROJECT_ID}\n",
        "es_endpoint = f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/schemas/default_schema\"\n",
        "json_data = {\n",
        "    \"structSchema\": {\n",
        "        \"type\": \"object\",\n",
        "        \"properties\": {\n",
        "            \"aggregate_rating\": {\n",
        "                \"type\": \"array\",\n",
        "                \"items\": {\n",
        "                    \"type\": \"number\",\n",
        "                    \"retrievable\": True,\n",
        "                    \"indexable\": True,\n",
        "                    \"dynamicFacetable\": True,\n",
        "                    \"siteSearchSchemaOrgPaths\": [\"_root.aggregateRating.ratingValue\"],\n",
        "                },\n",
        "            },\n",
        "            \"rating_count\": {\n",
        "                \"type\": \"array\",\n",
        "                \"items\": {\n",
        "                    \"type\": \"number\",\n",
        "                    \"retrievable\": True,\n",
        "                    \"indexable\": True,\n",
        "                    \"dynamicFacetable\": True,\n",
        "                    \"siteSearchSchemaOrgPaths\": [\"_root.aggregateRating.ratingCount\"],\n",
        "                },\n",
        "            },\n",
        "            \"price\": {\n",
        "                \"type\": \"array\",\n",
        "                \"items\": {\n",
        "                    \"type\": \"number\",\n",
        "                    \"retrievable\": True,\n",
        "                    \"indexable\": True,\n",
        "                    \"dynamicFacetable\": True,\n",
        "                    \"siteSearchSchemaOrgPaths\": [\n",
        "                        \"_root.workExample.potentialAction.expectsAcceptanceOf.price\"\n",
        "                    ],\n",
        "                },\n",
        "            },\n",
        "            \"author\": {\n",
        "                \"type\": \"array\",\n",
        "                \"items\": {\n",
        "                    \"type\": \"string\",\n",
        "                    \"retrievable\": True,\n",
        "                    \"indexable\": True,\n",
        "                    \"dynamicFacetable\": True,\n",
        "                    \"siteSearchSchemaOrgPaths\": [\"_root.author.name\"],\n",
        "                },\n",
        "            },\n",
        "            \"date_published\": {\n",
        "                \"type\": \"array\",\n",
        "                \"items\": {\n",
        "                    \"type\": \"datetime\",\n",
        "                    \"retrievable\": True,\n",
        "                    \"indexable\": True,\n",
        "                    \"siteSearchSchemaOrgPaths\": [\"_root.workExample.datePublished\"],\n",
        "                },\n",
        "            },\n",
        "        },\n",
        "        \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n",
        "    }\n",
        "}\n",
        "\n",
        "set_schema_response = authed_session.patch(es_endpoint, headers=header, json=json_data)\n",
        "\n",
        "print(json.dumps(set_schema_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xRCbxXEG2XmF"
      },
      "source": [
        "## [optional] Get the Schema\n",
        "\n",
        "Get the Schema and URL mapping to ensure it is updated according to your expectations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7TXWY_6nTH3u"
      },
      "outputs": [],
      "source": [
        "header = {\"X-Goog-User-Project\": PROJECT_ID}\n",
        "es_endpoint = f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/schemas/default_schema\"\n",
        "get_schema_response = authed_session.get(es_endpoint, headers=header)\n",
        "\n",
        "print(json.dumps(get_schema_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vESfZZ_QDLc8"
      },
      "source": [
        "# Step 3. Results w/wo Filtering"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nnnxTO8CC9Mz"
      },
      "source": [
        "## Search Without Filter\n",
        "Let's start by making a simple search on the datastore\n",
        "\n",
        "Note that the `Retreivable` Metadata fields defined in the schema are included in the `structData` field of the `result`).\n",
        "\n",
        "In our example, we issue the query \"house\" with a page size of 1, and get the following result:\n",
        "![basic_search.png](https://storage.googleapis.com/github-repo/generative-ai/search/vais-building-blocks/query_level_boosting_filtering_and_facets/basic_search.png)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Usr8OMTu5EUk"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\"query\": QUERY, \"pageSize\": PAGE_SIZE},\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uCRfkzGyC53w"
      },
      "source": [
        "## Search with Filter on a predefined field\n",
        "Now let's apply a simple filter based on URL pattern. This is typically useful when your domain of interest has an interesting categorization of subdomains. Since the Google App Store used for our example doesn't have subdomains, we specify a certain pattern to only retrieve books with \"the roots\" in their title. The corresponding pattern will look like: \"play.google.com/store/books/details/\\*_The_roots_\\*\". We search for a generic Query \"Books\" but only get results with \"_The_root_\" in their URLs.\n",
        "\n",
        "See more examples of filtering based on URLs [here](https://cloud.google.com/generative-ai-app-builder/docs/filter-website-search#examples-advanced-indexing)\n",
        "\n",
        "You can also access [Google-inferred page date](https://cloud.google.com/generative-ai-app-builder/docs/boost-search-results#predefined-date-example) (i.e. datePublished and dateModified) which you can similarly use for filtering and boosting without a need for a Schema update.\n",
        "\n",
        "Note that you ger a different (and a more comprehensive) set of predefined fields for [basic website search](https://cloud.google.com/generative-ai-app-builder/docs/filter-website-search#filter-expressions-basic-indexing)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qatUukazC4oH"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\n",
        "        \"query\": QUERY,\n",
        "        # Update this filter based on the structure of your domain/subdomains\n",
        "        \"filter\": 'siteSearch:\"https://play.google.com/store/books/details/*_The_roots_*\"',\n",
        "        \"pageSize\": PAGE_SIZE,\n",
        "    },\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gxblV5Bgx1fE"
      },
      "source": [
        "## Search with Filter on a user-defined metadata\n",
        "Next, let's apply sample filters based on user-defined metadata.\n",
        "\n",
        "In this example we limit our search to highly-rated free books. Let's also set a threshold for the number of ratings to make sure \"high rating\" is meaningful and reliable.  (i.e. price < 10 AND aggregate_rating > 4.5 AND rating_count > 10).\n",
        "\n",
        "You can find more details on [Filter expression syntax](https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata#filter-expression-syntax)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q8k4Z07kx1fF"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\n",
        "        \"query\": QUERY,\n",
        "        # Update this filter definition based on your use case and metadata\n",
        "        \"filter\": \"rating_count>10 AND aggregate_rating>4.5 AND price=0\",\n",
        "        \"pageSize\": PAGE_SIZE,\n",
        "    },\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeomJ_G-gcA9"
      },
      "source": [
        "# Step 4. Get Facets from Document Metadata\n",
        "\n",
        "Facets are used to enhance user experience by providing UI elements that allow users to narrow down their search universe. They are most commonly found on retail website where you can choose your brand, size, color, etc after making an initial search (or even without putting in any queries as you \"Browse\" the landing page).\n",
        "\n",
        "Typically upon selection of a particular facet by the end user, you will issue a subsequent search with a corresponding filter added to update the results accordingly. Each facet response contains the syntax of its corresponding filter as well.\n",
        "\n",
        "![facets.png](https://storage.googleapis.com/github-repo/generative-ai/search/vais-building-blocks/query_level_boosting_filtering_and_facets/facets.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rUtpm0E_mK-f"
      },
      "source": [
        "## Fixed and Dynamic facets\n",
        "\n",
        "In order to get Facets in the response, you need to specify `facetSpecs` in the request. You can find more details in the [documentation](https://cloud.google.com/generative-ai-app-builder/docs/reference/rest/v1/FacetSpec).\n",
        "\n",
        "Each facet within the list of `facetSpecs` can have a fixed, or a dynamic positioning.\n",
        "\n",
        "In the response, facets with fixed positioning (i.e. enableDynamicPosition = False) will always show up at top with the same ordering as in the request. The Dynamics facets are ordered lower, with their relative ordering decided based on the query and likelihood of users being interested in them.\n",
        "\n",
        "For the fields to be eligible for Dynamic faceting, they should be specified as both indexable and dynamicFacetable in the [Schema](https://cloud.google.com/generative-ai-app-builder/docs/provide-schema). You also need to send [user events](https://cloud.google.com/generative-ai-app-builder/docs/user-events?hl=en) to make effective use of the facets.  \n",
        "\n",
        "Below is a screenshot of the facet part of the response for a sample query used here:\n",
        "\n",
        "![facet_response.png](https://storage.googleapis.com/github-repo/generative-ai/search/vais-building-blocks/query_level_boosting_filtering_and_facets/facet_response.png)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GHDWUXW9xiAb"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\n",
        "        \"query\": QUERY,\n",
        "        # Update this facet definition based on your use case and metadata\n",
        "        \"facetSpecs\": [\n",
        "            {\"facetKey\": {\"key\": \"author\"}, \"limit\": 2, \"enableDynamicPosition\": False},\n",
        "            {\n",
        "                \"facetKey\": {\n",
        "                    \"key\": \"aggregate_rating\",\n",
        "                    \"intervals\": [\n",
        "                        {\"minimum\": 0, \"maximum\": 3},\n",
        "                        {\"minimum\": 3, \"maximum\": 4.5},\n",
        "                        {\"minimum\": 4.5, \"maximum\": 5},\n",
        "                    ],\n",
        "                },\n",
        "                \"limit\": 3,\n",
        "                \"enableDynamicPosition\": True,\n",
        "            },\n",
        "            {\n",
        "                \"facetKey\": {\n",
        "                    \"key\": \"rating_count\",\n",
        "                    \"intervals\": [\n",
        "                        {\"minimum\": 0, \"maximum\": 10},\n",
        "                        {\"minimum\": 10, \"maximum\": 100},\n",
        "                        {\"minimum\": 100},\n",
        "                    ],\n",
        "                },\n",
        "                \"limit\": 3,\n",
        "                \"enableDynamicPosition\": True,\n",
        "            },\n",
        "        ],\n",
        "        \"pageSize\": PAGE_SIZE,\n",
        "    },\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cpzZqjnc4M3g"
      },
      "source": [
        "# Step 5. Influence the Ranking via Boosting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "POtXQbRBc-SZ"
      },
      "source": [
        "## Basic Boosting\n",
        "As a basic demonstration of Boosting, let's look at a constant boost value on all documents meeting a certain criteria. A basic Boost has a condition (with the same syntax as [filters](https://cloud.google.com/generative-ai-app-builder/docs/filter-search-metadata#filter-expression-syntax)) and a Boost value.\n",
        "\n",
        "Boost value should be a number between -1.0 and +1.0 where negative numbers demote the matched documents (a.k.a. Bury). The boost function behaves roughly exponentially.\n",
        "\n",
        "It is generally advised to start with smaller boost values and adjust it as needed. If a document gets hit by several Boost conditions the boost amounts are additive.\n",
        "\n",
        "In this example we boost all the books written by \"Margaret Atwood\". The boost value in this example is set to 0.9. With this boost we get books by \"Margaret Atwood\" for a generic query like \"Book\", but if you search for a particular title not written by Margaret Atwood (e.g. \"house of cards\") you'd still get that title as the top result. to put it in physics terms, you can think of Boosting as a forcing function whereas Filters are constraints.\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jQuTzkiuc-Sb"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\n",
        "        \"boostSpec\": {\n",
        "            \"conditionBoostSpecs\": {\n",
        "                \"condition\": 'author: ANY(\"Margaret Atwood\")',\n",
        "                \"boost\": 0.9,\n",
        "            }\n",
        "        },\n",
        "        \"query\": QUERY,\n",
        "        \"pageSize\": PAGE_SIZE,\n",
        "    },\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tTlQ0-r7iFvU"
      },
      "source": [
        "## Advanced Boosting\n",
        "Now let's look at a more sophisticated example of a boost rule. As mentioned earlier in this notebook,it is generally advised to try VAIS results out of the box and/or leverage user events to fine tune the rankings based on user behavior. However, in some cases, customers are interested to apply a certain business logic to the results which makes custom rules inevitable.\n",
        "\n",
        "For this particular example we want to primarily leverage user provided ratings to influence search results, specifically rating count and rating average. We also leverage VAIS's ability to apply [piecewise linear boost functions](https://cloud.google.com/generative-ai-app-builder/docs/boost-search-results#custom-numerical-attr-boost) as opposed to fixed boost amounts. We apply different Boost values as the function of the average rating for different buckets of rating counts (see more details in comments of the code block below). We also apply a separate boost rule to boost books with a larger number of ratings irrespective of the average rating (i.e. generally popular books). To make sure that rule does not demote newer content unfairly, we supplement the boost rules by a freshness boost. Lastly to ensure we're not suggesting popular and highly rated, yet irrelevant books to all queries, we're adding a [relevancy threshold filter](https://cloud.google.com/generative-ai-app-builder/docs/filter-by-relevance).\n",
        "\n",
        "Note that the value and logic used here are for demonstration purposes. Please adjust them based on your business logic and metadata schema.\n",
        "\n",
        "You can find more examples of Boosting in [public documentation](https://cloud.google.com/generative-ai-app-builder/docs/boost-search-results)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hdIhVo8XiFvU"
      },
      "outputs": [],
      "source": [
        "QUERY = \"\"  # @param {type: 'string'}\n",
        "PAGE_SIZE = None  # @param {type: 'integer'}\n",
        "\n",
        "search_response = authed_session.post(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}/servingConfigs/default_search:search\",\n",
        "    headers={\"Content-Type\": \"application/json\"},\n",
        "    json={\n",
        "        \"boostSpec\": {\n",
        "            \"conditionBoostSpecs\": [  # The absolute level of boost values can be offset to adjust the balance between recipes and other template types\n",
        "                {\n",
        "                    \"condition\": \"rating_count>=10\",  # only apply to books with more than 10 ratings\n",
        "                    \"boostControlSpec\": {\n",
        "                        \"attributeType\": \"NUMERICAL\",\n",
        "                        \"interpolationType\": \"LINEAR\",\n",
        "                        \"fieldName\": \"aggregate_rating\",\n",
        "                        \"controlPoints\": [\n",
        "                            {\n",
        "                                \"attributeValue\": \"0.0\",\n",
        "                                \"boostAmount\": -0.8,\n",
        "                            },  # kill results with high rating count and low rating. They've had their chance!\n",
        "                            {\n",
        "                                \"attributeValue\": \"3.0\",\n",
        "                                \"boostAmount\": -0.6,\n",
        "                            },  # be aggressive for anything less than 3 stars\n",
        "                            {\n",
        "                                \"attributeValue\": \"4.5\",\n",
        "                                \"boostAmount\": 0.0,\n",
        "                            },  # People are typically generous. Let's assume 4.5 means ok\n",
        "                            {\n",
        "                                \"attributeValue\": \"5.0\",\n",
        "                                \"boostAmount\": 0.3,\n",
        "                            },  # go more aggressivly up as we get closer to 5. more than 35 votes very close to 5 means awesome.\n",
        "                        ],\n",
        "                    },\n",
        "                },\n",
        "                {\n",
        "                    \"condition\": \"rating_count<10\",  # Now let's consider books with fewer ratings\n",
        "                    \"boostControlSpec\": {\n",
        "                        \"attributeType\": \"NUMERICAL\",\n",
        "                        \"interpolationType\": \"LINEAR\",\n",
        "                        \"fieldName\": \"aggregate_rating\",\n",
        "                        \"controlPoints\": [\n",
        "                            {\n",
        "                                \"attributeValue\": \"0.0\",\n",
        "                                \"boostAmount\": -1.0,\n",
        "                            },  # I really don't want to see low rating AND low rating count\n",
        "                            {\n",
        "                                \"attributeValue\": \"4.5\",\n",
        "                                \"boostAmount\": 0,\n",
        "                            },  # with average rating of 4.5, let's give it a chance\n",
        "                            {\n",
        "                                \"attributeValue\": \"5.0\",\n",
        "                                \"boostAmount\": 0.1,\n",
        "                            },  # a small boost, but with fewer reviews, high rating may not mean much.\n",
        "                        ],\n",
        "                    },\n",
        "                },\n",
        "                {\n",
        "                    \"condition\": \"rating_count>=0\",  # no particular meaning, it's just to make the condition True\n",
        "                    \"boostControlSpec\": {\n",
        "                        \"attributeType\": \"NUMERICAL\",\n",
        "                        \"interpolationType\": \"LINEAR\",\n",
        "                        \"fieldName\": \"rating_count\",\n",
        "                        \"controlPoints\": [\n",
        "                            {\n",
        "                                \"attributeValue\": \"0\",\n",
        "                                \"boostAmount\": -0.3,\n",
        "                            },  # burry low rating count\n",
        "                            {\n",
        "                                \"attributeValue\": \"20\",\n",
        "                                \"boostAmount\": 0.05,\n",
        "                            },  # a steep boost curve from 0 to 20\n",
        "                            {\n",
        "                                \"attributeValue\": \"300\",\n",
        "                                \"boostAmount\": 0.2,\n",
        "                            },  # more gentle boost from 20 to 300\n",
        "                            {\n",
        "                                \"attributeValue\": \"1000\",\n",
        "                                \"boostAmount\": 0.35,\n",
        "                            },  # even mor gentle as we get passed 300, and saturate at 1000\n",
        "                        ],\n",
        "                    },\n",
        "                },\n",
        "                {\n",
        "                    \"condition\": \"rating_count>=0\",  # no particular meaning, it's just to make the condition True\n",
        "                    \"boostControlSpec\": {\n",
        "                        \"attributeType\": \"FRESHNESS\",\n",
        "                        \"interpolationType\": \"LINEAR\",\n",
        "                        \"fieldName\": \"date_published\",\n",
        "                        \"controlPoints\": [\n",
        "                            {\"attributeValue\": \"0d\", \"boostAmount\": 0.2},\n",
        "                            {\"attributeValue\": \"30d\", \"boostAmount\": 0.15},\n",
        "                            {\"attributeValue\": \"60d\", \"boostAmount\": 0.1},\n",
        "                            {\"attributeValue\": \"180d\", \"boostAmount\": 0.0},\n",
        "                        ],\n",
        "                    },\n",
        "                },\n",
        "            ]\n",
        "        },\n",
        "        \"query\": QUERY,\n",
        "        \"relevanceThreshold\": \"MEDIUM\",\n",
        "        \"pageSize\": PAGE_SIZE,\n",
        "    },\n",
        ")\n",
        "\n",
        "print(json.dumps(search_response.json(), indent=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e1kgs_XdDlHL"
      },
      "source": [
        "# Clean up"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tGuk4ZnJk0S7"
      },
      "source": [
        "## Delete the Search App\n",
        "\n",
        "Delete the App if you no longer need it\n",
        "\n",
        "Alternatively you can follow [these instructions](https://console.cloud.google.com/gen-app-builder/data-stores) to delete an App from the UI\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QEfxXtzfk0rx"
      },
      "outputs": [],
      "source": [
        "response = authed_session.delete(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/engines/{APP_ID}\",\n",
        "    headers={\"X-Goog-User-Project\": PROJECT_ID},\n",
        ")\n",
        "\n",
        "print(response.text)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tgm5idL4DjjU"
      },
      "source": [
        "##Delete the Datastores\n",
        "Delete the Datastore if you no longer need it\n",
        "\n",
        "Alternatively you can follow [these instructions](https://console.cloud.google.com/gen-app-builder/data-stores) to delete a Datastore from the UI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vj8BpuS62tgt"
      },
      "outputs": [],
      "source": [
        "response = authed_session.delete(\n",
        "    f\"https://discoveryengine.googleapis.com/{VAIS_BRANCH}/projects/{PROJECT_ID}/locations/{LOCATION}/collections/default_collection/dataStores/{DATASTORE_ID}\",\n",
        "    headers={\"X-Goog-User-Project\": PROJECT_ID},\n",
        ")\n",
        "\n",
        "print(response.text)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "query_level_boosting_filtering_and_facets.ipynb",
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
