/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 */

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

package org.opensearch.common.xcontent.support;

import org.opensearch.common.CheckedFunction;
import org.opensearch.common.util.set.Sets;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.xcontent.XContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.test.OpenSearchTestCase;

import java.io.IOException;
import java.util.Set;

import static java.util.Collections.emptySet;
import static java.util.Collections.singleton;

/**
 * Tests for {@link XContent} filtering.
 */
public abstract class AbstractFilteringTestCase extends OpenSearchTestCase {

    @FunctionalInterface
    protected interface Builder extends CheckedFunction<XContentBuilder, XContentBuilder, IOException> {}

    protected abstract void testFilter(Builder expected, Builder actual, Set<String> includes, Set<String> excludes) throws IOException;

    /** Sample test case **/
    protected static final Builder SAMPLE = builder -> builder.startObject()
        .field("title", "My awesome book")
        .field("pages", 456)
        .field("price", 27.99)
        .field("timestamp", 1428582942867L)
        .nullField("default")
        .startArray("tags")
        .value("opensearch")
        .value("java")
        .endArray()
        .startArray("authors")
        .startObject()
        .field("name", "John Doe")
        .field("lastname", "John")
        .field("firstname", "Doe")
        .endObject()
        .startObject()
        .field("name", "William Smith")
        .field("lastname", "William")
        .field("firstname", "Smith")
        .endObject()
        .endArray()
        .startObject("properties")
        .field("weight", 0.8d)
        .startObject("language")
        .startObject("en")
        .field("lang", "English")
        .field("available", true)
        .startArray("distributors")
        .startObject()
        .field("name", "The Book Shop")
        .startArray("addresses")
        .startObject()
        .field("name", "address #1")
        .field("street", "Hampton St")
        .field("city", "London")
        .endObject()
        .startObject()
        .field("name", "address #2")
        .field("street", "Queen St")
        .field("city", "Stornoway")
        .endObject()
        .endArray()
        .endObject()
        .startObject()
        .field("name", "Sussex Books House")
        .endObject()
        .endArray()
        .endObject()
        .startObject("fr")
        .field("lang", "French")
        .field("available", false)
        .startArray("distributors")
        .startObject()
        .field("name", "La Maison du Livre")
        .startArray("addresses")
        .startObject()
        .field("name", "address #1")
        .field("street", "Rue Mouffetard")
        .field("city", "Paris")
        .endObject()
        .endArray()
        .endObject()
        .startObject()
        .field("name", "Thetra")
        .endObject()
        .endArray()
        .endObject()
        .endObject()
        .endObject()
        .endObject();

    public void testNoFiltering() throws Exception {
        final Builder expected = SAMPLE;

        testFilter(expected, SAMPLE, emptySet(), emptySet());
        testFilter(expected, SAMPLE, singleton("*"), emptySet());
        testFilter(expected, SAMPLE, singleton("**"), emptySet());
        testFilter(expected, SAMPLE, emptySet(), singleton("xyz"));
    }

    public void testNoMatch() throws Exception {
        final Builder expected = builder -> builder.startObject().endObject();

        testFilter(expected, SAMPLE, singleton("xyz"), emptySet());
        testFilter(expected, SAMPLE, emptySet(), singleton("*"));
        testFilter(expected, SAMPLE, emptySet(), singleton("**"));
    }

    public void testSimpleFieldInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject().field("title", "My awesome book").endObject();

        testFilter(expected, SAMPLE, singleton("title"), emptySet());
    }

    public void testSimpleFieldExclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("title"));
    }

    public void testSimpleFieldWithWildcardInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("price", 27.99)
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, singleton("pr*"), emptySet());
    }

    public void testSimpleFieldWithWildcardExclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("pr*"));
    }

    public void testMultipleFieldsInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject().field("title", "My awesome book").field("pages", 456).endObject();

        testFilter(expected, SAMPLE, Sets.newHashSet("title", "pages"), emptySet());
    }

    public void testMultipleFieldsExclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), Sets.newHashSet("title", "pages"));
    }

    public void testSimpleArrayInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .endObject();

        testFilter(expected, SAMPLE, singleton("tags"), emptySet());
    }

    public void testSimpleArrayExclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("tags"));
    }

    public void testSimpleArrayOfObjectsInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .endObject();

        testFilter(expected, SAMPLE, singleton("authors"), emptySet());
        testFilter(expected, SAMPLE, singleton("authors.*"), emptySet());
        testFilter(expected, SAMPLE, singleton("authors.*name"), emptySet());
    }

    protected static final Builder SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE = builder -> builder.startObject()
        .field("title", "My awesome book")
        .field("pages", 456)
        .field("price", 27.99)
        .field("timestamp", 1428582942867L)
        .nullField("default")
        .startArray("tags")
        .value("opensearch")
        .value("java")
        .endArray()
        .startObject("properties")
        .field("weight", 0.8d)
        .startObject("language")
        .startObject("en")
        .field("lang", "English")
        .field("available", true)
        .startArray("distributors")
        .startObject()
        .field("name", "The Book Shop")
        .startArray("addresses")
        .startObject()
        .field("name", "address #1")
        .field("street", "Hampton St")
        .field("city", "London")
        .endObject()
        .startObject()
        .field("name", "address #2")
        .field("street", "Queen St")
        .field("city", "Stornoway")
        .endObject()
        .endArray()
        .endObject()
        .startObject()
        .field("name", "Sussex Books House")
        .endObject()
        .endArray()
        .endObject()
        .startObject("fr")
        .field("lang", "French")
        .field("available", false)
        .startArray("distributors")
        .startObject()
        .field("name", "La Maison du Livre")
        .startArray("addresses")
        .startObject()
        .field("name", "address #1")
        .field("street", "Rue Mouffetard")
        .field("city", "Paris")
        .endObject()
        .endArray()
        .endObject()
        .startObject()
        .field("name", "Thetra")
        .endObject()
        .endArray()
        .endObject()
        .endObject()
        .endObject()
        .endObject();

    public void testSimpleArrayOfObjectsExclusive() throws Exception {
        testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors"));
        testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors.*"));
        testFilter(SIMPLE_ARRAY_OF_OBJECTS_EXCLUSIVE, SAMPLE, emptySet(), singleton("authors.*name"));
    }

    public void testSimpleArrayOfObjectsPropertyInclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startArray("authors")
            .startObject()
            .field("lastname", "John")
            .endObject()
            .startObject()
            .field("lastname", "William")
            .endObject()
            .endArray()
            .endObject();

        testFilter(expected, SAMPLE, singleton("authors.lastname"), emptySet());
        testFilter(expected, SAMPLE, singleton("authors.l*"), emptySet());
    }

    public void testSimpleArrayOfObjectsPropertyExclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("authors.lastname"));
        testFilter(expected, SAMPLE, emptySet(), singleton("authors.l*"));
    }

    public void testRecurseField1Inclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .startObject("language")
            .startObject("en")
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, singleton("**.name"), emptySet());
    }

    public void testRecurseField1Exclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("**.name"));
    }

    public void testRecurseField2Inclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startObject("properties")
            .startObject("language")
            .startObject("en")
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, singleton("properties.**.name"), emptySet());
    }

    public void testRecurseField2Exclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.name"));
    }

    public void testRecurseField3Inclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startObject("properties")
            .startObject("language")
            .startObject("en")
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, singleton("properties.*.en.**.name"), emptySet());
    }

    public void testRecurseField3Exclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("properties.*.en.**.name"));
    }

    public void testRecurseField4Inclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .startObject("properties")
            .startObject("language")
            .startObject("en")
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .startArray("distributors")
            .startObject()
            .field("name", "La Maison du Livre")
            .endObject()
            .startObject()
            .field("name", "Thetra")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, singleton("properties.**.distributors.name"), emptySet());
    }

    public void testRecurseField4Exclusive() throws Exception {
        final Builder expected = builder -> builder.startObject()
            .field("title", "My awesome book")
            .field("pages", 456)
            .field("price", 27.99)
            .field("timestamp", 1428582942867L)
            .nullField("default")
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("name", "John Doe")
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("name", "William Smith")
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Hampton St")
            .field("city", "London")
            .endObject()
            .startObject()
            .field("name", "address #2")
            .field("street", "Queen St")
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .startArray("distributors")
            .startObject()
            .startArray("addresses")
            .startObject()
            .field("name", "address #1")
            .field("street", "Rue Mouffetard")
            .field("city", "Paris")
            .endObject()
            .endArray()
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, emptySet(), singleton("properties.**.distributors.name"));
    }

    public void testRawField() throws Exception {
        final Builder expectedRawField = builder -> builder.startObject()
            .field("foo", 0)
            .startObject("raw")
            .field("content", "hello world!")
            .endObject()
            .endObject();

        final Builder expectedRawFieldFiltered = builder -> builder.startObject().field("foo", 0).endObject();

        final Builder expectedRawFieldNotFiltered = builder -> builder.startObject()
            .startObject("raw")
            .field("content", "hello world!")
            .endObject()
            .endObject();

        Builder sampleWithRaw = builder -> {
            BytesReference raw = BytesReference.bytes(
                XContentBuilder.builder(builder.contentType().xContent()).startObject().field("content", "hello world!").endObject()
            );
            return builder.startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject();
        };

        // Test method: rawField(String fieldName, BytesReference content)
        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));

        sampleWithRaw = builder -> {
            BytesReference raw = BytesReference.bytes(
                XContentBuilder.builder(builder.contentType().xContent()).startObject().field("content", "hello world!").endObject()
            );
            return builder.startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject();
        };

        // Test method: rawField(String fieldName, InputStream content)
        testFilter(expectedRawField, sampleWithRaw, emptySet(), emptySet());
        testFilter(expectedRawFieldFiltered, sampleWithRaw, singleton("f*"), emptySet());
        testFilter(expectedRawFieldFiltered, sampleWithRaw, emptySet(), singleton("r*"));
        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, singleton("r*"), emptySet());
        testFilter(expectedRawFieldNotFiltered, sampleWithRaw, emptySet(), singleton("f*"));
    }

    public void testArrays() throws Exception {
        // Test: Array of values (no filtering)
        final Builder sampleArrayOfValues = builder -> builder.startObject()
            .startArray("tags")
            .value("lorem")
            .value("ipsum")
            .value("dolor")
            .endArray()
            .endObject();
        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("t*"), emptySet());
        testFilter(sampleArrayOfValues, sampleArrayOfValues, singleton("tags"), emptySet());
        testFilter(sampleArrayOfValues, sampleArrayOfValues, emptySet(), singleton("a"));

        // Test: Array of values (with filtering)
        Builder expected = builder -> builder.startObject().endObject();
        testFilter(expected, sampleArrayOfValues, singleton("foo"), emptySet());
        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("t*"));
        testFilter(expected, sampleArrayOfValues, emptySet(), singleton("tags"));

        // Test: Array of objects (no filtering)
        final Builder sampleArrayOfObjects = builder -> builder.startObject()
            .startArray("tags")
            .startObject()
            .field("lastname", "lorem")
            .endObject()
            .startObject()
            .field("firstname", "ipsum")
            .endObject()
            .endArray()
            .endObject();
        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("t*"), emptySet());
        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, singleton("tags"), emptySet());
        testFilter(sampleArrayOfObjects, sampleArrayOfObjects, emptySet(), singleton("a"));

        // Test: Array of objects (with filtering)
        testFilter(expected, sampleArrayOfObjects, singleton("foo"), emptySet());
        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*"));
        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("tags"));

        // Test: Array of objects (with partial filtering)
        expected = builder -> builder.startObject()
            .startArray("tags")
            .startObject()
            .field("firstname", "ipsum")
            .endObject()
            .endArray()
            .endObject();
        testFilter(expected, sampleArrayOfObjects, singleton("t*.firstname"), emptySet());
        testFilter(expected, sampleArrayOfObjects, emptySet(), singleton("t*.lastname"));
    }

    public void testEmptyObject() throws IOException {
        final Builder sample = builder -> builder.startObject().startObject("foo").endObject().endObject();

        Builder expected = builder -> builder.startObject().startObject("foo").endObject().endObject();
        testFilter(expected, sample, singleton("foo"), emptySet());
        testFilter(expected, sample, emptySet(), singleton("bar"));
        testFilter(expected, sample, singleton("f*"), singleton("baz"));

        expected = builder -> builder.startObject().endObject();
        testFilter(expected, sample, emptySet(), singleton("foo"));
        testFilter(expected, sample, singleton("bar"), emptySet());
        testFilter(expected, sample, singleton("f*"), singleton("foo"));
    }

    public void testSingleFieldWithBothExcludesIncludes() throws IOException {
        final Builder expected = builder -> builder.startObject().field("pages", 456).field("price", 27.99).endObject();

        testFilter(expected, SAMPLE, singleton("p*"), singleton("properties"));
    }

    public void testObjectsInArrayWithBothExcludesIncludes() throws IOException {
        Set<String> includes = Sets.newHashSet("tags", "authors");
        Set<String> excludes = singleton("authors.name");

        final Builder expected = builder -> builder.startObject()
            .startArray("tags")
            .value("opensearch")
            .value("java")
            .endArray()
            .startArray("authors")
            .startObject()
            .field("lastname", "John")
            .field("firstname", "Doe")
            .endObject()
            .startObject()
            .field("lastname", "William")
            .field("firstname", "Smith")
            .endObject()
            .endArray()
            .endObject();

        testFilter(expected, SAMPLE, includes, excludes);
    }

    public void testRecursiveObjectsInArrayWithBothExcludesIncludes() throws IOException {
        Set<String> includes = Sets.newHashSet("**.language", "properties.weight");
        Set<String> excludes = singleton("**.distributors");

        final Builder expected = builder -> builder.startObject()
            .startObject("properties")
            .field("weight", 0.8d)
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .endObject()
            .startObject("fr")
            .field("lang", "French")
            .field("available", false)
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, includes, excludes);
    }

    public void testRecursiveSameObjectWithBothExcludesIncludes() throws IOException {
        Set<String> includes = singleton("**.distributors");
        Set<String> excludes = singleton("**.distributors");

        final Builder expected = builder -> builder.startObject().endObject();
        testFilter(expected, SAMPLE, includes, excludes);
    }

    public void testRecursiveObjectsPropertiesWithBothExcludesIncludes() throws IOException {
        Set<String> includes = singleton("**.en.*");
        Set<String> excludes = Sets.newHashSet("**.distributors.*.name", "**.street");

        final Builder expected = builder -> builder.startObject()
            .startObject("properties")
            .startObject("language")
            .startObject("en")
            .field("lang", "English")
            .field("available", true)
            .startArray("distributors")
            .startObject()
            .field("name", "The Book Shop")
            .startArray("addresses")
            .startObject()
            .field("city", "London")
            .endObject()
            .startObject()
            .field("city", "Stornoway")
            .endObject()
            .endArray()
            .endObject()
            .startObject()
            .field("name", "Sussex Books House")
            .endObject()
            .endArray()
            .endObject()
            .endObject()
            .endObject()
            .endObject();

        testFilter(expected, SAMPLE, includes, excludes);
    }

    public void testWithLfAtEnd() throws IOException {
        final Builder sample = builder -> builder.startObject()
            .startObject("foo")
            .field("bar", "baz")
            .endObject()
            .endObject()
            .prettyPrint()
            .lfAtEnd();

        testFilter(sample, sample, singleton("foo"), emptySet());
        testFilter(sample, sample, emptySet(), singleton("bar"));
        testFilter(sample, sample, singleton("f*"), singleton("baz"));

        final Builder expected = builder -> builder.startObject().endObject().prettyPrint().lfAtEnd();
        testFilter(expected, sample, emptySet(), singleton("foo"));
        testFilter(expected, sample, singleton("bar"), emptySet());
        testFilter(expected, sample, singleton("f*"), singleton("foo"));
    }

    public void testBasics() throws Exception {
        final Builder sample = builder -> builder.startObject()
            .field("test1", "value1")
            .field("test2", "value2")
            .field("something_else", "value3")
            .endObject();

        Builder expected = builder -> builder.startObject().field("test1", "value1").endObject();
        testFilter(expected, sample, singleton("test1"), emptySet());

        expected = builder -> builder.startObject().field("test1", "value1").field("test2", "value2").endObject();
        testFilter(expected, sample, singleton("test*"), emptySet());

        expected = builder -> builder.startObject().field("test2", "value2").field("something_else", "value3").endObject();
        testFilter(expected, sample, emptySet(), singleton("test1"));

        // more complex object...
        final Builder complex = builder -> builder.startObject()
            .startObject("path1")
            .startArray("path2")
            .startObject()
            .field("test", "value1")
            .endObject()
            .startObject()
            .field("test", "value2")
            .endObject()
            .endArray()
            .endObject()
            .field("test1", "value1")
            .endObject();

        expected = builder -> builder.startObject()
            .startObject("path1")
            .startArray("path2")
            .startObject()
            .field("test", "value1")
            .endObject()
            .startObject()
            .field("test", "value2")
            .endObject()
            .endArray()
            .endObject()
            .endObject();
        testFilter(expected, complex, singleton("path1"), emptySet());
        testFilter(expected, complex, singleton("path1*"), emptySet());
        testFilter(expected, complex, singleton("path1.path2.*"), emptySet());

        expected = builder -> builder.startObject().field("test1", "value1").endObject();
        testFilter(expected, complex, singleton("test1*"), emptySet());
    }

    /**
     * Generalization of {@link XContentMapValuesTests#testSupplementaryCharactersInPaths()}
     */
    public void testFilterSupplementaryCharactersInPaths() throws IOException {
        final Builder sample = builder -> builder.startObject().field("搜索", 2).field("指数", 3).endObject();

        Builder expected = builder -> builder.startObject().field("搜索", 2).endObject();
        testFilter(expected, sample, singleton("搜索"), emptySet());

        expected = builder -> builder.startObject().field("指数", 3).endObject();
        testFilter(expected, sample, emptySet(), singleton("搜索"));
    }

    /**
     * Generalization of {@link XContentMapValuesTests#testSharedPrefixes()}
     */
    public void testFilterSharedPrefixes() throws IOException {
        final Builder sample = builder -> builder.startObject().field("foobar", 2).field("foobaz", 3).endObject();

        Builder expected = builder -> builder.startObject().field("foobar", 2).endObject();
        testFilter(expected, sample, singleton("foobar"), emptySet());

        expected = builder -> builder.startObject().field("foobaz", 3).endObject();
        testFilter(expected, sample, emptySet(), singleton("foobar"));
    }

    /**
     * Generalization of {@link XContentMapValuesTests#testPrefix()}
     */
    public void testFilterPrefix() throws IOException {
        final Builder sample = builder -> builder.startObject().array("photos", "foo", "bar").field("photosCount", 2).endObject();

        Builder expected = builder -> builder.startObject().field("photosCount", 2).endObject();
        testFilter(expected, sample, singleton("photosCount"), emptySet());
    }
}
