Advanced Search Queries and Filters

Users need to perform complex searches with multiple filters, boolean logic, and advanced query syntax. Understanding DataHub's search capabilities and query language helps users find data more effectively.

You Might Be Asking:

• How do I use advanced search operators?
• Can I combine multiple filters?
• What search syntax is supported?

1. Use field-specific search:

# Search in specific fields
name:customer_data
description:sales
tags:PII
platform:snowflake
domains:Finance

# Combine field searches
name:customer AND platform:snowflake
description:(revenue OR sales) AND tags:PII

2. Boolean operators and wildcards:

# AND operator (implicit or explicit)
customer data
customer AND data

# OR operator
customer OR user

# NOT operator
sales NOT test

# Wildcards
customer*  # Matches customer, customers, customer_data
*_temp     # Matches anything ending in _temp
cust?mer   # ? matches single character

# Phrase search
"customer data warehouse"

# Grouping with parentheses
(customer OR user) AND (sales OR revenue)

3. Advanced GraphQL search with filters:

query advancedSearch {
  search(
    input: {
      type: DATASET
      query: "customer"
      start: 0
      count: 50
      filters: [
        # Multiple filter conditions
        {
          field: "platform"
          values: ["snowflake", "bigquery"]
        },
        {
          field: "tags"
          values: ["urn:li:tag:PII"]
        },
        {
          field: "domains"
          values: ["urn:li:domain:Finance"]
        },
        {
          field: "origin"
          values: ["PROD"]
        }
      ]
      # Sort results
      searchFlags: {
        fulltext: true
        skipCache: false
      }
    }
  ) {
    total
    searchResults {
      entity {
        urn
        ... on Dataset {
          name
          description
          platform { name }
        }
      }
      matchedFields {
        name
        value
      }
    }
    facets {
      field
      aggregations {
        value
        count
      }
    }
  }
}

4. Programmatic complex search:

def advanced_search(
    query,
    platforms=None,
    tags=None,
    domains=None,
    has_owners=None,
    modified_after=None
):
    """
    Perform advanced search with multiple criteria
    """
    filters = []

    if platforms:
        filters.append({
            "field": "platform",
            "values": platforms
        })

    if tags:
        filters.append({
            "field": "tags",
            "values": [f"urn:li:tag:{tag}" for tag in tags]
        })

    if domains:
        filters.append({
            "field": "domains",
            "values": [f"urn:li:domain:{domain}" for domain in domains]
        })

    if has_owners is not None:
        filters.append({
            "field": "hasOwners",
            "values": [str(has_owners).lower()]
        })

    graphql_query = """
    query search($query: String!, $filters: [FacetFilterInput!]) {
      search(input: {
        type: DATASET
        query: $query
        filters: $filters
        start: 0
        count: 100
      }) {
        searchResults {
          entity {
            urn
            ... on Dataset { name platform { name } }
          }
        }
      }
    }
    """

    result = graphql_query(graphql_query, {
        "query": query,
        "filters": filters
    })

    return [r['entity'] for r in result['data']['search']['searchResults']]

# Usage
results = advanced_search(
    query="customer revenue",
    platforms=["snowflake", "bigquery"],
    tags=["PII", "Financial"],
    domains=["Finance"],
    has_owners=True
)

5. Search with aggregations and facets:

def search_with_facets(query):
    """
    Search and get facet aggregations
    """
    graphql_query = """
    query searchWithFacets($query: String!) {
      search(input: {
        type: DATASET
        query: $query
      }) {
        facets {
          field
          displayName
          aggregations {
            value
            count
            entity {
              urn
              ... on Tag { name }
              ... on Domain { name }
            }
          }
        }
      }
    }
    """

    result = graphql_query(graphql_query, {"query": query})
    facets = result['data']['search']['facets']

    # Process facets
    facet_summary = {}
    for facet in facets:
        facet_summary[facet['field']] = [
            {
                'value': agg['value'],
                'count': agg['count']
            }
            for agg in facet['aggregations']
        ]

    return facet_summary

# Usage
facets = search_with_facets("customer")
print("Platforms:", facets.get('platform'))
print("Tags:", facets.get('tags'))
print("Domains:", facets.get('domains'))

6. Column-level search:

query searchColumns {
  search(
    input: {
      type: DATASET
      query: "email"  # Search column names and descriptions
      filters: [
        {
          field: "fieldPaths"
          values: ["email", "email_address"]
        }
      ]
    }
  ) {
    searchResults {
      entity {
        ... on Dataset {
          name
          schemaMetadata {
            fields {
              fieldPath
              description
              nativeDataType
            }
          }
        }
      }
    }
  }
}

7. Save and reuse search queries:

# Save common searches
SAVED_SEARCHES = {
    "pii_datasets": {
        "query": "*",
        "filters": [
            {"field": "tags", "values": ["urn:li:tag:PII"]}
        ]
    },
    "finance_prod": {
        "query": "*",
        "filters": [
            {"field": "domains", "values": ["urn:li:domain:Finance"]},
            {"field": "origin", "values": ["PROD"]}
        ]
    },
    "unowned_critical": {
        "query": "*",
        "filters": [
            {"field": "hasOwners", "values": ["false"]},
            {"field": "tags", "values": ["urn:li:tag:Critical"]}
        ]
    }
}

def execute_saved_search(search_name):
    """Execute a saved search by name"""
    if search_name not in SAVED_SEARCHES:
        raise ValueError(f"Unknown search: {search_name}")

    search_params = SAVED_SEARCHES[search_name]
    return advanced_search(**search_params)

# Usage
pii_datasets = execute_saved_search("pii_datasets")

Search performance depends on Elasticsearch configuration and index size. Complex queries with many filters may be slower. Use specific field searches for better performance. Wildcard searches at the beginning of terms (*abc) are expensive. Consider using saved searches for frequently used complex queries.

• Search Guide
• Search API