# robots.txt for www.hextrust.com
# Last updated: 2025-08-11

############################################
# 1. Allow major search engine and LLM bots
############################################
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

User-agent: Applebot
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: GPTBot
Disallow:

User-agent: PerplexityBot
Disallow:

User-agent: AhrefsBot
Disallow:

User-agent: GoogleOther
Disallow:

User-agent: Google-Extended
Disallow:

############################################
# 2. Block suspicious or unwanted bots
############################################
User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

############################################
# 3. General rules for all other bots
############################################
User-agent: *
# Block pagination and duplicate content parameters
Disallow: /*?83d7099e_page=
Disallow: /*?e7e2c200_page=
Disallow: /*?e7e2c214_page=
Disallow: /*?ef8d0b19_page=

# Block search results pages
Disallow: /search
Disallow: /search?*
Disallow: /*?s=

# Block internal Webflow/system pages
Disallow: /_
Disallow: /401
Disallow: /404
Disallow: /cgi-bin/
Disallow: /private/

# Block marketing/tracking parameters
Disallow: /*?utm_campaign=
Disallow: /*?utm_medium=
Disallow: /*?utm_source=
Disallow: /*?utm_term=
Disallow: /*?utm_content=

# Block these params even when after others (multi-param URLs)
Disallow: /*&83d7099e_page=
Disallow: /*&e7e2c200_page=
Disallow: /*&e7e2c214_page=
Disallow: /*&ef8d0b19_page=
Disallow: /*&utm_campaign=
Disallow: /*&utm_medium=
Disallow: /*&utm_source=
Disallow: /*&utm_term=
Disallow: /*&utm_content=

############################################
# 4. Allow essential assets
############################################
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/

# Sitemap
Sitemap: https://www.hextrust.com/sitemap.xml


Sitemap: https://www.hextrust.com/sitemap.xml