# Robots.txt for https://www.coloradorafting.net/
# Last updated: July 20, 2026

# ---------------------------
Sitemap: https://www.coloradorafting.net/sitemap_index.xml
# ---------------------------

# GLOBAL RULES ------------
User-agent: *
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css$
Allow: /wp-content/themes/*.js$
# Rafting-specific gallery blocks (kept from original)
Disallow: /galleries/raft-guide-training-2018/
Disallow: /galleries/ava-campground/
Disallow: /galleries/ava-cabins/
# Block system files & old theme probes
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-*.php
Disallow: /wp-content/themes/ava-rafting/
# Allow AJAX entrypoint while protecting admin area
Allow: /wp-admin/admin-ajax.php
# Block HubSpot & AMP crawl noise
Disallow: */amp
Disallow: */amp/
Disallow: /*?__hstc=
Disallow: /*?__hssc=
Disallow: /*?__hsfp=
Disallow: /mirror/
Disallow: /tag/
# Block search, sorting, filtering and other query-based pages
Disallow: /*?s=
Disallow: /search/
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*orderby=
Disallow: /*?*attachment_id=
# Prevent indexing of URL queries and duplicate feeds/pages
Disallow: /*?p=
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /*/page/
Disallow: /*/embed/
Disallow: /*?sf_paged=
# Block WordPress internals, JSON API & common duplicate paths
Disallow: /wp-json/
Disallow: /trackback/
Disallow: /feed/
Disallow: /page/*?*
# Keep the AI descriptor file private by default (opened up per-bot below)
Disallow: /ai-about-us.txt

# GOOGLEBOT
User-agent: Googlebot
# (Google ignores crawl-delay)
Allow: /*.js$
Allow: /*.css$
Allow: /wp-content/uploads/
Allow: /ai-about-us.txt

# BINGBOT
User-agent: Bingbot
Crawl-delay: 2
Allow: /ai-about-us.txt

# IMAGE / VIDEO BOTS: let them fetch media but not admin internals
User-agent: Googlebot-Video
User-agent: Googlebot-Image
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Allow: /wp-content/uploads/

# OPENAI
User-agent: GPTBot
Allow: /ai-about-us.txt
User-agent: ChatGPT-User
Allow: /ai-about-us.txt
User-agent: OAI-SearchBot
Allow: /ai-about-us.txt
# ANTHROPIC
User-agent: ClaudeBot
Allow: /ai-about-us.txt
User-agent: Claude-User
Allow: /ai-about-us.txt
User-agent: Claude-SearchBot
Allow: /ai-about-us.txt
# GOOGLE
User-agent: Google-Extended
Allow: /ai-about-us.txt
User-agent: Gemini-Deep-Research
Allow: /ai-about-us.txt
# MICROSOFT
# (Bingbot handled above; its Allow: /ai-about-us.txt lives in that group)
# COHERE
User-agent: cohere-ai
Allow: /ai-about-us.txt
# APPLE
User-agent: Applebot-Extended
Allow: /ai-about-us.txt
# PERPLEXITY
User-agent: PerplexityBot
Allow: /ai-about-us.txt
User-agent: Perplexity-User
Allow: /ai-about-us.txt
# DUCKDUCKGO
User-agent: DuckAssistBot
Allow: /ai-about-us.txt
# META
User-agent: Meta-ExternalAgent
Allow: /ai-about-us.txt
User-agent: Meta-WebIndexer
Allow: /ai-about-us.txt
# BYTEDANCE / TIKTOK
User-agent: Bytespider
Allow: /ai-about-us.txt
# AMAZON
User-agent: Amazonbot
Allow: /ai-about-us.txt
# MISTRAL
User-agent: MistralAI-User
Allow: /ai-about-us.txt
# DIFFBOT
User-agent: Diffbot
Allow: /ai-about-us.txt
# COMMON CRAWL
User-agent: CCBot
Allow: /ai-about-us.txt
# INFOQ / ICC
User-agent: ICC-Crawler
Allow: /ai-about-us.txt
# WEBZ.IO / OMGILI
User-agent: webzio-extended
Allow: /ai-about-us.txt
# YOU.COM
User-agent: YouBot
Allow: /ai-about-us.txt