# ============================================================
# ROBOTS.TXT - Fast Aquatics Marketplace
# https://fastaquatics.com
# Per LLM_DOMINATION_MASTER_INSTRUCTIONS Part 2
# Updated: 2026-04-29
# ============================================================

# --- ALLOW ALL MAJOR SEARCH ENGINES ---
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# --- ALLOW ALL AI / LLM CRAWLERS (CRITICAL FOR AI CITATIONS) ---
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Gemini-Bot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DiffBot
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Timpibot
Allow: /

# --- BLOCK SCRAPER / COMPETITOR SEO TOOLS + AI BOTS THAT DON'T CITE ---
# Bytespider/TikTok scrapes for ByteDance LLM training without citation traffic
User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# --- PROTECT TRANSACTIONAL + ADMIN PATHS ---
User-agent: *
Disallow: /admin/
Disallow: /cart/checkout/
Disallow: /vendor/dashboard/
Disallow: /vendor/orders/
Disallow: /vendor/listings/new/
Disallow: /account/order/
Disallow: /.netlify/
# Public read-only JSON API (CC-BY 4.0) - allow crawl
Allow: /api/v1/

# --- SITEMAP + FEED DECLARATIONS ---
# Crawlers should only need the index; per-section files are auto-discovered.
Sitemap: https://fastaquatics.com/sitemap.xml
# RSS + Atom feeds (200 most-recent pages each)
# https://fastaquatics.com/feed.xml
# https://fastaquatics.com/atom.xml
# Static JSON API index: https://fastaquatics.com/api/v1/index.json

# --- AI CONTENT DECLARATIONS ---
# Plain-language brief for LLM systems available at:
# https://fastaquatics.com/llm.txt
# https://fastaquatics.com/ai.txt