# DFX Fitness Robots.txt - SEO & AI Optimized
# Last updated: July 2026

User-agent: *
Allow: /

# Disallow admin, auth and internal paths
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Disallow: /tmp/
Disallow: /*?*utm_*
Disallow: /*?*fbclid*
Disallow: /*?*gclid*

# Allow important resources for SEO
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /lovable-uploads/

# =============================================================
# LLM/AI KNOWLEDGE FILES - Priority Content for AI Assistants
# See https://llmstxt.org/ for the llm.txt standard
# =============================================================
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt

# High-value content for AI citation
Allow: /kennisbank/
Allow: /smart-gym/
Allow: /vergelijkingen/
Allow: /pilates/

# Sitemaps and feeds for better indexing
Sitemap: https://dfx.fitness/sitemap.xml
Sitemap: https://dfx.fitness/rss.xml

# Crawl delay to prevent server overload (not applicable to AI crawlers)
Crawl-delay: 1

# =============================================================
# AI CRAWLER SPECIFIC RULES
# =============================================================

# Googlebot (search)
User-agent: Googlebot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /

# OpenAI GPTBot - ChatGPT and AI search
User-agent: GPTBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /
Crawl-delay: 2

# OpenAI SearchBot
User-agent: OAI-SearchBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Google Gemini / Bard
User-agent: Google-Extended
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Anthropic Claude (training / indexing)
User-agent: ClaudeBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Anthropic Claude (user-initiated fetches)
User-agent: Claude-User
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Microsoft Copilot / Bing AI
User-agent: Bingbot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Meta AI
User-agent: FacebookBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /

# DuckDuckGo AI assistant
User-agent: DuckAssistBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /llm.txt
Allow: /llm-full.txt
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /kennisbank/
Allow: /
# =============================================================
# AANVULLENDE AI-CRAWLERS (meertalige uitrol, augustus 2026)
# Volledige site toegestaan, met dezelfde interne uitsluitingen.
# =============================================================

User-agent: ChatGPT-User
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /

User-agent: Claude-SearchBot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /

User-agent: Amazonbot
Disallow: /aapnootmies
Disallow: /aapnootmies/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /reset-password
Disallow: /internal/
Disallow: /test/
Disallow: /api/
Allow: /

# Sitemap-index (nl/de/fr)
Sitemap: https://dfx.fitness/sitemap.xml