# === Standard Crawlers & Global Rules ===
User-agent: *
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# === LLM & AI Training Bots (Explicitly Allowed) ===

# OpenAI (ChatGPT, GPT-4 Training & Search)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: anthropic-ai
User-agent: ClaudeBot
Allow: /

# Google (Gemini & Experimental)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /

# Perplexity (AI Search)
User-agent: PerplexityBot
Allow: /

# Common Crawl (Basis for mange open-source modeller)
User-agent: CCBot
Allow: /

# Meta (Llama / Facebook AI)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Allow: /

# Apple (Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# === Sitemap ===
Sitemap: https://minearvinger.dk/sitemap_index.xml