# https://www.robotstxt.org/robotstxt.html
# Sitemap declared up top so crawlers (including GSC) see it on first parse.
Sitemap: https://forkoff.xyz/sitemap.xml

# Default group — applies to all crawlers not named below.
# Allow: / before Disallow: rules so explicit allows aren't accidentally dropped.
User-agent: *
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Disallow: /md/
Disallow: /_next/
Disallow: /api/
# Video assets — block GSC video indexing of decorative background loops
# (per GSC "Video isn't on a watch page" report 2026-05-08, n=88 errors)
Disallow: /assets/v0-imagery/footer-banner.mp4
Disallow: /assets/v0-imagery/bg-hero.mp4
Disallow: /assets/services/podcast/bg.mp4
Disallow: /assets/v0-imagery/*.mp4

# AI Crawlers — explicit allow for the major LLM / AI-search user-agents.
# Listed individually because a named user-agent group overrides the wildcard above;
# each entry must re-declare allow/disallow rules. Order is grouped by vendor.

# OpenAI
User-agent: GPTBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

User-agent: ChatGPT-User
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

User-agent: OAI-SearchBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Anthropic
User-agent: ClaudeBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

User-agent: anthropic-ai
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

User-agent: Claude-Web
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Google AI training / Gemini
User-agent: Google-Extended
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Apple (Siri / Apple Intelligence)
User-agent: Applebot-Extended
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Meta (Llama / Meta AI)
User-agent: FacebookBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

User-agent: Meta-ExternalAgent
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# ByteDance (Doubao / Volcano / TikTok)
User-agent: Bytespider
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Amazon (Alexa / Titan)
User-agent: Amazonbot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Cohere
User-agent: cohere-ai
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# Common Crawl — used by most open-weight models
User-agent: CCBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

# DuckDuckGo AI-assist
User-agent: DuckAssistBot
# Content-Signal: ai-train=no, search=yes, ai-input=yes
Disallow: /md/
Allow: /
Disallow: /_next/
Disallow: /api/

Sitemap: https://forkoff.xyz/sitemap.xml

# AI Context
# See https://forkoff.xyz/llms.txt for AI-readable site context
# See https://forkoff.xyz/llmfull.txt for full detailed context