# === Google ===
# Googlebot obeys ONLY its most-specific user-agent group and ignores the
# `User-agent: *` group entirely, so any path we want blocked for Google must be
# repeated here. We keep the authenticated app shell (/app) out of Google's index
# but intentionally let Googlebot crawl /auth — those pages are fine to surface.
User-agent: Googlebot
Allow: /
Disallow: /app

# Block Google AI training (no attribution), keep standard search
User-agent: Google-Extended
Disallow: /

# === OpenAI ===
# Block training crawler
User-agent: GPTBot
Disallow: /

# Allow ChatGPT search (citations + backlinks)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# === Anthropic ===
# Block training crawler
User-agent: ClaudeBot
Disallow: /

# Allow Claude search (citations + backlinks)
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# === Perplexity ===
User-agent: PerplexityBot
Allow: /

# === Block other training crawlers ===
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

# === Default ===
User-agent: *
Allow: /
Allow: /courses
Allow: /hackathons
Allow: /marketplace
Allow: /launchpad
Allow: /bounties
Allow: /leaderboard
Allow: /community
Disallow: /app
Disallow: /auth

Sitemap: https://www.trydojo.io/sitemap.xml