User-agent: *
Disallow: /api/
Disallow: /functions/

# AI crawlers — discovery files and metered content API
# Free discovery: llms.txt, llms-full.txt, crawlers.json, security.txt
# Metered API: /api/crawlers/* (requires bearer key)
# Broad Disallow: /api/ blocks all future API routes by default;
# Allow: /api/crawlers/ is more specific and wins per RFC 9309 §2.2.2.

User-agent: GPTBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: Claude-Web
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: Anthropic-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: Google-Extended
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: CCBot
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: Bytespider
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

User-agent: cohere-ai
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /crawlers.json
Allow: /security.txt
Allow: /api/crawlers/
Allow: /
Disallow: /api/
Disallow: /functions/

# General crawlers
Sitemap: https://wallstreet.wiki/sitemap.xml