# Kismet AI Conversational Travel Platform
# https://proxy.kismet.travel

# ------------------------------------------------------------------
# LLM + OpenAI crawlers – explicit machine endpoint access
# ------------------------------------------------------------------
User-agent: ChatGPT-User
Allow: /mcp/
Allow: /

User-agent: OAI-SearchBot
Allow: /mcp/
Allow: /

User-agent: GPTBot
Allow: /mcp/
Allow: /

# ------------------------------------------------------------------
# Global directives
# ------------------------------------------------------------------
User-agent: *
Allow: /
Allow: /mcp/

# Disallow private application paths
Disallow: /api/
Disallow: /auth/
Disallow: /profile/
Disallow: /account/

# ------------------------------------------------------------------
# SEO crawler restrictions
# ------------------------------------------------------------------
User-agent: SemrushBot
Disallow: /rates
Disallow: /*/rates

User-agent: SemrushBot-SA
Disallow: /rates
Disallow: /*/rates

User-agent: SemrushBot-BA
Disallow: /rates
Disallow: /*/rates

User-agent: SemrushBot-SI
Disallow: /rates
Disallow: /*/rates

User-agent: SemrushBot-SWA
Disallow: /rates
Disallow: /*/rates

User-agent: SemrushBot-CT
Disallow: /rates
Disallow: /*/rates

# ------------------------------------------------------------------
# Other AI bot overrides (explicitly allowed)
# ------------------------------------------------------------------
User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: Anthropic-ai
Allow: /

# ------------------------------------------------------------------
# Search-engine crawlers (explicit allows for clarity)
# ------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: YandexBot
Allow: /

# ------------------------------------------------------------------
# Sitemaps & AI manifests
# ------------------------------------------------------------------
# Human-HTML sitemap
Sitemap: https://proxy.kismet.travel/sitemap.xml

# AI sitemap for machine endpoints
Sitemap: https://proxy.kismet.travel/ai-sitemap.xml

# Machine-readable travel graph (MCP) manifest
Sitemap: https://proxy.kismet.travel/.well-known/mcp.json

# ChatGPT / OpenAI plugin manifest
AI-Plugin: https://proxy.kismet.travel/.well-known/ai-plugin.json

# LLM discovery files
LLMs-Txt: https://proxy.kismet.travel/llms.txt
LLM-Index: https://proxy.kismet.travel/.well-known/llm-index.json