# Solid# Public Website - robots.txt
# https://solidnumber.com
# AI-Native Business Automation Platform

# ===========================================
# DEFAULT RULES (All User Agents)
# ===========================================
User-agent: *

# Block demo/template pages from indexing
Disallow: /demo/
Disallow: /demo-dexter/
Disallow: /demo-basecamp/
Disallow: /demo-shop/
Disallow: /demo-redsky/

# Block internal API routes but allow MCP for AI agents
Disallow: /api/
Allow: /api/mcp
Allow: /api/mcp/

# Allow main site pages
Allow: /
Allow: /about
Allow: /pricing
Allow: /features
Allow: /contact
Allow: /careers
Allow: /login
Allow: /integrations
Allow: /faq
Allow: /blog
Allow: /ai-index
Allow: /ai-payments
Allow: /solid-number

# Allow AI discovery files
Allow: /.well-known/
Allow: /ai.txt
Allow: /llms.txt
Allow: /sitemap.xml

# ===========================================
# GOOGLE AI CRAWLER (Gemini/SGE)
# ===========================================
User-agent: Google-Extended
Allow: /
Allow: /api/mcp
Allow: /.well-known/
Disallow: /demo/
Disallow: /api/
Allow: /api/mcp

# ===========================================
# OPENAI CRAWLER (ChatGPT/GPT)
# ===========================================
User-agent: GPTBot
Allow: /
Allow: /api/mcp
Allow: /.well-known/
Disallow: /demo/
Disallow: /api/
Allow: /api/mcp

User-agent: ChatGPT-User
Allow: /
Allow: /api/mcp
Allow: /.well-known/

# ===========================================
# ANTHROPIC CRAWLER (Claude)
# ===========================================
User-agent: anthropic-ai
Allow: /
Allow: /api/mcp
Allow: /.well-known/
Disallow: /demo/

User-agent: Claude-Web
Allow: /
Allow: /api/mcp
Allow: /.well-known/

# ===========================================
# PERPLEXITY CRAWLER
# ===========================================
User-agent: PerplexityBot
Allow: /
Allow: /api/mcp
Allow: /.well-known/
Disallow: /demo/

# ===========================================
# MICROSOFT/BING CRAWLER (Copilot)
# ===========================================
User-agent: Bingbot
Allow: /
Allow: /api/mcp
Allow: /.well-known/
Disallow: /demo/

# ===========================================
# COHERE CRAWLER
# ===========================================
User-agent: cohere-ai
Allow: /
Allow: /api/mcp
Allow: /.well-known/

# ===========================================
# META AI CRAWLER
# ===========================================
User-agent: FacebookBot
Allow: /
Allow: /.well-known/
Disallow: /api/

User-agent: meta-externalagent
Allow: /
Allow: /api/mcp
Allow: /.well-known/

# ===========================================
# SITEMAPS
# ===========================================
Sitemap: https://solidnumber.com/sitemap.xml

# ===========================================
# AI DISCOVERY FILES
# ===========================================
# MCP Server: https://solidnumber.com/api/mcp
# MCP Manifest: https://solidnumber.com/.well-known/mcp.json
# AI Plugin: https://solidnumber.com/.well-known/ai-plugin.json
# Anthropic Plugin: https://solidnumber.com/.well-known/anthropic-ai-plugin.json
# OpenAPI Spec: https://solidnumber.com/.well-known/openapi.json
# LLMs.txt: https://solidnumber.com/llms.txt
# AI.txt: https://solidnumber.com/ai.txt
# AI Index: https://solidnumber.com/ai-index