# GhostEngine - The Marketing Operating System
# https://ghostengine.ai

# Sitemap location for all crawlers
Sitemap: https://ghostengine.ai/sitemap.xml

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Social Media Crawlers (for rich previews)
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# Other Major Search Engines
User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# AI Crawlers (for LLM training - allowing for brand visibility)
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: PerplexityBot
Allow: /

# SEO Tools (for monitoring)
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: MJ12bot
Allow: /

# Default rule for all other bots
User-agent: *
Allow: /
Crawl-delay: 2

# Disallow paths that shouldn't be indexed
Disallow: /api/
Disallow: /*.json$
Disallow: /lovable-uploads/