User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: CCBot
User-agent: cohere-ai
User-agent: Cohere-AI
User-agent: Amazonbot
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Kagibot
User-agent: Mistral-AI
User-agent: MistralAI-User
User-agent: xAI-Bot
User-agent: Grok-Bot
Allow: /
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /rss.xml
Allow: /atom.xml
Disallow: /admin/*
Disallow: /auth/*
Disallow: /login*
Disallow: /signup*
Disallow: /onboarding
Disallow: /my/*
Disallow: /search*
Disallow: /users/search
Disallow: /submissions/*
Disallow: /submit
Disallow: /newsletter/confirm-email*
Disallow: /newsletter/unsubscribe*
Disallow: /products/*/embed
Disallow: /products/*/verify
Disallow: /pay/*
Disallow: /comments/product/*
Disallow: /reviews/product/*
Disallow: /dethrone/go/*
Disallow: /dethrone/feed
Disallow: /health
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?*boost=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: msnbot
User-agent: DuckDuckBot
User-agent: Slurp
User-agent: Yandex
Allow: /
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /rss.xml
Allow: /atom.xml
Disallow: /admin/*
Disallow: /auth/*
Disallow: /login*
Disallow: /signup*
Disallow: /onboarding
Disallow: /my/*
Disallow: /search*
Disallow: /users/search
Disallow: /submissions/*
Disallow: /submit
Disallow: /newsletter/confirm-email*
Disallow: /newsletter/unsubscribe*
Disallow: /products/*/embed
Disallow: /products/*/verify
Disallow: /pay/*
Disallow: /comments/product/*
Disallow: /reviews/product/*
Disallow: /dethrone/go/*
Disallow: /dethrone/feed
Disallow: /health
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?*boost=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

User-agent: *
Allow: /
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /rss.xml
Allow: /atom.xml
Disallow: /admin/*
Disallow: /auth/*
Disallow: /login*
Disallow: /signup*
Disallow: /onboarding
Disallow: /my/*
Disallow: /search*
Disallow: /users/search
Disallow: /submissions/*
Disallow: /submit
Disallow: /newsletter/confirm-email*
Disallow: /newsletter/unsubscribe*
Disallow: /products/*/embed
Disallow: /products/*/verify
Disallow: /pay/*

# Disable indexing of fragment, utility, and duplicate query endpoints.
Disallow: /comments/product/*
Disallow: /reviews/product/*
Disallow: /dethrone/go/*
Disallow: /dethrone/feed
Disallow: /health
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?*boost=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: rogerbot
Allow: /
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /rss.xml
Allow: /atom.xml
Disallow: /admin/*
Disallow: /auth/*
Disallow: /login*
Disallow: /signup*
Disallow: /onboarding
Disallow: /my/*
Disallow: /search*
Disallow: /users/search
Disallow: /submissions/*
Disallow: /submit
Disallow: /newsletter/confirm-email*
Disallow: /newsletter/unsubscribe*
Disallow: /products/*/embed
Disallow: /products/*/verify
Disallow: /pay/*
Disallow: /comments/product/*
Disallow: /reviews/product/*
Disallow: /dethrone/go/*
Disallow: /dethrone/feed
Disallow: /health
Disallow: /*?*utm_
Disallow: /*?*ref=
Disallow: /*?*gclid
Disallow: /*?*fbclid
Disallow: /*?*boost=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?q=
Crawl-delay: 5

User-agent: PetalBot
User-agent: Bytespider
Disallow: /

Sitemap: https://startupbase.io/sitemap.xml
# LLM-friendly summary: https://startupbase.io/llms.txt
# LLM-friendly full mirror: https://startupbase.io/llms-full.txt
# RSS feed: https://startupbase.io/rss.xml
# Atom feed: https://startupbase.io/atom.xml
# AI usage policy: https://startupbase.io/ai.txt
Host: startupbase.io