# https://shipfit.ai/robots.txt
# Sitemap reference must come first per the spec — GSC + Bingbot read it
# before applying any user-agent rules below.
Sitemap: https://shipfit.ai/sitemap-index.xml

# ─────────────────────────────────────────────────────────────────────
# Default policy: everything crawlable except tracking-param URL variants
# and the admin/api surfaces (these don't render on the marketing site
# today, but the disallow is defensive in case a future route lands).
# ─────────────────────────────────────────────────────────────────────
User-agent: *
Allow: /
Allow: /og/
Allow: /sitemap-index.xml
Disallow: /admin/
Disallow: /api/
Disallow: /*?utm_*
Disallow: /*?gclid*
Disallow: /*?fbclid*

# ─────────────────────────────────────────────────────────────────────
# LLM / AI crawler allow-list. ShipFit is happy to be crawled and cited
# by AI assistants. See /llms.txt for the curated entry point and
# /llms-full.txt for the full-content machine-readable feed.
# ─────────────────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ─────────────────────────────────────────────────────────────────────
# Competitor backlink scrapers (Ahrefs / Semrush / Majestic): NOT blocked
# by default. We want our own audits to work. Uncomment the block below
# if/when obscuring the link profile from competitors becomes the
# priority (tradeoff: it blinds your own Ahrefs/Semrush dashboards too).
# ─────────────────────────────────────────────────────────────────────
# User-agent: AhrefsBot
# Crawl-delay: 5
# User-agent: SemrushBot
# Crawl-delay: 5
# User-agent: MJ12bot
# Crawl-delay: 5

# Notable absences:
# - No Crawl-delay on Googlebot or Bingbot. Slowing them down burns
#   crawl budget on a 60-page site for no real upside.
# - No Disallow on /_astro/ — Astro asset URLs are content-hashed and
#   exist for legitimate page rendering; blocking them breaks rich
#   results that need CSS/JS to render the page for the bot.