# OEMup robots.txt
# Site:    https://www.oemup.app/
# Policy:  default crawlers allowed, AI search crawlers explicitly allowed,
#          AI training crawlers explicitly blocked, /api/ disallowed.
# Updated: 2026-05-28

# ----------------------------------------------------------------
# Default policy for all conventional search crawlers
# ----------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*&utm_
# Image and asset paths are explicitly allowed so Googlebot-Image
# and AI engines can index the screenshots / logo / OG covers.
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /*.webp$
Allow: /*.png$
Allow: /*.svg$

# ----------------------------------------------------------------
# Search-time AI crawlers — ALLOWED (these surface citations to users)
# ----------------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: PhindBot
Allow: /

# Google-Extended feeds Google AI Overviews (search-time, not training).
# Allowing this is required to be eligible for AI Overview citations.
User-agent: Google-Extended
Allow: /

# ----------------------------------------------------------------
# Training-only crawlers — BLOCKED (no value, just data extraction)
# ----------------------------------------------------------------
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: ProRataInc
Disallow: /

# ----------------------------------------------------------------
# SEO scraper bots — BLOCKED (consume bandwidth, no benefit to us)
# ----------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# ----------------------------------------------------------------
# Sitemaps + AI-discovery files
# ----------------------------------------------------------------
Sitemap: https://www.oemup.app/sitemap.xml
# AI-discovery files (LLMs:/LLMs-Full: are NOT valid robots.txt directives, so
# they are kept as comments to avoid "invalid format" errors. Crawlers find
# these at their well-known paths regardless:
#   https://www.oemup.app/llms.txt
#   https://www.oemup.app/llms-full.txt