# sereda.ai — robots.txt
# https://sereda.ai
# Updated: 2026-07-17

# ─────────────────────────────────────────────────────────────
#  ALLOWED CRAWLERS (Google, Bing, etc.)
# ─────────────────────────────────────────────────────────────
# `Allow: /` already permits every public path — the per-path Allow lines below
# it were redundant, and their locale list silently went stale (it/es blog was
# missing after SRD2-468 added those locales, implying a block that never
# existed). The authoritative list of what we WANT crawled is the sitemap;
# robots.txt only states what is off-limits.
User-agent: *
Allow: /

# Private / app areas — do not index
Disallow: /api/
Disallow: /sa/
Disallow: /partner/
Disallow: /create-project
Disallow: /blog/admin/
Disallow: /login
Disallow: /register
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /set-password
Disallow: /p/
Disallow: /meet/
Disallow: /schedule/
Disallow: /portal/
Disallow: /offer/
Disallow: /verify/
Disallow: /s/
Disallow: /welcome

Crawl-delay: 2

# ─────────────────────────────────────────────────────────────
#  YANDEX — fully blocked
# ─────────────────────────────────────────────────────────────
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: YandexMetrika
Disallow: /

# ─────────────────────────────────────────────────────────────
#  AGGRESSIVE SEO CRAWLERS — blocked
# ─────────────────────────────────────────────────────────────
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Allow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ia_archiver
Disallow: /

# ─────────────────────────────────────────────────────────────
#  AI TRAINING CRAWLERS
#  We allow indexing for answers (Perplexity, Bing AI),
#  but block mass training scrapers.
#  See also: https://sereda.ai/llms.txt
# ─────────────────────────────────────────────────────────────

# OpenAI — block training, allow browsing plugin
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Common Crawl (used by many LLM datasets)
User-agent: CCBot
Disallow: /

# Perplexity — allow (answer engine, drives real traffic)
User-agent: PerplexityBot
Allow: /

# Meta
User-agent: FacebookBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Apple
User-agent: Applebot-Extended
Disallow: /

# ─────────────────────────────────────────────────────────────
#  SITEMAPS
# ─────────────────────────────────────────────────────────────
Sitemap: https://sereda.ai/sitemap.xml