# =========================================================
# ROBOTS.TXT - https://www.23andme.org/
# (c) 2026 23andMe Research Institute. All rights reserved.
# =========================================================

# ---------------------------------------------------------
# 1. DEFAULT RULE - all crawlers not named below
# ---------------------------------------------------------
User-agent: *
Allow: /
# Privacy & account security (sensitive genetic data)
Disallow: /account/
Disallow: /login/
# E-commerce & checkout
Disallow: /cart/
Disallow: /checkout/
Disallow: /*?*add-to-cart
# Internal API & framework
Disallow: /api/
Disallow: /_next/data/
Disallow: /cgi-bin/
# Internal search & tracking parameters
Disallow: /search
Disallow: /*?q=
Disallow: /*?*sessionid
Disallow: /*?*utm_
Disallow: /*?*gclid
Disallow: /*?*fbclid

# ---------------------------------------------------------
# 2. TRADITIONAL SEARCH ENGINES (organic + AI Overviews)
# ---------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Slurp
User-agent: Baiduspider
User-agent: Yandex
Allow: /
Disallow: /account/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /api/
Disallow: /_next/data/

# ---------------------------------------------------------
# 3. AI RETRIEVAL & ANSWER ENGINES (real-time citations)
# ---------------------------------------------------------
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: Google-CloudVertexBot
User-agent: Gemini-Deep-Research
User-agent: MistralAI-User
User-agent: DuckAssistBot
Allow: /
Disallow: /account/
Disallow: /dna-data/
Disallow: /raw-data/
Disallow: /start/
Disallow: /login/
Disallow: /cart/
Disallow: /checkout/
Disallow: /api/

# ---------------------------------------------------------
# 4. BULK SCRAPERS & THIRD-PARTY SEO BOTS - BLOCKED
# ---------------------------------------------------------
User-agent: CCBot
User-agent: DataForSeoBot
User-agent: Bytespider
User-agent: ImagesiftBot
User-agent: Scrapy
User-agent: PetalBot
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: Barkrowler
User-agent: BLEXBot
User-agent: Diffbot
Disallow: /

# ---------------------------------------------------------
# 6. SITEMAPS
# ---------------------------------------------------------
Sitemap: https://www.23andme.org/sitemap_index.xml
Sitemap: https://www.23andme.org/sitemap.xml
Sitemap: https://www.23andme.org/blog/sitemap.xml
Sitemap: https://www.23andme.org/ancestry/last-names/sitemap.xml
Sitemap: https://www.23andme.org/clinicians/sitemap_index.xml
Sitemap: https://www.23andme.org/education/wp-sitemap.xml
Sitemap: https://www.23andme.org/research/sitemap.xml
Sitemap: https://www.23andme.org/media-center/sitemap.xml

# AI site guide (llms.txt):
# https://www.23andme.org/llms.txt