# Robots.txt for Memoro.ai
# https://www.memoro.ai
# Last updated: 2025-01-04

# Default rules for all bots
User-agent: *
Allow: /
Allow: /en/
Allow: /features
Allow: /prices
Allow: /download
Allow: /blog
Allow: /team
Allow: /testimonials

# Block technical and admin paths
Disallow: /api/
Disallow: /_astro/
Disallow: /admin/
Disallow: /private/
Disallow: /404
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /search

# Block old German URLs (now redirected)
Disallow: /de/

# Block duplicate content patterns
Disallow: /*?*
Disallow: /*#*

# Crawl-delay in seconds (removed for better crawling)
# Crawl-delay: 1

# Sitemap location
Sitemap: https://memoro.ai/sitemap-index.xml

# Common search engine bots
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 1

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /