User-agent: *
Allow: /

Disallow: /admin
Disallow: /auth/
Disallow: /profile
Disallow: /settings
Disallow: /corrections
Disallow: /lists
Disallow: /reset-hints
Disallow: /api/

# Keep Wayback captures focused on user-facing pages, not app internals.
User-agent: archive.org_bot
User-agent: special_archiver
User-agent: ia_archiver
User-agent: ia_archiver-web.archive.org
Allow: /
Disallow: /_nicegui/
Disallow: /static/
Disallow: /favicon.ico
Disallow: /search
Disallow: /parallels
Disallow: /admin
Disallow: /auth/
Disallow: /profile
Disallow: /settings
Disallow: /corrections
Disallow: /lists
Disallow: /reset-hints
Disallow: /api/

# SEO-tool crawlers: no search-engine value, constant heavy load on
# /browse (2026-07-08: SemrushBot crawled ~7K pages/day, 24/7).
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: serpstatbot
Disallow: /

Sitemap: https://genizahsearch.com/sitemap.xml