# Global rules for all crawlers
User-agent: *
# 2026-08-31: generated search URLs (search=/first=/format=/frames=) consumed
# ~97% of Googlebot's crawl budget -- 443 hits to works/Search vs 34 to real
# papers in one day -- and produce endless near-duplicate and empty result
# pages (soft 404s). Crawlers should reach papers, not the search machinery.
# These Disallow lines MUST precede the broad Allow lines below: Google picks
# the longest match, but many crawlers take the first match that hits.
Disallow: /cgi-bin/works/Search
Disallow: /cgi-bin/works/SearchToFrameL
Disallow: /cgi-bin/works/SearchForm
Disallow: /cgi-bin/works/AdvancedSearchForm
Disallow: /cgi-bin/works/LoginOrRegister
Disallow: /cgi-bin/works/BasketAddOne
Disallow: /cgi-bin/refs/Search
Disallow: /cgi-bin/refs/SearchToFrameL
Disallow: /cgi-bin/refs/SearchForm
Disallow: /cgi-bin/refs/LoginOrRegister
Disallow: /cgi-bin/refs/BasketAddOne
Disallow: /cgi-bin/refs/Go
Disallow: /download/
Disallow: /stream/
Disallow: /temp/
Disallow: /cache/
Disallow: /admin/
Disallow: /api/
Disallow: /*?q=
Disallow: /*?page=
Disallow: /*&sort=
Allow: /cgi-bin/works/paper/
Allow: /cgi-bin/works
Allow: /cgi-bin/refs
Allow: /search
Allow: /browse
Allow: /categories
Allow: /tags

# Google Scholar - FULL ACCESS for academic indexing
User-agent: Googlebot-Scholar  
Disallow: /cgi-bin/works/Search
Disallow: /cgi-bin/works/SearchToFrameL
Disallow: /cgi-bin/works/SearchForm
Disallow: /cgi-bin/works/AdvancedSearchForm
Disallow: /cgi-bin/works/LoginOrRegister
Disallow: /cgi-bin/works/BasketAddOne
Disallow: /cgi-bin/refs/Search
Disallow: /cgi-bin/refs/SearchToFrameL
Disallow: /cgi-bin/refs/SearchForm
Disallow: /cgi-bin/refs/LoginOrRegister
Disallow: /cgi-bin/refs/BasketAddOne
Disallow: /cgi-bin/refs/Go
Allow: /cgi-bin/works/paper/
Allow: /data/works/att/
Allow: /*.pdf$
Allow: /*.doc$
Allow: /*.docx$
Allow: /cgi-bin/works
Allow: /cgi-bin/refs
Allow: /

# Regular Googlebot - standard access
# NOTE: a crawler obeys only its most specific matching group, so Googlebot
# ignores "User-agent: *" entirely -- these rules must be repeated here.
User-agent: Googlebot
Disallow: /cgi-bin/works/Search
Disallow: /cgi-bin/works/SearchToFrameL
Disallow: /cgi-bin/works/SearchForm
Disallow: /cgi-bin/works/AdvancedSearchForm
Disallow: /cgi-bin/works/LoginOrRegister
Disallow: /cgi-bin/works/BasketAddOne
Disallow: /cgi-bin/refs/Search
Disallow: /cgi-bin/refs/SearchToFrameL
Disallow: /cgi-bin/refs/SearchForm
Disallow: /cgi-bin/refs/LoginOrRegister
Disallow: /cgi-bin/refs/BasketAddOne
Disallow: /cgi-bin/refs/Go
Disallow: /download/
Disallow: /stream/
Disallow: /temp/
Disallow: /cache/
Disallow: /admin/
Disallow: /api/
Allow: /cgi-bin/works/paper/
Allow: /cgi-bin/works
Allow: /cgi-bin/refs
Allow: /*.pdf$
Allow: /*.doc$
Allow: /*.docx$

# Googlebot-Image - no image indexing needed
User-agent: Googlebot-Image
Disallow: /

# Legitimate academic/research services
User-agent: TurnitinBot
Allow: /cgi-bin/works
Allow: /cgi-bin/refs
Allow: /*.pdf$
Allow: /*.doc$
Allow: /*.docx$

# Social media link previews (allowed)
User-agent: facebookexternalhit
Allow: /cgi-bin/works
Allow: /cgi-bin/refs
Allow: /*.pdf$

User-agent: Twitterbot
Allow: /cgi-bin/works
Allow: /cgi-bin/refs

# Note: Commercial bots (Amazonbot, AhrefsBot, Bytespider, etc.) 
# are blocked by server-level 429 responses regardless of robots.txt

# Add sitemap reference
Sitemap: https://papers.cumincad.org/sitemap/sitemap.xml
Sitemap: https://papers.cumincad.org/sitemap/sitemap.txt