# The Agents Index. Crawl freely.
# Machine map: https://theagentsindex.com/llms.txt

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /api/search
Allow: /api/submit
Allow: /api/submissions/
Allow: /api/lead
Allow: /api/newsletter/subscribe
Disallow: /api/
Disallow: /dashboard
Disallow: /go/
Disallow: /*?*sort=
Disallow: /*?*page=
Disallow: /*?*filter=
Disallow: /compare/*-vs-*-vs-*
Disallow: /compare/s/

Sitemap: https://theagentsindex.com/sitemap.xml