# robots.txt for https://www.abstria.com/

User-agent: *
Allow: /

# - Block private / non-indexable areas -
Disallow: /app/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /account/
Disallow: /settings/
Disallow: /user/
Disallow: /users/
Disallow: /profile/
Disallow: /auth/
Disallow: /login
Disallow: /logout
Disallow: /signup
Disallow: /register
Disallow: /callback/
Disallow: /oauth/
Disallow: /reset/
Disallow: /invite/
Disallow: /verify/

# - Block API & machine endpoints -
Disallow: /api/
Disallow: /graphql
Disallow: /graphql/
Disallow: /.well-known/

# - Framework internals (safe for Next.js / modern stacks) -
# Allow Next.js image optimization and static assets
Allow: /_next/static/
Allow: /_next/image
Allow: /_next/image?
# Block Next.js data fetching routes (but not images/static)
Disallow: /_next/data/
Disallow: /_next/server/

# - Allow public assets (images, videos, icons, etc.) -
Allow: /assets/
Allow: /public/
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.webp
Allow: /*.ico
Allow: /*.mp4
Allow: /*.webm

# - Reduce duplicate crawling from tracking params -
# Block pages with common tracking parameters (not asset URLs)
Disallow: /*?utm_
Disallow: /*?*&utm_
Disallow: /*?gclid=
Disallow: /*?*&gclid=
Disallow: /*?fbclid=
Disallow: /*?*&fbclid=
Disallow: /*?ref=
Disallow: /*?*&ref=

# - Named AI crawler groups (explicit; same access policy as User-agent: *) -
# Per the robots.txt spec, named groups do not inherit rules from User-agent: *,
# so the same disallow rules are repeated here for these bots.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: Bytespider
Allow: /
Disallow: /app/
Disallow: /dashboard/
Disallow: /admin/
Disallow: /account/
Disallow: /settings/
Disallow: /user/
Disallow: /users/
Disallow: /profile/
Disallow: /auth/
Disallow: /login
Disallow: /logout
Disallow: /signup
Disallow: /register
Disallow: /callback/
Disallow: /oauth/
Disallow: /reset/
Disallow: /invite/
Disallow: /verify/
Disallow: /api/
Disallow: /graphql
Disallow: /graphql/
Disallow: /.well-known/
Disallow: /_next/data/
Disallow: /_next/server/

# - Sitemap (update if different) -
Sitemap: https://www.abstria.com/sitemap.xml