# ToolifyGen robots.txt
# https://toolifygen.com

User-agent: *
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /profile
Disallow: /saved
Disallow: /saved-tools
Disallow: /settings
Disallow: /tooltip-demo
Disallow: /*?auth=
Disallow: /*?redirect=
# Note: ?ref=, ?utm_*, ?fbclid=, ?gclid= are NOT blocked here.
# Canonical tags consolidate those variants into the clean URL; blocking
# them in robots.txt would prevent Google from seeing the canonical.

# Crawl rate for well-behaved bots
Crawl-delay: 1

# Googlebot (ignores Crawl-delay, but respects Allow/Disallow)
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /profile
Disallow: /saved
Disallow: /saved-tools
Disallow: /settings
Disallow: /tooltip-demo
Disallow: /*?auth=
Disallow: /*?redirect=

# Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Block AI training crawlers (preserve content value)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Sitemaps
Sitemap: https://toolifygen.com/sitemap.xml
Sitemap: https://games.toolifygen.com/sitemap.xml
Sitemap: https://thumbnail.toolifygen.com/sitemap.xml
Host: https://toolifygen.com