User-agent: Twitterbot
User-agent: Mediapartners-Google
User-agent: Google-Display-Ads-Bot
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
User-agent: Googlebot
Allow: /

User-agent: bingbot
Crawl-Delay: 30
Allow: /

User-agent: Feedly
Crawl-Delay: 30
Allow: /rss
Allow: /

User-agent: CriteoBot
Disallow: /

User-agent: Slurp
Disallow: /

User-agent: Megalodon
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-Agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: claude-web
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: DiffBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Riddler
Disallow: /

User-agent: MLBot
Disallow: /

User-agent: JUST-CRAWLER
Disallow: /

User-agent: DuckDuckBot
Disallow: /

User-Agent: Sogou web spider
Disallow: /

User-Agent: Rogerbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: peer39_crawler
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ias-or
Disallow: /

User-agent: *
Disallow: /search
Disallow: /create
Disallow: /rss
Disallow: /api