# robots.txt for i4net.co.il
# Updated: 2026-06-07

User-agent: *
Allow: /
Disallow: /blog-admin
Disallow: /admin
Disallow: /api

# Specific AI Crawlers - Welcome!
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: cohere-ai
Allow: /

# Search Engine Crawlers
User-agent: Googlebot
Allow: /
Crawl-delay: 0.5

User-agent: Bingbot
Allow: /
Crawl-delay: 0.5

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# Sitemap
Sitemap: https://www.i4net.co.il/sitemap.xml

# AI Language Model Optimization Files
LLMs: https://www.i4net.co.il/llms.txt
AI-Context: https://www.i4net.co.il/ai-context.json
AI-Training: https://www.i4net.co.il/ai-training-data.txt
AI-Plugin: https://www.i4net.co.il/.well-known/ai-plugin.json

# Host
Host: www.i4net.co.il