##############################################
# robots.txt – Skillang.com
# Optimized for SEO & Security
# Last-updated: 22-Aug-2025
##############################################

# — 1. Global rules (apply to ALL crawlers) —
User-agent: *
# Block confidential sections
Disallow: /admin/          # Strapi/CMS back-office
Disallow: /internal/       # Staging, experiments

# ⚠ Important: Allow API content for SEO (only public)
Allow: /api/public/
Disallow: /api/secure/
Disallow: /api/private/

# Allow assets for rendering
Allow: /static/
Allow: /uploads/

# Sitemap
Sitemap: https://www.skillang.com/sitemap.xml

# — 2. Google & Bing (ensure full crawl) —
User-agent: Googlebot
Disallow:    # nothing blocked
Allow: /

User-agent: Googlebot-Image
Disallow:
Allow: /

User-agent: AdsBot-Google
Disallow:
Allow: /

User-agent: Bingbot
Disallow:
Allow: /

# — 3. Block AI/data-mining bots (safe) —
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Bytespider
Disallow: /

# — 4. Optional (Yandex only) —
Host: www.skillang.com