# robots.txt - Bapenda Pangandaran
# Hanya block path teknis & scraper agresif. Path admin sengaja TIDAK dicantumkan.

User-agent: *
Disallow: /assets/uploads/signing/
Disallow: /assets/invoice/
Disallow: /assets/barcode/
Disallow: /application/
Disallow: /system/
Disallow: /vendor/
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*?*sort=
Disallow: /*?*filter=
Crawl-delay: 5

# Scraper SEO komersial (tidak memberi value untuk situs pemerintah)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Bot AI training (opsional, lindungi konten dari scraping LLM)
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

# Sitemap (uncomment ketika sitemap.xml sudah dibuat)
Sitemap: https://bapenda.pangandarankab.go.id/sitemap.xml