User-agent: *   # robots.txt directed to all spiders, 

Disallow: /cgi-bin
Disallow: /logs
Disallow: /public_html/stats
Disallow: /admin
Disallow: /assets
Disallow: /includes