#disallow annoymous robot to harvest all files and folders.
User-agent: bot/

Disallow:/

User-agent: bot-

disallow:/

User-agent: spider

Disallow:/

User-agent: robot

Disallow:/

User-agent: crawl

Disallow:/