# /robots.txt file for http://webcrawler.com/ 
# mail webmaster@webcrawler.com for constructive criticism  

User-agent: * 
Disallow: /bbclone2
Disallow: /bbclone2/*
Disallow: /blog
Disallow: /blog/*
Disallow: /blog2
Disallow: /blog2/*
Disallow: /cgi-bin 
Disallow: /ebay 
Disallow: /firefox 
Disallow: /gbook 
Disallow: /guestbook 
Disallow: /images 
Disallow: /mail 
Disallow: /nonhtmlimages 
Disallow: /old-snig 
Disallow: /suspended.page 
Disallow: /upload 
Disallow: /z
Crawl-delay: 7200


User-agent: webcrawler 
Disallow: *

User-agent: Slurp 
Disallow: / 
# /robots.txt file for http://webcrawler.com/ 
# mail webmaster@webcrawler.com for constructive criticism  

User-agent: * 
Disallow: /bbclone2
Disallow: /bbclone2/*
Disallow: /blog
Disallow: /blog/*
Disallow: /blog2
Disallow: /blog2/*
Disallow: /cgi-bin 
Disallow: /ebay 
Disallow: /firefox 
Disallow: /gbook 
Disallow: /guestbook 
Disallow: /images 
Disallow: /mail 
Disallow: /nonhtmlimages 
Disallow: /old-snig 
Disallow: /suspended.page 
Disallow: /upload 
Disallow: /z
Crawl-delay: 7200


User-agent: webcrawler 
Disallow: *

User-agent: Slurp 
Disallow: / 
Disallow: /bbclone2
Disallow: /bbclone2/*
Disallow: /blog
Disallow: /blog/*
Disallow: /blog2
Disallow: /blog2/*
Disallow: /cgi-bin 
Disallow: /dd 
Disallow: /ebay 
Disallow: /firefox 
Disallow: /gbook 
Disallow: /guestbook 
Disallow: /images 
Disallow: /mail 
Disallow: /nonhtmlimages 
Disallow: /old-snig 
Disallow: /suspended.page 
Disallow: /upload 
Disallow: /z
Crawl-delay: 7200  Disallow: /cgi-bin 
Disallow: /dd 
Disallow: /ebay 
Disallow: /firefox 
Disallow: /gbook 
Disallow: /guestbook 
Disallow: /images 
Disallow: /mail 
Disallow: /nonhtmlimages 
Disallow: /old-snig 
Disallow: /pagesnig 
Disallow: /suspended.page 
Disallow: /upload 
Disallow: /z 
Crawl-delay: 7200