User-agent: Googlebot-Image Disallow:/ User-agent: ia_archiver Disallow:/ User-agent: yahoo-mmcrawler Disallow:/ User-agent: psbot Disallow:/ # Begin Exclusion From Directories from robots.txt User-agent: * Disallow: /cgi-bin/ Disallow: /*.js$ Disallow: /*.cgi$ Disallow: /*.xhtml$ Disallow: /*.inc$ Disallow: /*.css$ Disallow: /themes/ Disallow: /manage.php Disallow: /manage/ #Crawl-delay: 10