# # robots.txt for http://www.w3.org/ # # $Id: robots.txt,v 1.45 2006/06/05 01:11:19 ted Exp $ # # For use by search.w3.org #User-agent: W3C-gsa #Disallow: /Out-Of-Date User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) Disallow: / # W3C Link checker User-agent: W3C-checklink Disallow: #GOOGLE ROBOT User-agent: Googlebot Disallow: #YAHOO ROBOT SLURP User-agent: Slurp Disallow: #GOOGLE ROBOT IMAGE User-agent: Googlebot-Image Disallow: #YAHOO ROBOT CRAWLER User-agent: yahoo-mmcrawler Disallow: #MSN ROBOT User-agent: psbot Disallow: # exclude some access-controlled areas User-agent: * Disallow: /_notes Disallow: /backup Disallow: /BoletoWebCaixa Disallow: /extra Disallow: /img Disallow: /css Disallow: /flv Disallow: /Library Disallow: /tableless Disallow: /imagens Disallow: /admin Disallow: /palestras Disallow: /biblioteca User-agent: * Disallow: Disallow: /flv Disallow: /extra Disallow: /backup Disallow: /img