# # robots.txt for http://www.w3.org/ # # $Id: robots.txt,v 1.45 2006/06/05 01:11:19 ted Exp $ # # For use by search.w3.org User-agent: W3C-gsa Allow: / User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot) Allow: / # W3C Link checker User-agent: W3C-checklink Allow: # exclude some access-controlled areas User-agent: * Allow: / User-Agent: almaden Allow: / User-Agent: ASPSeek Allow: / User-Agent: Axmo Allow: / User-Agent: BaiduSpider Allow: / User-Agent: booch Allow: / User-Agent: DTS Agent Allow: / User-Agent: Downloader Allow: / User-Agent: EmailCollector Allow: / User-Agent: EmailSiphon Allow: / User-Agent: EmailWolf Allow: / User-Agent: Expired Domain Sleuth Allow: / User-Agent: Franklin Locator Allow: / User-Agent: Gaisbot Allow: / User-Agent: grub Allow: / User-Agent: HughCrawler Allow: / User-Agent: iaea.org Allow: / User-Agent: lcabotAccept Allow: / User-Agent: IconSurf Allow: / User-Agent: Iltrovatore-Setaccio Allow: / User-Agent: Indy Library Allow: / User-Agent: IUPUI Allow: / User-Agent: Kittiecentral Allow: / User-Agent: iaea.org Allow: / User-Agent: larbin Allow: / User-Agent: lwp-trivial Allow: / User-Agent: MetaTagRobot Allow: / User-Agent: Missigua Locator Allow: / User-Agent: NetResearchServer Allow: / User-Agent: NextGenSearch Allow: / User-Agent: NPbot Allow: / User-Agent: Nutch Allow: / User-Agent: ObjectsSearch Allow: / User-Agent: Oracle Ultra Search Allow: / User-Agent: PEERbot Allow: / User-Agent: PictureOfInternet Allow: / User-Agent: PlantyNet Allow: / User-Agent: QuepasaCreep Allow: / User-Agent: ScSpider Allow: / User-Agent: SOFT411 Allow: / User-Agent: spider.acont.de Allow: / User-Agent: Sqworm Allow: / User-Agent: SSM Agent Allow: / User-Agent: TAMU Allow: / User-Agent: TheUsefulbot Allow: / User-Agent: TurnitinBot Allow: / User-Agent: Tutorial Crawler Allow: / User-Agent: TutorGig Allow: / User-Agent: WebCopier Allow: / User-Agent: WebZIP Allow: / User-Agent: ZipppBot Allow: / User-Agent: Xenu Allow: / User-Agent: Wotbox Allow: / User-Agent: Wget Allow: / User-Agent: NaverBot Allow: / User-Agent: mozDex Allow: /