# # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/robotstxt.html User-agent: * Crawl-delay: 10 # CSS, JS, Images Allow: /misc/*.css$ Allow: /misc/*.css? Allow: /misc/*.js$ Allow: /misc/*.js? Allow: /misc/*.gif Allow: /misc/*.jpg Allow: /misc/*.jpeg Allow: /misc/*.png Allow: /modules/*.css$ Allow: /modules/*.css? Allow: /modules/*.js$ Allow: /modules/*.js? Allow: /modules/*.gif Allow: /modules/*.jpg Allow: /modules/*.jpeg Allow: /modules/*.png Allow: /profiles/*.css$ Allow: /profiles/*.css? Allow: /profiles/*.js$ Allow: /profiles/*.js? Allow: /profiles/*.gif Allow: /profiles/*.jpg Allow: /profiles/*.jpeg Allow: /profiles/*.png Allow: /themes/*.css$ Allow: /themes/*.css? Allow: /themes/*.js$ Allow: /themes/*.js? Allow: /themes/*.gif Allow: /themes/*.jpg Allow: /themes/*.jpeg Allow: /themes/*.png # Directories Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /patches/ #Disallow: /profiles/ Disallow: /scripts/ Disallow: /sites/all/ Disallow: /sites/default/ Disallow: /themes/ # Files Disallow: /web.config Disallow: /aws.html Disallow: /apc.php Disallow: /authorize.php Disallow: /redirect.php Disallow: /imageDisplay.php Disallow: /kpi.php Disallow: /sso.php Disallow: /cron.php Disallow: /info.php Disallow: /install.php Disallow: /update.php Disallow: /xmlrpc.php Disallow: /crossdomain.xml Disallow: /install.php # Paths (clean URLs) Disallow: /admin/ Disallow: /comment/reply/ Disallow: /hybridauth/ Disallow: /filter/tips/ Disallow: /node/add/ # Disallow: /search/ Disallow: /profile/ Disallow: /profile-contact/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /user/logout/ Disallow: /en/ Disallow: /en/* Disallow: /user Disallow: /user/ Disallow: /user/* Disallow: /users/ Disallow: /node/ Disallow: /node Disallow: /node/* Disallow: /*sort= # Paths (no clean URLs) Disallow: /?q=admin/ Disallow: /?q=comment/reply/ Disallow: /?q=filter/tips/ Disallow: /?q=hybridauth/ Disallow: /?q=node/add/ # Disallow: /?q=search/ Disallow: /?q=profile/ Disallow: /?q=profile-contact/ Disallow: /?q=profiles/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ Disallow: /?q=user/logout/ Disallow: /?q=en/ Disallow: /?q=en/* Disallow: /?q=user/ Disallow: /?q=user/* Disallow: /?q=users/ Disallow: /?q=node/ Disallow: /?q=node Disallow: /?q=node/* Disallow: /quiz/ # Paths (newsletters broken links fix) Disallow: /content/20* # Disallow URLs with destination parameter Disallow: /vlt-auth* Disallow: /vlt-autologin-extern* Disallow: /civi-ext-url* Disallow: /user/login?destination=* Disallow: /user/register?destination=* Disallow: /user?destination=* Disallow: /user?destination=* # Disallow some bots User-agent: 007ac9 Disallow: / User-agent: 007ac9 Crawler Disallow: / User-agent: 2ADAMbot Disallow: / User-agent: 2ADAMbot/1.0 Disallow: / User-agent: 200PleaseBot Disallow: / User-agent: 360Spider Disallow: / User-agent: 8484 Boston Project v 1.0 Disallow: / User-agent: Abonti Disallow: / User-agent: Abonti/0.92 Disallow: / User-agent: Aboundex Disallow: / User-agent: AdvBot Disallow: / User-agent: AdvBot/2.0 Disallow: / User-agent: Affectv Robot v1.0/Nutch-1.6 Disallow: / User-agent: acoonbot Disallow: / User-agent: Adidxbot Disallow: / User-agent: admantx Disallow: / User-agent: AhrefsBot Disallow: / User-agent: ahrefsbot Disallow: / User-agent: aihitbot Disallow: / User-agent: AISearchBot Disallow: / User-agent: Alexibot Disallow: / User-agent: Amazonbot Disallow: /e-catalogue/ Disallow: /e-catalogue/* User-agent: Amazonbot/0.1 Disallow: /e-catalogue/ Disallow: /e-catalogue/* User-agent: antibot Disallow: / User-agent: Apexoo Spider 1.0 Disallow: / User-agent: Aqua_Products Disallow: / User-agent: arabot Disallow: / User-agent: Arachnoidea Disallow: / User-agent: ArchitextSpider Disallow: / User-agent: ASPSimply Disallow: / User-agent: asterias Disallow: / User-agent: AttributorCorporationDMCABot Disallow: / User-agent: Avant+Browser+ Disallow: / User-agent: Axonize-bot Disallow: / User-agent: b2w/0.1 Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: backlinkcrawler Disallow: / User-agent: Battleztar Bazinga Disallow: / User-agent: Battleztar Bazinga/0.01 Disallow: / User-agent: BDCbot Disallow: / User-agent: becomebot Disallow: / User-agent: BecomeBot/3.0 Disallow: / User-agent: Bender Disallow: / User-agent: betaBot Disallow: / User-agent: bibnum.bnf Disallow: / User-agent: biglotron Disallow: / User-agent: binlar_2.6.3 larbin2.6.3@unspecified.mail Disallow: / User-agent: binlar_2.6.3 phanendra_kalapala@McAfee.com Disallow: / User-agent: binlar_2.6.3 test@mgmt.mic Disallow: / User-agent: Bitacle Disallow: / User-agent: Bitacle bot Disallow: / User-agent: Bitacle bot/1.1 Disallow: / User-agent: Bitacle* Disallow: / User-agent: bl.uk_lddc_bot Disallow: / User-agent: Black Hole Disallow: / User-agent: Blackboard Safeassign Disallow: / User-agent: Blazer 1.0 Disallow: / User-agent: BLEXBot Disallow: / User-agent: BLEXBot/1.0 Disallow: / User-agent: blexbot Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: BLP_bbot Disallow: / User-agent: BLP_bbot/0.1 Disallow: / User-agent: BobCrawl Disallow: / User-agent: BOIA-Accessibility-agent Disallow: / User-agent: BOIA-Accessibility-agent/PR 1.0 Disallow: / User-agent: boitho Disallow: / User-agent: boitho.com Disallow: / User-agent: Bookmark search tool Disallow: / User-agent: Bork-edition Disallow: / User-agent: BOT for JCE Disallow: / User-agent: BOT/0.1 (BOT for JCE) Disallow: / User-agent: bot/1.0 Disallow: / User-agent: bot/1.0+(bot;+http://;+bot@bot.bot) Disallow: / User-agent: BotALot Disallow: / User-agent: botmobi Disallow: / User-agent: brainobot Disallow: / User-agent: bsalsa Disallow: / User-agent: BUbiNG Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: BunnySlippers Disallow: / User-agent: Butterfly Disallow: / User-agent: buzzbot Disallow: / User-agent: ca-crawler Disallow: / User-agent: ca-crawler/1.0 Disallow: / User-agent: CamelHttpStream/1.0 Evolution/3.2.2 Disallow: / User-agent: careerbot Disallow: / User-agent: Cazoodle-Bot Disallow: / User-agent: CazoodleBot Disallow: / User-agent: CB/Nutch-1.7 Disallow: / User-agent: CC Metadata Scaper Disallow: / User-agent: ccbot Disallow: / User-agent: CCBot Disallow: / User-agent: CCBot/2.0 Disallow: / User-agent: Cegbfeieh Disallow: / User-agent: changedetection Disallow: / User-agent: Charlotte Disallow: / User-agent: CherryPicker Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: citeseerxbot Disallow: / User-agent: classroombot Disallow: / User-agent: Cliqzbot Disallow: / User-agent: Cliqzbot/0.1 Disallow: / User-agent: CMS Crawler Disallow: / User-agent: coccoc Disallow: / User-agent: colly - https://github.com/gocolly/colly/v2 Disallow: / User-agent: Commons-HttpClient Disallow: / User-agent: content crawler spider Disallow: / User-agent: convera Disallow: / User-agent: ConveraCrawler/0.9d Disallow: / User-agent: ConveraCrawler/0.9e Disallow: / User-agent: Copernic Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: Copyscape Disallow: / User-agent: cosmos Disallow: / User-agent: CrazyWebCrawler Disallow: / User-agent: CrazyWebCrawler-Spider Disallow: / User-agent: crawler4j Disallow: / User-agent: crawl13.exabot.com Disallow: / User-agent: crawl6.exabot.com Disallow: / User-agent: Crescent Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CrystalSemanticsBot Disallow: / User-agent: crystalsemanticsbot Disallow: / User-agent: CyberPatrol Disallow: / User-agent: cyberpatrol Disallow: / User-agent: cyberpatrolcrawler Disallow: / User-agent: datacha0s/2.0 Disallow: / User-agent: DataparkSearch/4.47 (+http://dataparksearch.org/bot) Disallow: / User-agent: DataparkSearch/4.xx (http://www.dataparksearch.org/) Disallow: / User-agent: DBLBot Disallow: / User-agent: dcbot Disallow: / User-agent: deadlinkchecker Disallow: / User-agent: DepSpid Disallow: / User-agent: discobot Disallow: / User-agent: Discobot Disallow: / User-agent: discoverybot Disallow: / User-agent: DittoSpyder Disallow: / User-agent: DoCoMo Disallow: / User-agent: domaincrawler Disallow: / User-agent: dotbot Disallow: / User-agent: drupact Disallow: / User-agent: DTAAgent Disallow: / User-agent: duggmirror Disallow: / User-agent: eagle Disallow: / User-agent: ec2linkfinder Disallow: / User-agent: eCommerceBot Disallow: / User-agent: edisterbot Disallow: / User-agent: elisabot Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Embedly Disallow: / User-agent: EPiServer Link Checker Disallow: / User-agent: envolk Disallow: / User-agent: EroCrawler Disallow: / User-agent: EUDORA Disallow: / User-agent: EuripBot Disallow: / User-agent: EuripBot/2.0 Disallow: / User-agent: Everest-Vulcan Disallow: / User-agent: exabot Disallow: / User-agent: Exabot Disallow: / User-agent: Exabot-Images/1.0 Disallow: / User-agent: Exabot-Images/3.0 Disallow: / User-agent: Exabot/2.0 Disallow: / User-agent: Exabot/3.0 Disallow: / User-agent: exooba Disallow: / User-agent: ExtractorPro Disallow: / User-agent: Ezooms Disallow: / User-agent: ezooms Disallow: / User-agent: FairAd Client Disallow: / User-agent: FairShare Disallow: / User-agent: fast Disallow: / User-agent: FAST Enterprise Crawler Disallow: / User-agent: fast Enterprise crawler Disallow: / User-agent: fast metaweb crawler Disallow: / User-agent: FAST-WebCrawler Disallow: / User-agent: fast-Webcrawler Disallow: / User-agent: Fetch API Request Disallow: / User-agent: findlink Disallow: / User-agent: findlinks/1.1.1-a5 Disallow: / User-agent: findlinks/1.1.3-beta9 Disallow: / User-agent: Findxbot Disallow: / User-agent: Findxbot/1.0 Disallow: / User-agent: findthatfile Disallow: / User-agent: Flaming AttackBot Disallow: / User-agent: Flamingo_SearchEngine Disallow: / User-agent: flatlandbot Disallow: / User-agent: Flipboard Robot Disallow: / User-agent: fluffy Disallow: / User-agent: FollowSiteBot Disallow: / User-agent: Foobot Disallow: / User-agent: fr-crawlerbinlarbinlar_2.6.3 binlar_2.6.3@unspecified.mail Disallow: / User-agent: FunWebProducts Disallow: / User-agent: FunWebProducts f39-2350 Disallow: / User-agent: FyberSpider Disallow: / User-agent: g00g1e.net Disallow: / User-agent: Gaisbot Disallow: / User-agent: GarlikCrawler/1.1 (http://garlik.com/, crawler@garlik.com) Disallow: / User-agent: genieBot Disallow: / User-agent: genieBot (wgao@genieknows.com) Disallow: / User-agent: genieBot+(wgao@genieknows.com) Disallow: / User-agent: genieBot+wgao@genieknows.com Disallow: / User-agent: getrequest Disallow: / User-agent: getrequest/1.0 Disallow: / User-agent: GetRight/4.2 Disallow: / User-agent: gigablast Disallow: / User-agent: GigablastOpenSource Disallow: / User-agent: GigablastOpenSource/1.0 Disallow: / User-agent: gigabot Disallow: / User-agent: Gigabot Disallow: / User-agent: Gigabot* Disallow: / User-agent: Gigabot/2.0 Disallow: / User-agent: GingerCrawler Disallow: / User-agent: Gluten Free Crawler Disallow: / User-agent: Gluten Free Crawler/1.0 Disallow: / User-agent: gnam gnam spider Disallow: / User-agent: Go-http-clientphpcrawl Disallow: / User-agent: GrapeshotCrawler Disallow: / User-agent: grub Disallow: / User-agent: grub-client Disallow: / User-agent: grub.org Disallow: / User-agent: gsa-crawler Disallow: / User-agent: gslfbot Disallow: / User-agent: GT::WWW Disallow: / User-agent: Gulliver Disallow: / User-agent: Haiula Disallow: / User-agent: Haiula/1.4 Disallow: / User-agent: HaosouSpider Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: HenryTheMiragoRobot Disallow: / User-agent: heritrix Disallow: / User-agent: heritrix/1.14.4 Disallow: / User-agent: heritrix/3.1.1 Disallow: / User-agent: heritrix/1.12.1 +http://www.page-store.com) Disallow: / User-agent: Hivemind Disallow: / User-agent: hloader Disallow: / User-agent: HMSE_Robot Disallow: / User-agent: HornySexSearch Disallow: / User-agent: http client Disallow: / User-agent: HTTP::Lite Disallow: / User-agent: http://www.almaden.ibm.com/cs/crawler Disallow: / User-agent: http://www.cuill.com/twiceler/robot.html Disallow: / User-agent: HTTP/1.1 zibber-v0.1(www.zibb.com/crawler/) Disallow: / User-agent: httplib Disallow: / User-agent: httpunit Disallow: / User-agent: HTTrack Disallow: / User-agent: httrack Disallow: / User-agent: HubSpot Links Crawler 1.0 Disallow: / User-agent: humanlinks Disallow: / User-agent: HyperCrawl Disallow: / User-agent: HyperCrawl/0.2 Disallow: / User-agent: ichiro Disallow: / User-agent: Ichiro Robot Disallow: / User-agent: ichiro/2.0 Disallow: / User-agent: ichiro/2.0 (ichiro@nttr.co.jp) Disallow: / User-agent: ichiro/2.0+(http://help.goo.ne.jp/door/crawler.html) Disallow: / User-agent: IconSurf Disallow: / User-agent: Indy Library Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: InfoPath Disallow: / User-agent: InfoPath.1 Disallow: / User-agent: InfoPath.2 Disallow: / User-agent: IntegraTelecom Disallow: / User-agent: integrity/5 Disallow: / User-agent: integromedb Disallow: / User-agent: intelium_bot Disallow: / User-agent: InterfaxScanBot Disallow: / User-agent: InterNaetBoten Disallow: / User-agent: InterNaetBoten/0.99 Disallow: / User-agent: IOI Disallow: / User-agent: ip-web-crawler.com Disallow: / User-agent: ips-agent Disallow: / User-agent: IRLbot Disallow: / User-agent: IRLbot/3.0 Disallow: / User-agent: Iron33/1.0.2 Disallow: / User-agent: ISC Systems iRc Search 2.1 Disallow: / User-agent: Isidorus Disallow: / User-agent: IstellaBot Disallow: / User-agent: it2media-domain-crawler Disallow: / User-agent: jakarta Disallow: / User-agent: Jakarta Commons Disallow: / User-agent: Jakarta Commons-HttpClient Disallow: / User-agent: Jakarta Commons-HttpClient/3.0.1 Disallow: / User-agent: Jakarta Commons-HttpClient/3.0-rcx Disallow: / User-agent: Jakarta Commons-HttpClient/3.1 Disallow: / User-agent: JamesBOT Disallow: / User-agent: Java Disallow: / User-agent: Java Bot 2.0 Disallow: / User-agent: Java/1.4.1_04 Disallow: / User-agent: Java/1.5.0_05 Disallow: / User-agent: Java/1.5.0_06 Disallow: / User-agent: Java/1.5.0_10 Disallow: / User-agent: Java/1.6.0_04 Disallow: / User-agent: Java/1.6.0_07 Disallow: / User-agent: Java/1.6.0_10-rc2 Disallow: / User-agent: Java/1.6.0_11 Disallow: / User-agent: Java/1.6.0_13 Disallow: / User-agent: Java/1.6.0_15 Disallow: / User-agent: Java/1.6.0_24 Disallow: / User-agent: Java/1.6.0_26 Disallow: / User-agent: Java/1.6.0_32 Disallow: / User-agent: Java/1.7.0_09 Disallow: / User-agent: Java/1.7.0_45 Disallow: / User-agent: Java/1.7.0_55 Disallow: / User-agent: Java/1.7.0_60 Disallow: / User-agent: Java/1.7.0_60-ea Disallow: / User-agent: Java/1.7.0_65 Disallow: / User-agent: Java/1.7.0_67 Disallow: / User-agent: Java/1.7.0_71 Disallow: / User-agent: Java/1.8.0_20 Disallow: / User-agent: Java/1.8.0_25 Disallow: / User-agent: Java/1.8.0_31 Disallow: / User-agent: JavaBot Disallow: / User-agent: JavaBot 2.0 Disallow: / User-agent: JennyBot Disallow: / User-agent: JetBrains Disallow: / User-agent: JetBrains 5.0 Disallow: / User-agent: JikeSpider Disallow: / User-agent: Jyxobot Disallow: / User-agent: jyxobot Disallow: / User-agent: Jyxobot Disallow: / User-agent: kalooga Disallow: / User-agent: KaloogaBot Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: Keyword Density/0.9 Disallow: / User-agent: kyluka Disallow: / User-agent: LaBot Disallow: / User-agent: larbin Disallow: / User-agent: larbin_2.6.1 Disallow: / User-agent: larbin_2.6.1+(larbin2.6.1@unspecified.mail) Disallow: / User-agent: laycat Disallow: / User-agent: lb-spider Disallow: / User-agent: LexiBot Disallow: / User-agent: LexiBotWeb Image Collector Disallow: / User-agent: LexxeBot Disallow: / User-agent: LexxeBot/1.0 Disallow: / User-agent: LibreOffice Disallow: / User-agent: libwww Disallow: / User-agent: libwww-perl Disallow: / User-agent: libwww-perl/5.65 Disallow: / User-agent: Linguee Disallow: / User-agent: Linguee Bot Disallow: / User-agent: LinkAider Disallow: / User-agent: Linkdex Disallow: / User-agent: linkdex Disallow: / User-agent: LinkdexBot Disallow: / User-agent: linkdexbot/2.0 Disallow: / User-agent: linkdexbot/2.1 Disallow: / User-agent: linkdex.com/v2.0 Disallow: / User-agent: linkdex.com/v2.1 Disallow: / User-agent: linkdexbot-mobile/2.1 Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: LinkpadBot Disallow: / User-agent: LinkpadBot/1.06 Disallow: / User-agent: LinkScan/8.1a Unix Disallow: / User-agent: LinkWalker Disallow: / User-agent: LinkWalker/2.0 Disallow: / User-agent: LinkWalker/2.0 - www.seventwentyfour.com Disallow: / User-agent: Lipperhey Disallow: / User-agent: Lipperhey SEO Service Disallow: / User-agent: Lipperhey-Kaus-Australis Disallow: / User-agent: Lipperhey-Kaus-Australis/5.0 Disallow: / User-agent: Lipperhey SEO Service Disallow: / User-agent: listicka Disallow: / User-agent: LMQueueBot/0.2 Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: LocalBot Disallow: / User-agent: LocalcomBot Disallow: / User-agent: looksmart Disallow: / User-agent: lssbot Disallow: / User-agent: lssrocketcrawler Disallow: / User-agent: LSSRocketCrawler Disallow: / User-agent: LSSRocketCrawler/1.0 Disallow: / User-agent: LSSRocketCrawler/1.0 LightspeedSystems Disallow: / User-agent: ltx71 Disallow: / User-agent: LWNutch/Nutch-1.4 Disallow: / User-agent: lwp-trivial Disallow: / User-agent: lwp-trivial/1.34 Disallow: / User-agent: magpie-crawler Disallow: / User-agent: Mail.RU Disallow: / User-agent: Mail.RU_Bot Disallow: / User-agent: Mail.RU_Bot/2.0 Disallow: / User-agent: Mail.RU_Bot/Fast/2.0 Disallow: / User-agent: MantraAgent Disallow: / User-agent: Mata Hari Disallow: / User-agent: MaxPointCrawler Disallow: / User-agent: MaxPointCrawler/Nutch-1.1 Disallow: / User-agent: meanpathbot Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: MegaIndex.ru/2.0 Disallow: / User-agent: MFC_Tear_Sample Disallow: / User-agent: MIIxpc Disallow: / User-agent: MIIxpc/4.2 Disallow: / User-agent: MixBot Disallow: / User-agent: Missigua Locator Disallow: / User-agent: Mister PiX Disallow: / User-agent: MJ12bot Disallow: / User-agent: MJ12bot/v1.0.8 (http://majestic12.co.uk/bot.php?+) Disallow: / User-agent: MJ12bot/v1.2.0 Disallow: / User-agent: MJ12bot/v1.2.0 (http://majestic12.co.uk/bot.php? ) Disallow: / User-agent: mlbot Disallow: / User-agent: MLBot Disallow: / User-agent: Modified GMMBotRightHere Disallow: / User-agent: moget Disallow: / User-agent: moget/2.1 Disallow: / User-agent: moget/2.1 (moget@goo.ne.jp) Disallow: / User-agent: MojeekBot\\/ Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt; DTS Agent Disallow: / User-agent: MSR-ISRCCrawler Disallow: / User-agent: msrbot Disallow: / User-agent: My Nutch Spider/Nutch-1.9 Disallow: / User-agent: nambu Disallow: / User-agent: NationalDirectory-SuperSpider Disallow: / User-agent: NaverBot Disallow: / User-agent: naverbot Disallow: / User-agent: NerdyBot Disallow: / User-agent: NerdByNature.Bot Disallow: / User-agent: NetAnts Disallow: / User-agent: netEstate NE Crawler Disallow: / User-agent: NetMechanic Disallow: / User-agent: netresearchserver Disallow: / User-agent: netseer Disallow: / User-agent: NetSeer crawler Disallow: / User-agent: nexen Disallow: / User-agent: NextGenSearchBot Disallow: / User-agent: NG/2.0 Disallow: / User-agent: ngbot Disallow: / User-agent: nicebot Disallow: / User-agent: NICErsPRO Disallow: / User-agent: niki-bot Disallow: / User-agent: NING/1.0 Disallow: / User-agent: node.io Disallow: / User-agent: node.js Disallow: / User-agent: noxtrumbot/1.0 Disallow: / User-agent: Nutch Disallow: / User-agent: nutch Disallow: / User-agent: obot Disallow: / User-agent: oBot/2.3.1 Disallow: / User-agent: Ocelli Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Offline Explorer/1.5 Disallow: / User-agent: Offline Explorer/1.9 Disallow: / User-agent: Offline Explorer/2.5 Disallow: / User-agent: omgilibot Disallow: / User-agent: omgilibot/0.4 Disallow: / User-agent: OmniExplorer Disallow: / User-agent: OmniExplorer_Bot Disallow: / User-agent: OmniExplorer_Bot/1.07 Disallow: / User-agent: OmniExplorer_Bot/1.09 Disallow: / User-agent: OmniExplorer_Bot/1.10 Disallow: / User-agent: OnTownsBot Disallow: / User-agent: OOZBOT Disallow: / User-agent: Openbot Disallow: / User-agent: Openfind Disallow: / User-agent: Openfind data gathere Disallow: / User-agent: Openstat Disallow: / User-agent: Openstat/0.1 Disallow: / User-agent: OpenX Disallow: / User-agent: OptimizationCrawler Disallow: / User-agent: OptimizationCrawler/0.2 Disallow: / User-agent: Oracle Ultra Search Disallow: / User-agent: page_prefetcher Disallow: / User-agent: page2rss Disallow: / User-agent: PageAnalyzer Disallow: / User-agent: PageAnalyzer/1.1 Disallow: / User-agent: PagesInventory Disallow: / User-agent: Pagespeedbot Disallow: / User-agent: Pagespeed/1.1 Disallow: / User-agent: Pagespeed/1.1 Fetcher Disallow: / User-agent: panscient Disallow: / User-agent: panscient.com Disallow: / User-agent: PathDefender Disallow: / User-agent: PathDefender/1.0 Disallow: / User-agent: Perl LWP Disallow: / User-agent: PetalBot Disallow: / User-agent: Plukkie Disallow: / User-agent: Plukkie/1.5 Disallow: / User-agent: PerMan Disallow: / User-agent: PHPCrawl Disallow: / User-agent: Piffany_Web_Cacher_v0.91 Disallow: / User-agent: Pixray-Seeker Disallow: / User-agent: POGS/2.0 Disallow: / User-agent: postrank Disallow: / User-agent: PRIVACY_ENFAQ.jsp Disallow: / User-agent: Prlog Disallow: / User-agent: Prlog/1.0 Disallow: / User-agent: ProPowerBot/2.14 Disallow: / User-agent: ProWebWalker Disallow: / User-agent: proximic Disallow: / User-agent: psbot Disallow: / User-agent: psbot/0.1 Disallow: / User-agent: psbot/0.1+(+http://www.picsearch.com/bot.html) Disallow: / User-agent: PSIBots Disallow: / User-agent: Python-urllib Disallow: / User-agent: QEAVis Disallow: / User-agent: QueryN Metasearch Disallow: / User-agent: R6_FeedFetcher Disallow: / User-agent: R6_CommentReader Disallow: / User-agent: Radiation Retriever 1.1 Disallow: / User-agent: RankActiveLinkBot Disallow: / User-agent: Rankivabot Disallow: / User-agent: REAP-Crawler Disallow: / User-agent: RepoMonkey Disallow: / User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: / User-agent: RetrevoPageAnalyzer Disallow: / User-agent: Riddler Disallow: / User-agent: RiceComputerArchitecture Disallow: / User-agent: RMA Disallow: / User-agent: robotgenius Disallow: / User-agent: Robozilla Disallow: / User-agent: Rome Client Disallow: / User-agent: rootlink Disallow: / User-agent: ru_com_viewer Disallow: / User-agent: RU_Bot/2.0 Disallow: / User-agent: RufusBot Disallow: / User-agent: SafeSearch microdata crawler Disallow: / User-agent: SapphireWebCrawler Disallow: / User-agent: sbider Disallow: / User-agent: SBIder Disallow: / User-agent: SBIder/SBIder-0.8.2-dev Disallow: / User-agent: scooter Disallow: / User-agent: Scooter/2.0 Disallow: / User-agent: scoutjet Disallow: / User-agent: Scrapy Disallow: / User-agent: scrapy Disallow: / User-agent: Scrapy/VERSION (+http://scrapy.org) Disallow: / User-agent: Scrapy/0.24.4 Disallow: / User-agent: scrapybot Disallow: / User-agent: scribdbot Disallow: / User-agent: Scrubby Disallow: / User-agent: searchme Disallow: / User-agent: SearchmetricsBot Disallow: / User-agent: searchpreview Disallow: / User-agent: seekbot Disallow: / User-agent: SemrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: Sensis Web Crawler Disallow: / User-agent: Sensis Web Crawler (search_comments\\at\\sensis\\dot\\com\\dot\\au) Disallow: / User-agent: Sensis+Web+Crawler+ Disallow: / User-agent: seokicks Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: seokicks-robot Disallow: / User-agent: Servage Disallow: / User-agent: ServageRobot Disallow: / User-agent: setooz Disallow: / User-agent: seznambot Disallow: / User-agent: SeznamBot Disallow: / User-agent: ShopWiki Disallow: / User-agent: ShowyouBot Disallow: / User-agent: SheerBoredom.Experimental.Robot Disallow: / User-agent: SheerBoredom.Experimental.Robot/0.2 Disallow: / User-agent: Sidewinder Disallow: / User-agent: Sika Disallow: / User-agent: SiteBot Disallow: / User-agent: sitebot Disallow: / User-agent: SiteBot/0.1 Disallow: / User-agent: sitecat Disallow: / User-agent: SiteExplorer Disallow: / User-agent: SiteExplorer/1.0b Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: siteexplorer.info Disallow: / User-agent: Sitemap: http://www.jaxcarpentry.com/sitemap.xml Disallow: / User-agent: SiteSnagger Disallow: / User-agent: SkimBot Disallow: / User-agent: SkimBot/1.0 Disallow: / User-agent: Slack-ImgProxy Disallow: / User-agent: Slicehost Disallow: / User-agent: Slurp.so/1.0 Disallow: / User-agent: slurp@inktomi Disallow: / User-agent: slurp@inktomi.com Disallow: / User-agent: Slurp/2.0-KiteHourly Disallow: / User-agent: Slurp/2.0-OwlWeekly Disallow: / User-agent: Slurp/2.0j Disallow: / User-agent: Slurp/3.0-AU Disallow: / User-agent: SlySearch Disallow: / User-agent: SlySearch/1.0 Disallow: / User-agent: SlySearch/1.x Disallow: / User-agent: SMTBot Disallow: / User-agent: SMTBot/1.0 Disallow: / User-agent: Snapbot Disallow: / User-agent: Snapbot/1.0 Disallow: / User-agent: SnapPreviewBot Disallow: / User-agent: Snappy Disallow: / User-agent: Snappybot Disallow: / User-agent: snk Disallow: / User-agent: Snoopy Disallow: / User-agent: sogou Disallow: / User-agent: Sogou Pic Spider Disallow: / User-agent: Sogou Pic Spider/3.0 Disallow: / User-agent: sogou spider Disallow: / User-agent: Sogou spider Disallow: / User-agent: Sogou web spider Disallow: / User-agent: sogou+spider Disallow: / User-agent: Sonic Disallow: / User-agent: Sosospider Disallow: / User-agent: SpankBot Disallow: / User-agent: spanner Disallow: / User-agent: spbot Disallow: / User-agent: spbot/4.4.2 Disallow: / User-agent: Speedy Disallow: / User-agent: speedy Disallow: / User-agent: speedyspider Disallow: / User-agent: sphsearch Disallow: / User-agent: sproose Disallow: / User-agent: Steeler Disallow: / User-agent: Steeler/3.5 Disallow: / User-agent: StudioFACA Disallow: / User-agent: StudioFACA Search Disallow: / User-agent: sukibot Disallow: / User-agent: sukibot_heritrix Disallow: / User-agent: sukibot_heritrix/3.1.1 Disallow: / User-agent: summify Disallow: / User-agent: SuperBot/4.4.0.60 (Windows XP) Disallow: / User-agent: SuperPages Disallow: / User-agent: SurveyBot Disallow: / User-agent: SurveyBot/2.3 Disallow: / User-agent: SurveyBot/2.3 (Whois Source) Disallow: / User-agent: suzuran Disallow: / User-agent: Synthesio Crawler release MonaLisa Disallow: / User-agent: Sysomos Disallow: / User-agent: Szukacz/1.4 Disallow: / User-agent: T-Online Browser Disallow: / User-agent: T-Rex Disallow: / User-agent: tagoobot Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: TeleportProMIIxpc Disallow: / User-agent: Telesoft Disallow: / User-agent: teoma Disallow: / User-agent: Teoma Disallow: / User-agent: The Intraformant Disallow: / User-agent: TheNomad Disallow: / User-agent: therarestparser Disallow: / User-agent: TightTwatBot Disallow: / User-agent: Titan Disallow: / User-agent: toCrawl/UrlDispatcher Disallow: / User-agent: toplistbot Disallow: / User-agent: Trident Disallow: / User-agent: Trove Disallow: / User-agent: trovitBot Disallow: / User-agent: True_Robot Disallow: / User-agent: True_Robot /1.0turingos Disallow: / User-agent: True_Robot/1.0 Disallow: / User-agent: TrueBot Disallow: / User-agent: TrueBot/1.0 Disallow: / User-agent: turingos Disallow: / User-agent: turnitinbot Disallow: / User-agent: TurnitinBot Disallow: / User-agent: TurnitinBot/1.5 Disallow: / User-agent: TurnitinBot/2.0 http://www.turnitin.com/robot/crawlerinfo.html Disallow: / User-agent: Tutorial Crawler Disallow: / User-agent: twengabot Disallow: / User-agent: twiceler Disallow: / User-agent: Twiceler Disallow: / User-agent: Twiceler-0.9 Disallow: / User-agent: Twiceler-0.9+http://www.cuill.com/twiceler/robot.html Disallow: / User-agent: Typhoeus Disallow: / User-agent: uberbot Disallow: / User-agent: uipbot/1.0 Disallow: / User-agent: UltraSeek Disallow: / User-agent: uMBot-LN Disallow: / User-agent: uMBot-LN/1.0 Disallow: / User-agent: UnisterBot Disallow: / User-agent: updown_tester Disallow: / User-agent: URI::Fetch Disallow: / User-agent: URL Control Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: URLChecker Disallow: / User-agent: urlappendbot Disallow: / User-agent: urllib Disallow: / User-agent: URLy Warning Disallow: / User-agent: UsineNouvelleCrawler Disallow: / User-agent: uw_cse_xwc Disallow: / User-agent: vBSEO Disallow: / User-agent: VeriCiteCrawler Disallow: / User-agent: VeriCiteCrawler/Nutch-1.9 Disallow: / User-agent: VCI Disallow: / User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: / User-agent: Voila Robot Disallow: / User-agent: VCIVCI Disallow: / User-agent: ViolaBot Disallow: / User-agent: voilabot Disallow: / User-agent: voltron Disallow: / User-agent: Vorboss Web Crawler Disallow: / User-agent: Vorboss Web Crawler/Nutch-2.3 Disallow: / User-agent: Voyager Disallow: / User-agent: voyager Disallow: / User-agent: voyager/1.0 Disallow: / User-agent: voyager/2.0 Disallow: / User-agent: vscooter Disallow: / User-agent: VWBot Disallow: / User-agent: WBSearchBot Disallow: / User-agent: wbsearchbot Disallow: / User-agent: Web Downloader/4.5 Disallow: / User-agent: Web Downloader/5.7 Disallow: / User-agent: Web Downloader/6.9 Disallow: / User-agent: Web Image Collector Disallow: / User-agent: webalta Disallow: / User-agent: WebCookies Disallow: / User-agent: WebCookies/1.0 Disallow: / User-agent: WebarooBot Disallow: / User-agent: WebAuto Disallow: / User-agent: WebBandit Disallow: / User-agent: WebBandit/3.50 Disallow: / User-agent: Webcollage Disallow: / User-agent: webcompanycrawler Disallow: / User-agent: WebCopier Disallow: / User-agent: WebCopier v3.2a Disallow: / User-agent: WebCopier v3.3 Disallow: / User-agent: WebCopier v3.3.0 Disallow: / User-agent: WebCopier v3.3.2 Disallow: / User-agent: WebCopier vx.xa Disallow: / User-agent: Web corpus crawler Disallow: / User-agent: webcrawler Disallow: / User-agent: WebCrawler Disallow: / User-agent: Web-sniffer Disallow: / User-agent: Web-sniffer/1.1.0 Disallow: / User-agent: WebDataCentreBot Disallow: / User-agent: WebEnhancer Disallow: / User-agent: WebmasterWorld Extractor Disallow: / User-agent: WebmasterWorldForumBot Disallow: / User-agent: webmon Disallow: / User-agent: WebSauger Disallow: / User-agent: Websense Disallow: / User-agent: Website Quester Disallow: / User-agent: Webspider Disallow: / User-agent: Webster Pro Disallow: / User-agent: WebStripper Disallow: / User-agent: WebViewer Disallow: / User-agent: WebZip Disallow: / User-agent: WebZIP/3.5 (http://www.spidersoft.com) Disallow: / User-agent: WebZip/4.0 Disallow: / User-agent: Wegtam Disallow: / User-agent: Wegtam Crawler Disallow: / User-agent: Wells Search II Disallow: / User-agent: WEP Search Disallow: / User-agent: WeSEE:Search Disallow: / User-agent: wget Disallow: / User-agent: WGet Disallow: / User-agent: Wget Disallow: / User-agent: Wget/1.10.2 Disallow: / User-agent: Wget/1.5.3 Disallow: / User-agent: Wget/1.6 Disallow: / User-agent: Wget/1.9.1 Disallow: / User-agent: Wget/1.9+cvs-stable (Red Hat modified) Disallow: / User-agent: WhatWeb Disallow: / User-agent: WhatWeb/0.4.8-dev Disallow: / User-agent: WillyBot Disallow: / User-agent: Win32 Disallow: / User-agent: wocbot Disallow: / User-agent: woobot Disallow: / User-agent: worio Disallow: / User-agent: woriobot Disallow: / User-agent: WorldBrewBot Disallow: / User-agent: WorldBrewBot/2.1 Disallow: / User-agent: wotbox Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: www.deadlinkchecker.com Disallow: / User-agent: Xenu Link Sleuth Disallow: / User-agent: Xenu Link Sleuth 1.2i Disallow: / User-agent: Xenu's Disallow: / User-agent: Xenu's Link Sleuth 1.1c Disallow: / User-agent: XoviBot Disallow: / User-agent: XoviBot/2.0 Disallow: / User-agent: Y!J Disallow: / User-agent: y!j-bsc Disallow: / User-agent: Y!J-BSC Disallow: / User-agent: Y!J-BSC/1.0 Disallow: / User-agent: y!j-bsc/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html) Disallow: / User-agent: y!j/1.0 Disallow: / User-agent: yacybot Disallow: / User-agent: yanga Disallow: / User-agent: Yanga Disallow: / User-agent: Yasni Disallow: / User-agent: yeti Disallow: / User-agent: Yeti Disallow: / User-agent: Yetibot Disallow: / User-agent: yoogliFetchAgent/0.1 Disallow: / User-agent: YisouSpider Disallow: / User-agent: Zao Disallow: / User-agent: zermelo Disallow: / User-agent: zeus Disallow: / User-agent: Zeus Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: Zeus Link Scout Disallow: / User-agent: zibber Disallow: / User-agent: zibber v0.1 Disallow: / User-agent: zibber-v0.1(www.zibb.com/crawler/) Disallow: / User-agent: ZupeeCrawler Disallow: / User-agent: zyborg Disallow: / User-agent: ZyBorg/1.0 Disallow: / User-agent: zzabmbot Disallow: / User-agent: zzabmbot/1.0 Disallow: / User-Agent: Amazon CloudFront Disallow: / Allow: /*.js Allow: /*.css Allow: /*.png Allow: /*.jpg Allow: /*.jpeg Allow: /*.gif Allow: /*.ttf Allow: /*.ico Allow: /*.pdf