User-agent: * Crawl-delay: 10 Disallow: admin.php Disallow: config Disallow: header Disallow: footer Disallow: pntables Disallow: referer Disallow: /images Disallow: /includes Disallow: /modules/NS- Disallow: /pnadodb Disallow: /user.php Disallow: /admin.php Disallow: /modules.php Disallow: /tools.htm Disallow: /tm6.htm Disallow: /tm5.htm Disallow: /tm4.htm Disallow: /tm3.htm Disallow: /tm2.htm Disallow: /tm1.htm Disallow: /texcorne/prise_all/incitingexi.htm Disallow: /store.php Disallow: /surveillance.htm Disallow: /texcorne/prise_all/incitingexi.htm Disallow: /stone.htm Disallow: /statiy.htm Disallow: /sklad.htm Disallow: /segments.htm Disallow: /roads.htm Disallow: /razors.htm Disallow: /projects.htm Disallow: /products.htm Disallow: /prise_progect.htm Disallow: /prise_all.htm Disallow: /Prise_all.files/editdata.mso Disallow: /prise.htm Disallow: /pila.htm Disallow: /partners.htm Disallow: /Oppg.3.avd/tm6.htm Disallow: /NS-Polls/ Disallow: /news.htm Disallow: /MajorMinority.htm Disallow: /main_price.htm Disallow: /leznyk.htm Disallow: /legal/ Disallow: /interior.htm Disallow: /gr1.htm Disallow: /gr2.htm Disallow: /gr3.htm Disallow: /gr4.htm Disallow: /gr5.htm Disallow: /gr6.htm Disallow: /gr7.htm Disallow: /gr8.htm Disallow: /gr9.htm Disallow: /gr10.htm Disallow: /gr11.htm Disallow: /gr12.htm Disallow: /gm1.htm Disallow: /gm2.htm Disallow: /gm3.htm Disallow: /gm4.htm Disallow: /gm5.htm Disallow: /gm6.htm Disallow: /gm7.htm Disallow: /gm8.htm Disallow: /gm9.htm Disallow: /gi1.htm Disallow: /gi2.htm Disallow: /gi3.htm Disallow: /gi4.htm Disallow: /gi5.htm Disallow: /gi6.htm Disallow: /gi7.htm Disallow: /gi8.htm Disallow: /gg1.htm Disallow: /gg2.htm Disallow: /gg3.htm Disallow: /gg4.htm Disallow: /gg5.htm Disallow: /gg6.htm Disallow: /gb1.htm Disallow: /gb2.htm Disallow: /gb3.htm Disallow: /FFALinks1H/gr2.htm Disallow: /en/ Disallow: /drills.htm Disallow: /da.net.ua Disallow: /awt/tm5/edu.htm Disallow: /bottin/gr12/zgcards.htm Disallow: /circular.htm Disallow: /contacts.htm User-Agent: TurnitinBot/2.0 (http://www.turnitin.com/robot/crawlerinfo.html) Disallow: / User-Agent: ia_archiver Disallow: / User-Agent: gazz/5.0 (gazz@nttr.co.jp) Disallow: / User-Agent: grub-client Disallow: / User-Agent: IBM_Planetwide Disallow: / User-Agent: METASpider Disallow: / User-Agent: Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) METASpider Disallow: / User-Agent: NPBot Disallow: / User-Agent: Slurp Disallow: / User-Agent: appie Disallow: / User-Agent: AnalogX Proxy 4.01 (www.analogx.com) Disallow: / User-Agent: Gigabot/2.0 Disallow: / User-Agent: LG_catalog Disallow: / User-Agent: moget Disallow: / User-Agent: Scooter Disallow: / User-Agent: Scooter/3.3 Disallow: / User-Agent: AVFetch Disallow: / User-Agent: AltaVista Disallow: / User-Agent: Mercator Disallow: / User-Agent: oBot Disallow: / User-Agent: NG/2.0 Disallow: / User-Agent: psbot Disallow: / User-Agent: ConveraCrawler/0.5 Disallow: / User-Agent: IRLbot/1.0 (+http://irl.cs.tamu.edu/crawler) Disallow: / User-Agent: Wget/1.9.1 Disallow: / User-Agent: Java/1.4.2-p6 Disallow: / User-Agent: pipeLiner/0.10 (PipeLine Spider; http://www.pipeline-search.com/webmaster.html) Disallow: / User-Agent: MantraAgent Disallow: / User-Agent: Gigabot Disallow: /