Sitemap: http://www.theukhighstreet.com/google_index.xml

User-agent: *
Crawl-delay: 3
Disallow: jump.cgi
Disallow: pjump.cgi
Disallow: like.cgi
Disallow: /*_J\d+$
Disallow: /*_PJ\d+$
Disallow: /perl/bad_link.cgi
Disallow: /perl/adverts.pl
Disallow: /error/
Disallow: /cgi-bin/
Disallow: /pics/
Disallow: /images/
Disallow: /perl/sitestats.cgi
Disallow: /count.js
Disallow: /perl/stats/
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: ShopWiki
Disallow: /

User-agent: spbot
Disallow: /

User-agent: MJ12bot
Disallow: / 

User-agent: Sosospider 
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: pompos
Disallow: /
#User-agent: ia_archiver
#Disallow: /

User-agent: niXXiebot
Disallow: /

User-agent: Googlebot
Disallow: /rss/
Disallow: /perl/page.cgi?g=
Disallow: jump.cgi
Disallow: pjump.cgi
Disallow: like.cgi
Disallow: /*_J\d+$
Disallow: /*_PJ\d+$
Disallow: /perl/bad_link.cgi
Disallow: /perl/adverts.pl
Disallow: /error/
Disallow: /cgi-bin/
Disallow: /pics/
Disallow: /images/
Disallow: /perl/sitestats.cgi
Disallow: /count.js
Disallow: /perl/stats/
Disallow: /*.gif$
Disallow: /*.jpg$