# /robots.txt file for http://webcrawler.com/ # mail webmaster@webcrawler.com for constructive criticism User-agent: * Disallow: /cgi-bin/ Disallow: /conditions/ Disallow: /css/ Disallow: /images/ Disallow: /img/ Disallow: /includes/ Disallow: /lang/ Disallow: /pages/ Disallow: /picture_library/ Disallow: /plesk-stat/ Disallow: /promo/ Disallow: /style/ Disallow: /test/ Disallow: /testshop/ Disallow: /tpl/ Disallow: /voorwaarden/ Disallow: /webshop/ Disallow: phpinfo.php