# robots.txt for http://www.becit.dk/ # comments to webmaster@becit.dk User-agent: * Allow: /public/ # This folder can be indexed Allow: /~.html/ # These pages can be indexed Disallow: / # Go away for the rest