# # robots.txt for http://www.kolzchut.org.il # # Start by allowing all bots User-agent: * Allow: / # Then prevent *some* advertising-related bots: #Google AdSense bot User-agent: Mediapartners-Google Disallow: / #Google image crawler (for image search engine) User-agent: Googlebot-Image Disallow: / ### no access to editing links and the like ### # The following are disallowed by a robots meta tag, # but going by the robots meta tag is not good enough # because the bot traffic still hits the server User-agent: * Disallow: /w/*/index.php Disallow: /w/*/api.php # but allow access to recentchanges User-agent: * Allow: /w/he/index.php?title=%D7%9E%D7%99%D7%95%D7%97%D7%93:%D7%A9%D7%99%D7%A0%D7%95%D7%99%D7%99%D7%9D_%D7%90%D7%97%D7%A8%D7%95%D7%A0%D7%99%D7%9D&feed=atom # Leave our internal directories alone User-agent: * Disallow: /w/*/(includes|extensions|skins) # no point in a bot accessing the forms User-agent: * Disallow: /ChangeRequest/ Disallow: /forms/ # dynamic pages are disallowed using the robots meta tag inside the wiki. # But we still disallow specific pages, because some bastards hit them anyway... # I'm looking at you, bing User-agent: * Disallow: /he/%D7%9E%D7%99%D7%95%D7%97%D7%93:Drilldown