Sitemap: http://www.file-extensions.org/sitemap.xml

User-agent: *
Disallow: /permalink
Disallow: /search/
Disallow: /channel.html
Crawl-delay: 1

# AS START
User-agent: 360Spider
User-agent: 80legs.com
User-agent: AhrefsBot
User-agent: businessdbbot
User-agent: CCBot
User-agent: CovarioIDS
User-agent: curl/
User-agent: discobot
User-agent: EasouSpider
User-agent: EMail Exractor
User-agent: Ezooms
User-agent: FDM 3.x
User-agent: FlaxCrawler
User-agent: Grabber
User-agent: Grapeshot
User-agent: GSLFbot
User-agent: heritrix
User-agent: HTTrack
User-agent: intelium_bot
User-agent: Java/
User-agent: larbin
User-agent: lemurwebcrawler
User-agent: libwww-perl
User-agent: LinksCrawler
User-agent: MetamojiCrawler
User-agent: MJ12bot
User-agent: Nutch
User-agent: OpenAcoon
User-agent: PHP/
User-agent: Plukkie
User-agent: proximic
User-agent: Python-urllib
User-agent: Redvertigo-Project
User-agent: rogerbot
User-agent: Ruby
User-agent: SemrushBot
User-agent: SISTRIX
User-agent: SiteTruth.com
User-agent: spbot
User-agent: WBSearchBot
User-agent: WeblexBot
User-agent: WIRE/0.
Disallow: /
# AS END

User-agent: Mediapartners-Google
Disallow: