User-agent: *
Disallow: /dataset/rate/
Disallow: /revision/
Disallow: /dataset/activity/*
Disallow: /dataset/groups/*
Disallow: /dataset/showcases/*
Disallow: /dataset/*/history
Disallow: /dataset/*/resource/*
Disallow: /datastore/*
Disallow: /datarequest/*
Disallow: /user/*
Disallow: /api/
Disallow: /resource/*
Disallow: /*?*
Disallow: /*?
Disallow: /*.n3
Disallow: /*.ttl
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.gz$
Disallow: /*.wmv$
Disallow: /*.cgi$
Disallow: /*.xhtml$
Disallow: /*/dataset/rate/
Disallow: /*/revision/
Disallow: /*/dataset/activity/*
Disallow: /*/dataset/groups/*
Disallow: /*/dataset/showcases/*
Disallow: /*/dataset/*/history
Disallow: /*/dataset/*/resource/*
Disallow: /*/datastore/*
Disallow: /*/datarequest/*
Disallow: /*/user/*
Disallow: /ar/
Disallow: /bg/
Disallow: /ca
Disallow: /cs_CZ/
Disallow: /da_DK/
Disallow: /de/
Disallow: /dv/
Disallow: /el/
Disallow: /en_AU/
Disallow: /en_GB/
Disallow: /es/
Disallow: /es_AR/
Disallow: /fa_IR/
Disallow: /fi/
Disallow: /fr/
Disallow: /he/
Disallow: /hu/
Disallow: /id/
Disallow: /is/
Disallow: /it/
Disallow: /ja/
Disallow: /km/
Disallow: /ko_KR/
Disallow: /lt/
Disallow: /lv/
Disallow: /mn_MN/
Disallow: /my_MM/
Disallow: /ne/
Disallow: /nl/
Disallow: /no/
Disallow: /pl/
Disallow: /pt_BR/
Disallow: /pt_PT/
Disallow: /ro/
Disallow: /ru/
Disallow: /sk/
Disallow: /sl/
Disallow: /sq/
Disallow: /sr/
Disallow: /sr_Latn/
Disallow: /sv/
Disallow: /th/
Disallow: /tl/
Disallow: /tr/
Disallow: /uk_UA/
Disallow: /vi/
Disallow: /zh_CN/
Disallow: /zh_HK/
Disallow: /zh_TW/

User-Agent: *
Crawl-Delay: 15

User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Companybook-Crawler
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: netEstate NE Crawler
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /