# Crawlers SetupF
User-agent: *

# Allowable Index
# Mind that Allow is not an official standard
Allow: /*?p=
Allow: /media/catalog/
Allow: /media/wysiwyg/

# Directories
Disallow: */404/
Disallow: */app/
Disallow: */cgi-bin/
Disallow: */downloader/
Disallow: */errors/
Disallow: */includes/
Disallow: */lib/
Disallow: */magento/

Disallow: */media/captcha/
Disallow: */media/css_secure/
Disallow: */media/customer/
Disallow: */media/dhl/
Disallow: */media/downloadable/
Disallow: */media/import/
Disallow: */media/pdf/
Disallow: */media/sales/
Disallow: */media/tmp/
Disallow: */media/xmlconnect/
Disallow: */pkginfo/
Disallow: */report/
Disallow: */scripts/
Disallow: */shell/
Disallow: */stats/
Disallow: */var/

# Paths (clean URLs)
Disallow: */*index.php*
Disallow: */catalog/product_compare/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */catalog/product/gallery/
Disallow: */catalogsearch/
Disallow: */checkout/
Disallow: */control/
Disallow: */contacts/
Disallow: */customer/
Disallow: */customize/
Disallow: */newsletter/
Disallow: */poll/
Disallow: */review/
Disallow: */sendfriend/
Disallow: */tag/
Disallow: */wishlist/
Disallow: */*virtuemart*
Disallow: */*?pop=0*
Disallow: */*flypage.tpl*
Disallow: */*raamfolie*
Disallow: */*list-all-products*
Disallow: */*com_gk3*
Disallow: */*author*
Disallow: */*awcountdown*
# Efex
Disallow: */advancedsearch/
Disallow: */addtocart/
Disallow: */cartquickpro/
Disallow: */productalert/
Disallow: */catalogsearch/advanced/
Disallow: */search/
Disallow: */sales/guest/form/
Disallow: */my-account/
Disallow: */vendor/
Disallow: */update/
Disallow: */setup/
Disallow: */phpserver/
Disallow: */dev/
Disallow: */bin/
Disallow: */page_cache/block/*
Disallow: */stores/store/redirect/*
Disallow: */stores/store/switch/*
Disallow: /cdn-cgi/l/email-protection


# Pages
Disallow: */privacy-policy-cookie-restriction-mode/
Disallow: */terms/

# Files
Disallow: */cron.php
Disallow: */cron.sh
Disallow: */error_log
Disallow: */install.php
Disallow: */LICENSE.html
Disallow: */LICENSE.txt
Disallow: */LICENSE_AFL.txt
Disallow: */STATUS.txt
Disallow: */get.php # Magento 1.5+

# filters
Disallow: /*?price=
Disallow: /shopby/

# Paths (no clean URLs)
#Disallow: /*.js$
#Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?p=*&
Disallow: /*?p=*
Disallow: /*?pagenr=*
Disallow: /*?pagenr=*&
Disallow: /*?SID=
Disallow: /?___from_store=
Disallow: /*___from_store=
Disallow: /?dir=
Disallow: /?mode=
Disallow: /?limit=
Disallow: /rss*
Disallow: /*PHPSESSID
Disallow: */sort-by/*
Disallow: */show/*
Disallow: */?gclid=*
Disallow: /*is_ajax
# Efex
Disallow: /*?page=
Disallow: /*?customFilters=
# Disallow: /*.map$
Disallow: /*?pile_type=*
Disallow: /*?size=*
Disallow: /*?cat=*
Disallow: /*?price=*

# Crawler bots
User-agent: oai-searchbot 
Crawl-delay: 5
User-agent: Bingbot
Crawl-delay: 2
User-agent: msnbot
Crawl-delay: 2
User-agent: Yahoo! Slurp
Crawl-delay: 5
User-agent: DuckDuckBot 
Crawl-delay: 2
User-agent: Yandex
Crawl-delay: 10
User-agent: YandexBot 
Crawl-delay: 10
User-agent: BaiDuSpider
Disallow: /
User-agent: BSpider
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: Vagabondo
Disallow: /
User-agent: Exabot
Disallow: /
user-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: Moz
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: ShopWiki
Disallow: /
User-agent: CazoodleBot
Disallow: /
User-agent: Fasterfox
Disallow: /
User-agent: Jyxobot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: AspiegelBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: barkrowler
Disallow: /
User-agent: GeedoProductSearch
Disallow: /

# AI bots
User-agent: GPTBot
Crawl-delay: 5
User-agent: gptbot
Crawl-delay: 5
User-agent: ChatGPT-User
Crawl-delay: 5
User-agent: ClaudeBot
Crawl-delay: 5
User-agent: anthropic-ai
Crawl-delay: 5
User-agent: CCBot
Crawl-delay: 5
User-agent: FacebookBot
Crawl-delay: 5
User-agent: meta-externalagent
Crawl-delay: 5
User-agent: Google-Extended
Crawl-delay: 5
User-agent: GoogleAI
Crawl-delay: 5
User-agent: Gemini
Crawl-delay: 5
User-agent: DeepMind-Explore
Crawl-delay: 5
User-agent: Bytespider
Crawl-delay: 5
User-agent: Amazonbot
Crawl-delay: 5
User-agent: Applebot
Crawl-delay: 5
User-agent: cohere-ai
Crawl-delay: 5
User-agent: YouBot
Crawl-delay: 5
User-agent: AIResearchBot
Crawl-delay: 5
User-agent: SeekrBot
Crawl-delay: 5
User-agent: NeevaBot
Crawl-delay: 5
User-agent: PerplexityBot
Crawl-delay: 5
User-agent: Writer
Crawl-delay: 5


User-agent: Screaming Frog SEO Spider
Allow: /
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.png$
Disallow: /*.bmp$
Disallow: /*.xml$
Disallow: /*.css$
Disallow: /*.js$

# Sitemaps