# ===============================
# General Bot Access Rules
# ===============================

User-agent: *

# Directories to block
Disallow: /admin/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /directory/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pub/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Clean URLs (Magento system paths)
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /contacts/
Disallow: /control/
Disallow: /customize/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/
Disallow: /index.php/
Disallow: /newsletter/
Disallow: /rewards/account/
Disallow: /rewards_catalog/
Disallow: /stores/store/redirect/
Disallow: /wishlist/

# Technical files
Disallow: /cgi-bin/
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
Disallow: /COPYING.txt
Disallow: /Gruntfile.js
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt

# Block .php URLs and session IDs
Disallow: /*.php$
Disallow: /*?p=*&
Disallow: /*?SID=

# ===============================
# Filter Parameters to Block
# ===============================

Disallow: /*?brand
Disallow: /*&brand
Disallow: /*?cafeine
Disallow: /*&cafeine
Disallow: /*?cat
Disallow: /*&cat
Disallow: /*?club_country
Disallow: /*&club_country
Disallow: /*?coffee_machine_users
Disallow: /*&coffee_machine_users
Disallow: /*?coffee_strength
Disallow: /*&coffee_strength
Disallow: /*?cup_per_hour
Disallow: /*&cup_per_hour
Disallow: /*?extension_type
Disallow: /*&extension_type
Disallow: /*?fit_size
Disallow: /*&fit_size
Disallow: /*?inhoud
Disallow: /*&inhoud
Disallow: /*?koffie_branding
Disallow: /*&koffie_branding
Disallow: /*?koffie_geur_smaak
Disallow: /*&koffie_geur_smaak
Disallow: /*?koffie_suitable_for
Disallow: /*&koffie_suitable_for
Disallow: /*?koffie_zetwijze
Disallow: /*&koffie_zetwijze
Disallow: /*?material
Disallow: /*&material
Disallow: /*?player_names
Disallow: /*&player_names
Disallow: /*?price
Disallow: /*&price
Disallow: /*?producttype
Disallow: /*&producttype
Disallow: /*?quality_mark
Disallow: /*&quality_mark
Disallow: /*?robusta_arabica
Disallow: /*&robusta_arabica
Disallow: /*?season
Disallow: /*&season
Disallow: /*?size
Disallow: /*&size
Disallow: /*?sub_product_type
Disallow: /*&sub_product_type
Disallow: /*?tea_varieties
Disallow: /*&tea_varieties

# ===============================
# Tracking & Sorting Parameters
# ===============================

Disallow: /*?tt
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?fbclid=
Disallow: /*&fbclid=

# Disable sorting options (avoid duplicate content)
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=

# ===============================
# Specific Bot Rules
# ===============================

User-agent: Baiduspider
Disallow: /

User-agent: CazoodleBot
Disallow: /

User-agent: Fasterfox
Disallow: /

User-agent: teracent-feed-processing
Disallow: /

User-agent: Jyxobot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ShopWiki
Disallow: /

User-agent: Ingrid
Crawl-delay: 20

User-agent: msnbot
Crawl-delay: 20

User-agent: Slurp
Crawl-delay: 20

User-agent: rogerbot
Crawl-delay: 2

# ===============================
# Sitemap
# ===============================
Sitemap: https://cafedujour.nl/sitemap.xml