# Crawl Setup
User-agent: *
 
# Directories
Disallow: */404/
Disallow: */app/
Disallow: */cgi-bin/
Disallow: */downloader/
Disallow: */includes/
Disallow: */jwplayer/
Disallow: */lib/
Disallow: */magento/
Disallow: */media/tmp/
Disallow: */orders_shipping/
Disallow: */pkginfo/
Disallow: */report/
Disallow: */shell/
Disallow: */scripts/
Disallow: */stats/
Disallow: */var/
 
# Internal Search Engine & filters
Disallow: */search/
Disallow: */?color
 Disallow: */?texture
Disallow: */?name
Disallow: /*SID=
Disallow: */?olfactory
Disallow: */?size_variant
Disallow: */?collection_es
Disallow: */?name
Disallow: */referrer
 
# Paths (clean URLs)
Disallow: */index.php/
Disallow: */catalog/product_compare/
Disallow: */catalog/category/view/
Disallow: */catalog/product/view/
Disallow: */catalogsearch/
Disallow: */catalogsearch/result/
Disallow: */checkout/
Disallow: */control/
Disallow: */contacts/
Disallow: */customer/
Disallow: */newsletter/
Disallow: */poll/
Disallow: */tag/
 
# Files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt
 
#Filter pages
Disallow: *?olfactory
Disallow: *?size_variant
Disallow: *?collection_es