# Enables robots.txt rules for all crawlers
User-agent: *
Sitemap: https://www.dynamo-dresden.de/typo3temp/sitemapindex.xml

Disallow: /forum/?*
Disallow: /forum/members/*

# # Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files
Disallow: /*.git$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*.tar.gz$
Disallow: /*.zip$
Disallow: /session.php

# # GENERAL SETTINGS For MAGENTO
Disallow: /shop/admin/

# # Do not index the general technical Magento directory
Disallow: /shop/app/
Disallow: /shop/downloader/
Disallow: /shop/errors/
Disallow: /shop/includes/
Disallow: /shop/lib/
Disallow: /shop/pkginfo/
Disallow: /shop/shell/
Disallow: /shop/skin/
Disallow: /shop/var/

# # Do not index the shared files Magento
Disallow: /shop/api.php
Disallow: /shop/cron.php
Disallow: /shop/cron.sh
Disallow: /shop/error_log
Disallow: /shop/get.php
Disallow: /shop/install.php
Disallow: /shop/LICENSE.html
Disallow: /shop/LICENSE.txt
Disallow: /shop/LICENSE_AFL.txt
Disallow: /shop/README.txt
Disallow: /shop/RELEASE_NOTES.txt
Disallow: /shop/STATUS.txt


# # MAGENTO SEA IMPROVEMENT

# # Do not index the page subcategories that are sorted or filtered.
Disallow: /shop/*?Dir*
Disallow: /shop/*?Dir=desc
Disallow: /shop/*?Dir=asc
Disallow: /shop/*?Limit=all
Disallow: /shop/*?Mode*

# # Do not index the second copy of the home page (example.com / index.php /). Un-comment only if you have activated Magento SEO URLs.
Disallow: /shop/index.php/

# # Do not index the link from the session ID
Disallow: /shop/*?SID=

# # Do not index the page checkout and user account
Disallow: /shop/checkout/
Disallow: /shop/catalog/product_compare/
Disallow: /shop/onestepcheckout/
Disallow: /shop/customer/
Disallow: /shop/customer/account/
Disallow: /shop/customer/account/login/
Disallow: /shop/newsletter/
Disallow: /shop/poll/
Disallow: /shop/review/
Disallow: /shop/sendfriend/
Disallow: /shop/wishlist/

# # Do not index the search page and CEO, non-optimized link categories
Disallow: /shop/catalogsearch/
Disallow: /shop/catalog/category/view/
Disallow: /shop/catalog/product/view/
Disallow: /shop/catalog/product/gallery/

# # Exclude Various Robots
User-agent: MJ12bot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: FatBot
Disallow: /

User-agent: SputnikBot
Disallow: /

User-agent: trendictionbot0.5.0
Disallow: /

User-agent: Owlin
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: Stashbot
Disallow: /

User-agent: MetaJobBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: linkdexbot
Disallow: /