# for non httpS|aalborg.com|aalborginstruments
# Disallows removed to let crawls redirect to proper domains and be Disallowed there

User-agent: *
Crawl-Delay: 3
# Allow: */sort/20a                                       # default sort
# Allow: */sort/*/zenid
# Allow: */sort/*/PHPSESSID
# Allow: */sort/*/phpsessid
# Allow: */sort/*/language/
# Allow: */language/*/sort/
# Allow: */change_resolution/*/sort/
# Disallow: */sort/
# Disallow: */sort_order/                                 # cal lib
# Disallow: *main_page/popup_image/
# Disallow: *main_page/contact_us
# Disallow: *main_page/login
# Disallow: *main_page/cookie_usage
# Disallow: *main_page/unsubscribe
# Disallow: *main_page*autocomplete
# Disallow: *main_page/popup_search_help
# Disallow: /captcha_img.php
# Allow:    *main_page=configurator&conf_id=6*&action=selected&sku=L
# Allow:    *main_page=configurator&conf_id=12&action=selected&sku=PSV
# Allow:    *main_page=configurator&conf_id=13&action=selected&sku=SMV
# Allow:    *main_page=configurator&conf_id=14&action=selected&sku=V
# Allow:    *main_page=configurator&conf_id=15&action=selected&sku=V
# Allow:    *main_page=configurator&conf_id=17&action=selected&sku=F
# Allow:    *main_page=configurator&conf_id=21&action=selected&sku=SDPROC
# Allow:    *main_page=configurator&conf_id=25&action=selected&sku=TIO
# Allow:    *main_page=configurator&conf_id=26&action=selected&sku=PWE
# Disallow: *main_page=configurator&conf_id=**&action=selected&sku=
# Disallow: *main_page/configurator*/sku/*
# Allow:    *main_page/configurator_search_tubes_configure/*/zenid
# Disallow: *main_page*configurator_search_tubes_configure*
# Allow:    *main_page/configurator_tubes_selection/*/zenid
# Disallow: *main_page*configurator_tubes_selection*
# Disallow: *main_page/popup_configurator_what_this
# Disallow: /_*/*

# Sep-2020
# User-agent: AhrefsSiteAudit
# Allow: */sort/20a                                       # default sort
# Disallow: */sort/
# Disallow: */sort_order/                                 # cal lib
# Disallow: *main_page/popup_image/
# Disallow: *main_page/contact_us
# Disallow: *main_page/login
# Disallow: *main_page/cookie_usage
# ##      Disallow: *main_page/configurator/sku/*/action/recover/
# Disallow: *main_page/configurator_search_tubes_configure/
# Disallow: *main_page/configurator_tubes_selection/

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot          # moz
User-agent: MegaIndex
Crawl-Delay: 5
# Disallow: /*http://www.adobe.com/products/acrobat/readstep2.html/*

User-agent: PetalBot
User-agent: Exabot
User-agent: test-bot
Crawl-Delay: 10

User-agent: Seekport
Crawl-Delay: 5
# Disallow: */cPath/
# Disallow: */products_id/
# Disallow: */configurator/sku/
# Disallow: */configurator_*
# Disallow: */calibrator_flowtubes/*