# =============================================================================
#  Molport — robots.txt
#  Updated: 2026-05-11 (post May 2026 indexation crisis)
#
#  Policy: STRICT allow-list of Google + Bing + ChatGPT only.
#  Every other crawler is blocked (matches IPAddressHelper.java in the WAR).
#
#  Goals:
#   * Block query-parameter URL variants Google was indexing as duplicates
#     (?pageindex, ?utm_*, ?searchtype, ?searchkey, ?root-formula, ?sort,
#     ?view) — confirmed in GSC Performance / Table CSVs.
#   * Block transactional / private paths (orders, cart, checkout, account,
#     backoffice, password flows, internal AJAX endpoints).
#   * Block every crawler outside the three approved families.
#   * Declare ONE sitemap entry (sitemap-index.xml).
#
#  Out-of-policy bots: ClaudeBot, PerplexityBot, Applebot, YandexBot,
#  DuckDuckBot, Baiduspider, Sogou, CCBot, Cohere, Diffbot, Meta, plus all
#  social-preview agents (Facebook, LinkedIn, Twitter) and every SEO scraper.
#  These are all blocked here AND at the Java filter level — see
#  IPAddressHelper.isBlockedBot().
# =============================================================================

# -----------------------------------------------------------------------------
# Googlebot — full crawl, no Crawl-delay.
# -----------------------------------------------------------------------------
User-agent: Googlebot
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/user/
Disallow: /shop/backoffice/
Disallow: /shop/signin
Disallow: /shop/signup
Disallow: /shop/reset-password
Disallow: /shop/quote-request/
Disallow: /shop/list-search/
Disallow: /shop/load-search
Disallow: /shop/save-search/
Disallow: /shop/delete-search/
Disallow: /shop/edit-structure
Disallow: /shop/shopping-cart
Disallow: /shop/add-to-cart-request
Disallow: /shop/add-to-cart-request/
Disallow: /shop/submit-add-to-cart-request
Disallow: /shop/submit-add-to-cart-request/
Disallow: /shop/update-cart-request
Disallow: /shop/update-cart-request/
Disallow: /shop/price-request
Disallow: /shop/price-request/
Disallow: /shop/submit-price-request
Disallow: /shop/submit-price-request/
Disallow: /shop/public-proposal-for-client
Disallow: /shop/public-proposal-for-client/
Disallow: /shop/directory-molecule-image
Disallow: /shop/directory-molecule-image/
Disallow: /shop/directory-molecule-image-box
Disallow: /shop/directory-molecule-image-box/
Disallow: /shop/find-fragments
Disallow: /shop/find-fragments/
Disallow: /shop/find-similar
Disallow: /shop/find-similar/
Disallow: /shop/compounds-similar-to-this-molecule
Disallow: /shop/molecule-image-with-hit-coloring
Disallow: /shop/molecule-image-with-hit-coloring/
Disallow: /shop/molecular-weight
Disallow: /shop/molecular-weight/
Disallow: /shop/moleculepreviewbox
Disallow: /shop/moleculepreviewbox/
# Parameter-variant duplicates:
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*utm_term=
Disallow: /*?*utm_content=
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*pageindex=
Disallow: /*?*searchtype=
Disallow: /*?*searchkey=
Disallow: /*?*root-formula=
Disallow: /*?*sort=
Disallow: /*?*view=

User-agent: Googlebot-Image
Allow: /shop/molecule-image
Allow: /shop/frontend/
Allow: /shop/img/
Disallow: /shop/backoffice/

User-agent: Googlebot-News
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/

User-agent: Google-Extended
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/backoffice/
Disallow: /shop/user/

User-agent: AdsBot-Google
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/backoffice/

# -----------------------------------------------------------------------------
# Bingbot family
# -----------------------------------------------------------------------------
User-agent: Bingbot
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/user/
Disallow: /shop/backoffice/
Disallow: /shop/signin
Disallow: /shop/signup
Disallow: /shop/reset-password
Disallow: /shop/quote-request/
Disallow: /shop/list-search/
Disallow: /shop/edit-structure
Disallow: /shop/shopping-cart
Disallow: /shop/add-to-cart-request
Disallow: /shop/add-to-cart-request/
Disallow: /shop/price-request
Disallow: /shop/price-request/
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*pageindex=
Disallow: /*?*searchtype=
Disallow: /*?*searchkey=
Disallow: /*?*root-formula=
Disallow: /*?*sort=
Disallow: /*?*view=

User-agent: BingPreview
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/backoffice/

User-agent: msnbot
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/backoffice/

# -----------------------------------------------------------------------------
# ChatGPT / OpenAI family
# -----------------------------------------------------------------------------
User-agent: GPTBot
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/user/
Disallow: /shop/backoffice/
Disallow: /shop/signin
Disallow: /shop/signup
Disallow: /shop/reset-password
Disallow: /shop/quote-request/
Disallow: /shop/list-search/

User-agent: OAI-SearchBot
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/backoffice/

User-agent: ChatGPT-User
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/backoffice/

# -----------------------------------------------------------------------------
# Everything else: explicitly denied.
#
# These bots are NOT in our allow-list per business decision (2026-05-11).
# They are also blocked at the Java filter level (IPAddressHelper.java).
# Both layers needed: robots.txt politely asks them not to crawl; the Java
# filter enforces it for those that ignore robots.txt.
# -----------------------------------------------------------------------------

# Anthropic
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: anthropic-ai
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /
User-agent: Perplexity-User
Disallow: /

# Apple
User-agent: Applebot
Disallow: /
User-agent: Applebot-Extended
Disallow: /

# Other search
User-agent: YandexBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: DuckDuckBot
Disallow: /

# Other AI
User-agent: cohere-ai
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /

# SEO scrapers
User-agent: SemrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: AwarioRssBot
Disallow: /
User-agent: AwarioSmartBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: SerpstatBot
Disallow: /

# -----------------------------------------------------------------------------
# Default rules — same restrictive policy.
# Anything not in the allow-list above lands here and gets blocked from the
# transactional paths. The Java filter handles the rest.
# -----------------------------------------------------------------------------
User-agent: *
Disallow: /shop/order/
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /shop/my-account/
Disallow: /shop/user/
Disallow: /shop/backoffice/
Disallow: /shop/signin
Disallow: /shop/signup
Disallow: /shop/reset-password
Disallow: /shop/quote-request/
Disallow: /shop/list-search/
Disallow: /shop/load-search
Disallow: /shop/save-search/
Disallow: /shop/delete-search/
Disallow: /shop/edit-structure
Disallow: /shop/shopping-cart
Disallow: /shop/add-to-cart-request
Disallow: /shop/add-to-cart-request/
Disallow: /shop/submit-add-to-cart-request
Disallow: /shop/submit-add-to-cart-request/
Disallow: /shop/update-cart-request
Disallow: /shop/update-cart-request/
Disallow: /shop/price-request
Disallow: /shop/price-request/
Disallow: /shop/submit-price-request
Disallow: /shop/submit-price-request/
Disallow: /shop/public-proposal-for-client
Disallow: /shop/public-proposal-for-client/
Disallow: /shop/directory-molecule-image
Disallow: /shop/directory-molecule-image/
Disallow: /shop/directory-molecule-image-box
Disallow: /shop/directory-molecule-image-box/
Disallow: /shop/find-fragments
Disallow: /shop/find-fragments/
Disallow: /shop/find-similar
Disallow: /shop/find-similar/
Disallow: /shop/compounds-similar-to-this-molecule
Disallow: /shop/molecule-image-with-hit-coloring
Disallow: /shop/molecule-image-with-hit-coloring/
Disallow: /shop/molecular-weight
Disallow: /shop/molecular-weight/
Disallow: /shop/moleculepreviewbox
Disallow: /shop/moleculepreviewbox/
Disallow: /denied.html
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*pageindex=
Disallow: /*?*searchtype=
Disallow: /*?*searchkey=
Disallow: /*?*root-formula=
Disallow: /*?*sort=
Disallow: /*?*view=

# -----------------------------------------------------------------------------
# Sitemaps — all entry points (robots.txt may declare multiple).
#   * sitemap_index.xml  — GENERATED index (underscore) → molecule packs
#                          sitemap1.xml.gz … sitemapN.xml.gz  (this is the real
#                          file the Sitemap Generator writes at host root)
#   * sitemap-index.xml   — legacy hyphen alias (kept in case prod serves it;
#                           a non-existent sitemap entry is simply ignored)
#   * sitemap-static.xml  — hand-curated high-value landing pages
#   * sitemap-images.xml  — image sitemap
# These MUST be reachable at host root (https://www.molport.com/<file>), not
# under /shop/. See deployment notes in Documentation/SEO.
# -----------------------------------------------------------------------------
Sitemap: https://www.molport.com/sitemap_index.xml
Sitemap: https://www.molport.com/sitemap-index.xml
Sitemap: https://www.molport.com/sitemap-static.xml
Sitemap: https://www.molport.com/sitemap-images.xml