# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed

# Custom
Disallow: /*node
Disallow: /*.pdf
Disallow: /*.PDF
Disallow: /*.doc
Disallow: /*.DOC
Disallow: /*tekzennew
Disallow: /*index1.php
Disallow: /forward/
Disallow: /forward/*
Disallow: /print/
Disallow: /print/*
Disallow: /*checkout
Disallow: /cart
Disallow: /*categories
Disallow: /*search_api_views_fulltext
Disallow: /*page=
Disallow: /*facet=
Disallow: /*value=
Disallow: /*division=
Disallow: /*filter=
Disallow: /*reload=
Disallow: /*mkt_tok=
Disallow: /*destination=
Disallow: /*aspx
Disallow: /*ajax
Disallow: /*leaflet
Disallow: /*superfish
Disallow: /*mvc
Disallow: /*sendfriend
Disallow: /*wishlist
Disallow: /*product_compare
Disallow: /*frontend
Disallow: /*sensordb-images
Disallow: /*?f%5B
Disallow: /*?*f%5B
Disallow: /*?f[
Disallow: /*?*f[
Disallow: /applications?
Disallow: /resources?

# Disallow the ads bots from Google and MSN from crawling cart or checkout pages
User-agent: AdsBot-Google
User-agent: adidxbot
Disallow: /cart
Disallow: /checkout
Disallow: /checkout*
Disallow: /*checkout

Disallow: /*?*f[
Disallow: /*?*f%5B
Disallow: /applications?
Disallow: /resources?

User-agent: MJ12bot
user-agent: SeekportBot
user-agent: Linespider
user-agent: Baiduspider
User-agent: Yandex
User-agent: YandexBot
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: sogou spider
Disallow: /

# Meta AI-training crawler: ~80k faceted-page requests/day during Jul 2026 flood
User-agent: meta-externalagent
Disallow: /