#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *

# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# VMZ-2364: Removed — these Allow rules only existed to out-rank the
# '/*topic*' and '/*keyword*' Disallow rules for paginated URLs; with those
# removed they are no-ops. Re-add by uncommenting if the Disallows return.
# Allow: /vz-ratgeber?p=*
# Allow: /vz-ratgeber/merkblaetter?p=*
# Allow: /vz-ratgeber/vz-news?p=*
# Allow: /vz-ratgeber/buecher?p=*
# Allow: /vz-ratgeber/fachartikel?p=*
# Allow: /vz-ratgeber/erklaervideos?p=*
# Allow: /vz-ratgeber/vortraege-webinare?p=*
# Allow: /checks-und-analysen?p=*
Allow: /wissen/

# Directories
Disallow: /core/
Disallow: /profiles/
Disallow: /sites/default/files/documents/
Disallow: /faq/modal/
# Files
Disallow: /README.txt
Disallow: /web.config
Disallow: /*.pdf$
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /formulare/
Disallow: /node/
Disallow: /node/add/
Disallow: /offer/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /user
Disallow: /login
Disallow: /devel/
Disallow: /run-cron
Disallow: /update.php
Disallow: /events?*
Disallow: /fr/events?*
Disallow: /it/events?*
Disallow: /search/all/*
Disallow: /fr/search/all/*
Disallow: /it/search/all/*
# VMZ-2364: Removed — URLs with filter/pagination query parameters now carry
# a 'noindex, follow' X-Robots-Tag header. Re-add by uncommenting if needed.
# Disallow: /*topic*
# Disallow: /*keyword*
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /news/load-more*

# Block legacy market detail URL (redirects to SEO-friendly paths)
Disallow: /boersen-und-maerkte/maerkte/details

# Block AI training scrapers (no referral value, pure data collection)
User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Brightbot
Disallow: /

User-agent: LAIONDownloader
Disallow: /

User-agent: Timpibot
Disallow: /

Sitemap: https://www.vermoegenszentrum.de/sitemap.xml