# Robots.txt for WordPress auction sites
# https://www.matan.org.il/robots.txt

User-agent: *
Allow: /

# Block admin area
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block WordPress system files
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /wp-mail.php
Disallow: /wp-trackback.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /wp-comments-post.php
Disallow: /wp-links-opml.php

# Block search and comments
Disallow: /?s=
Disallow: /*?s=
Disallow: /*&s=
Disallow: /search/
Disallow: *?replytocom
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/feed/
Disallow: /comments/
Disallow: /wp-comments-post.php

# Block utility WordPress pages
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /date/
Disallow: /attachment/

# Allow uploads
Allow: /wp-content/uploads/
Allow: /wp-content/uploads/*.jpg
Allow: /wp-content/uploads/*.jpeg
Allow: /wp-content/uploads/*.png
Allow: /wp-content/uploads/*.gif
Allow: /wp-content/uploads/*.pdf
Allow: /wp-content/uploads/*.doc
Allow: /wp-content/uploads/*.docx

# Auction system rules
# Allow public auctions, lots, and pagination
Allow: /auction/
Allow: /auction/*/part/
Allow: /auction/*/page/
Allow: /auction/*/lot*

# Block private auction paths
Disallow: /auction/*/admin/
Disallow: /auction/*/private/
Disallow: /bidspirit/*

# Block search and filter pages (uco-auctions plugin)
# Only URLs with query parameters are blocked; clean auction and lot pages remain accessible
Disallow: /*?category_by=
Disallow: /*&category_by=
Disallow: /*?phrase_by=
Disallow: /*&phrase_by=
Disallow: /*?sort_by=
Disallow: /*&sort_by=
Disallow: /*?is_sold=
Disallow: /*&is_sold=
Disallow: /*?per_page=
Disallow: /*&per_page=
Disallow: /*?taxonomy=
Disallow: /*&taxonomy=

# Block search and filter pages (Woocommerce plugin)
Disallow: /*?orderby=
Disallow: /*&orderby=
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?filter_
Disallow: /*&filter_
Disallow: /*?query_type_
Disallow: /*&query_type_
Disallow: /*?min_price=
Disallow: /*&min_price=
Disallow: /*?max_price=
Disallow: /*&max_price=
Disallow: /*?rating_filter=
Disallow: /*&rating_filter=
Disallow: /*?filter_
Disallow: /*&filter_
Disallow: /*?wc-ajax=
Disallow: /*&wc-ajax=
Disallow: /wc-api/
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?remove_item=
Disallow: /*&remove_item=
Disallow: /*?add_to_wishlist=
Disallow: /*&add_to_wishlist=
Disallow: /*?action=yith-woocompare-add-product
Disallow: /*?currency=
Disallow: /*&currency=
Disallow: /*?lang=
Disallow: /*&lang=

# Block API and AJAX (except public actions)
#Disallow: /wp-json/
#Disallow: /wp-json/wp/v2/

# Block cache and temporary files
Disallow: /wp-content/cache/
Disallow: /wp-content/backup-db/
Disallow: /wp-content/backups/
Disallow: /wp-content/upgrade/
Disallow: /wp-content/uploads/cache/
Disallow: /wp-content/uploads/backup/
Disallow: /wp-content/uploads/backups/
Disallow: /wp-content/wphb-logs/
Disallow: /wp-content/wphb-cache/

# Block logs and debug files
Disallow: /wp-content/debug.log
Disallow: /wp-content/error_log
Disallow: /wp-content/error.log
Disallow: /wp-content/php_error.log
Disallow: /wp-content/php_errors.log

# Block configuration files
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /web.config
Disallow: /robots.txt.bak
Disallow: /sitemap.xml.bak

# UTM block
Disallow: /*?utm_
Disallow: /*&utm_
Disallow: /*?gclid=
Disallow: /*&gclid=
Disallow: /*?fbclid=
Disallow: /*&fbclid=
Disallow: /*?yclid=
Disallow: /*&yclid=

# Search engine crawlers
User-agent: Bingbot
Allow: /
Crawl-delay: 10

User-agent: Slurp
Allow: /
Crawl-delay: 10

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 10

User-agent: Baiduspider
Allow: /
Crawl-delay: 10

User-agent: YandexBot
Allow: /
Crawl-delay: 10

# Block aggressive bots
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Sitemap
Sitemap: https://www.matan.org.il/sitemap.xml