# ============================================================
# robots.txt — https://www.malhadinhanova.pt
# Herdade da Malhadinha Nova - Relais & Châteaux, Portugal
# Deploy at: https://https://www.malhadinhanova.pt/robots.txt
# Last updated: 2026-06-09
# ============================================================


# ------------------------------------------------------------
# SECTION 1 — BASELINE: ALL CRAWLERS
# ------------------------------------------------------------
User-agent: *
Disallow: /manutencao/

# ------------------------------------------------------------
# SECTION 2 — SEO / MARKETING TOOL CRAWLERS (crawl-rate control)
# ------------------------------------------------------------
User-agent: AhrefsBot
User-agent: MJ12bot
User-agent: Seekport Crawler
User-agent: SemrushBot
Crawl-delay: 5
Disallow: /manutencao/
 
User-agent: msnbot
User-agent: bingbot
Crawl-delay: 1
Disallow: /manutencao/

# ------------------------------------------------------------
# SECTION 3 — AI CRAWLERS (mirror of ai.txt SECTION 2)
# ------------------------------------------------------------
 
# ===== GROUP T — TRAINING / DATASET CRAWLERS =====
# Uncomment for PROFILE B and PROFILE C.
# User-agent: GPTBot
# User-agent: ClaudeBot
# User-agent: Google-Extended
# User-agent: Applebot-Extended
# User-agent: Meta-ExternalAgent
# User-agent: FacebookBot
# User-agent: CCBot
# User-agent: Bytespider
# User-agent: cohere-ai
# Disallow: /
 
# ===== GROUP T (legacy) — DEPRECATED ANTHROPIC TOKENS =====
# Optional belt-and-braces; replaced by ClaudeBot/Claude-User/
# Claude-SearchBot.
# User-agent: anthropic-ai
# User-agent: Claude-Web
# Disallow: /
 
# ===== GROUP S — SEARCH / ANSWER-INDEX CRAWLERS =====
# Uncomment ONLY for PROFILE C (blocking removes the site from
# AI search results and citations).
# User-agent: OAI-SearchBot
# User-agent: Claude-SearchBot
# User-agent: PerplexityBot
# User-agent: DuckAssistBot
# User-agent: Amazonbot
# Disallow: /
 
# ===== GROUP U — USER-INITIATED FETCHERS (live RAG) =====
# Uncomment ONLY for PROFILE C. Note: Perplexity-User generally
# ignores robots.txt; Bytespider compliance is unreliable — for
# hard enforcement use WAF / User-Agent rules (Apache/nginx).
# User-agent: ChatGPT-User
# User-agent: Claude-User
# User-agent: Perplexity-User
# User-agent: MistralAI-User
# Disallow: /

# ------------------------------------------------------------
# SECTION 4 — SITEMAP
# ------------------------------------------------------------
Sitemap: https://www.malhadinhanova.pt/sitemap.xml