# MediForge — © 2026 MediForge (ABN 67 884 705 627). robots.txt
# Legitimate search-engine indexing is welcome. AI-training crawlers are disallowed.
# See ai.txt and legal.html#copyright for our AI-training and content-use policy.
# Contact: security@mediforge.net

# ---------- Search engines: allowed, indexing welcomed ----------
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: DuckDuckBot
Allow: /

# ---------- AI-training / LLM-data crawlers: disallowed ----------
# We do not consent to this content being used for AI/ML model training.
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: YouBot
Disallow: /

# ---------- Generic scrapers: slowed, not fully blocked (Sitemaps below still apply) ----------
User-agent: *
Allow: /
Crawl-delay: 5
Disallow: /docs/
Disallow: /app/
Disallow: /dashboard.html
Disallow: /finance-cost-tracker.html
Disallow: /finance-breakeven.html
Disallow: /finance-automation.html
Disallow: /finance-mfa-guide.html
Disallow: /deploy-aws.html
Disallow: /readiness-checklist.html
Disallow: /social-setup-guide.html

Sitemap: https://mediforge.net/sitemap.xml
Sitemap: https://mediforge.net/local/sitemap-local.xml