# --- AI Retrieval Bots (ALLOW — these cite DFP in AI-generated answers) ---
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: GeminiBot
Allow: /

# --- AI Training Bots (BLOCK — training only, no citation benefit) ---
# Aligns with our "no AI training on customer data" policy (see /security).
# Retrieval bots above stay allowed so DFP can be cited in AI answers.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# --- Default rules ---
User-agent: *
Disallow: /php_files/
Disallow: /admin/
Disallow: /customer/
Disallow: /leads.php
Disallow: /lead_profile.php
Disallow: /forgot_password.php
Disallow: /reset_password.php
Disallow: /payment_success.php
Disallow: /payment_cancelled.php
Disallow: /stripe_webhook.php
Disallow: /chatbot_api.php
Disallow: /subscribe.php
Disallow: /help.php
Disallow: /login.php
Disallow: /status.php
Disallow: /backups/
Disallow: /old/
Disallow: /reports/
Disallow: /report/
Disallow: /markets/
# Spec 294: shareable-link capability URLs — also served with X-Robots-Tag.
# The public sample report is a marketing demo (in sitemap + linked from
# public pages) — allow it explicitly; longest-match wins for Googlebot so
# per-customer /deal_analysis/<token> capability URLs stay blocked.
Allow: /deal_analysis/sample_report
Disallow: /lender_summary/
Disallow: /pipeline_report/
Disallow: /deal_analysis/
Allow: /css/
Allow: /js/
Allow: /

Sitemap: https://dealflowpro.io/sitemap.xml