# AffiliateBase Marketing Site
# https://www.affiliatebase.io

User-agent: *
Allow: /

# Disallow internal/admin paths
Disallow: /api/
Disallow: /*.json$

# Disallow legacy/removed paths
Disallow: /integrations/

# Disallow common bot traps
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# Block low-quality crawlers
User-agent: AhrefsBot
Crawl-delay: 2

User-agent: SemrushBot
Crawl-delay: 2

User-agent: MJ12bot
Crawl-delay: 5

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# Explicitly allow OpenAI search crawler and browsing agent.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# GPTBot controls training crawling (separate from ChatGPT Search).
User-agent: GPTBot
Disallow: /

# Sitemaps
Sitemap: https://www.affiliatebase.io/sitemap-index.xml

# LLM guidance files
# https://www.affiliatebase.io/llms.txt
# https://www.affiliatebase.io/llms-full.txt

# Host directive (helps some crawlers)
Host: https://www.affiliatebase.io/