User-agent: *
Allow: /

# Key pages for crawling
Allow: /category/*
Allow: /listing/*
Allow: /search*
Allow: /listings*
Allow: /help/*

# Disallow private areas
Disallow: /admin*
Disallow: /auth*
Disallow: /my-*
Disallow: /edit-*
Disallow: /review-*
Disallow: /messages*
Disallow: /profile*
Disallow: /import*
Disallow: /safepay-checkout/*

# Crawl delay (be respectful)
Crawl-delay: 1

# Sitemap location (sitemap index — references static + dynamic)
Sitemap: https://thetradingpost.ai/sitemap.xml

# Additional directives for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot  
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /
Allow: /category/*
Allow: /listing/*
Crawl-delay: 2

User-agent: CCBot
Allow: /
Allow: /category/*  
Allow: /listing/*
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Allow: /category/*
Allow: /listing/*
Crawl-delay: 2

# For AI training data collection
User-agent: GPTBot
Allow: /
Allow: /category/*
Allow: /listing/*

User-agent: PerplexityBot
Allow: /
Allow: /category/*
Allow: /listing/*

User-agent: anthropic-ai
Allow: /
Allow: /category/*
Allow: /listing/*

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /