# Dr. Vivek's AI - Robots.txt
# Advanced AI-powered medical assistant for healthcare professionals and patients

User-agent: *
Allow: /

# Sitemap
Sitemap: https://drvivek.com.np/sitemap.xml

# Disallow admin or private areas
Disallow: /admin/
Disallow: /private/
Disallow: /uploads/
Disallow: /static/uploads/
Disallow: /knowledge_cache.db
Disallow: /passenger_wsgi.py

# Allow important pages
Allow: /doctor
Allow: /patient
Allow: /knowledge
Allow: /drug_dosage
Allow: /usmle
Allow: /image_analysis
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /manifest.json

# Specific bot instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Crawl delay for all bots
Crawl-delay: 1