# robots.txt for iwebad.com
# 允许主流搜索引擎抓取，屏蔽无用采集器和敏感目录

User-agent: *
# 屏蔽帝国CMS核心动态目录，节省服务器性能
Disallow: /e/admin/
Disallow: /e/class/
Disallow: /e/data/
Disallow: /e/enews/
Disallow: /e/install/
Disallow: /e/member/
Disallow: /e/payapi/
Disallow: /e/sch/
Disallow: /e/search/
Disallow: /e/shopsys/
Disallow: /e/tool/
Disallow: /d/

# 屏蔽搜索结果页和评论页（日志中压力最大的地方）
Disallow: /search/
Disallow: /comment/
Disallow: /u/*/comment/

# 屏蔽特定的商业/恶意爬虫（节省带宽和IO）
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: python-requests
Disallow: /

# 允许主流搜索引擎抓取
User-agent: Baiduspider
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: 360Spider
Allow: /

# 指定网站地图（如果有的话）
Sitemap: https://iwebad.com/sitemap/index.xml