# ==========================================
# ROBOTS.TXT — Full AI and SEO Friendly Setup
# Last updated: 2025-11-19
# ==========================================

User-agent: *
Disallow:

# -------------------------------
# MAJOR SEARCH ENGINES (SEO)
# -------------------------------

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp  # Yahoo
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Exabot
Allow: /

User-agent: facebot  # Facebook preview
Allow: /

User-agent: ia_archiver  # Alexa (archiving)
Allow: /

# -------------------------------
# AI CRAWLERS & LLM INDEXING
# -------------------------------

# OpenAI (ChatGPT browsing & GPTBot)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google-Extended (Gemini indexing)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Amazon (Rufus AI, others)
User-agent: Amazonbot
Allow: /

# NeevaAI (now part of Snowflake)
User-agent: Neevabot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# GigaBot (Common Crawl, used by AI research)
User-agent: Gigabot
Allow: /

# CCBot (Common Crawl)
User-agent: CCBot
Allow: /

# Meta AI (LLaMA pretraining sources)
User-agent: FacebookBot
Allow: /

User-agent: MetaAI
Allow: /

# Andi (AI search engine)
User-agent: andi
Allow: /

# phind.com (developer-focused AI)
User-agent: PhindBot
Allow: /

# Writer.com AI
User-agent: writer
Allow: /

# Quora AI (Poe)
User-agent: Quora-Bot
Allow: /

# Kagi AI
User-agent: KagiBot
Allow: /

# OpenLinkProfiler (backlink index, AI-adjacent)
User-agent: openlinkprofiler
Allow: /


# -------------------------------
# CHINESE SEARCH ENGINES (BLOCKED)
# -------------------------------

User-agent: Baiduspider
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: HaosouSpider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: EtaoSpider
Disallow: /

User-agent: YoudaoBot
Disallow: /

User-agent: Shenma
Disallow: /


# -------------------------------
# SENSITIVE PATHS (Disallowed)
# -------------------------------

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /private/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cart/
Disallow: /checkout/
Disallow: /account/
Disallow: /search/

# -------------------------------
# SITEMAP
# -------------------------------

Sitemap: https://tascc.ca/sitemap_index.xml