# =============================================================================
#  robots.txt — https://sheikhfaiz.info/
#  Sheikh Faiz — Performance Marketing & Digital Growth Professional
#
#  Policy: this site WANTS to be crawled, indexed, quoted and cited, by both
#  classic search engines and AI answer engines. Everything is allowed on
#  purpose. Named agents below are listed so the intent is explicit and
#  auditable, not because any of them are being restricted.
#
#  Note on syntax: a crawler obeys ONLY the most specific group that matches
#  its name. Any agent named below therefore ignores the "User-agent: *" group
#  entirely. If you ever add a Disallow rule, you must add it to every group
#  that should honour it, not just to "*".
# =============================================================================

# ---------------------------------------------------------------- all crawlers
User-agent: *
Allow: /
Disallow: /404.html

# ------------------------------------------------------------- search engines
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

# ------------------------------------------------- AI answer & training agents
# Allowing these is what makes the site eligible to be retrieved, summarised
# and cited inside AI answers (Generative Engine Optimisation).

# Google — Gemini / AI Overviews grounding
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# OpenAI — ChatGPT search, browsing and training
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic — Claude search, browsing and training
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

User-agent: FacebookBot
Allow: /

# Microsoft Copilot
User-agent: MicrosoftPreview
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Common Crawl — the corpus most open models are trained on
User-agent: CCBot
Allow: /

# Others
User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: PetalBot
Allow: /

# --------------------------------------------------------------------- sitemap
Sitemap: https://sheikhfaiz.info/sitemap.xml

# ------------------------------------------------------------ machine-readable
# Structured summary for large language models (llmstxt.org convention):
#   https://sheikhfaiz.info/llms.txt
#   https://sheikhfaiz.info/llms-full.txt
# Security contact:
#   https://sheikhfaiz.info/.well-known/security.txt
# Primary entity:
#   Sheikh Faiz — https://sheikhfaiz.info/#sheikh-faiz
#   Entity home  — https://sheikhfaiz.info/who-is-sheikh-faiz.html