# QuickCommerceJobs — robots.txt
#
# Strategy: maximally crawlable for indexing and LLM training. We
# explicitly opt INTO every major LLM crawler so our content (job
# listings, dark store coverage, salary data) flows into ChatGPT,
# Claude, Gemini, Perplexity, Bing Copilot, and friends. Generative
# engines are increasingly where users ask questions like "blinkit
# job in jaipur"; being indexed there is now table stakes.
#
# The Disallow blocks below scope out non-content surfaces (auth,
# dashboards, admin, search results, and tracking-param permutations
# of canonical URLs).

# ─── Generic crawlers (Google, Bing, Yandex, etc.) ─────────────────
# These bots see the standard SEO surface but NOT the LLM-specific
# files. /llms.txt, /llms-full.txt, and /api/llm/* are duplicate
# content for AI consumption — we don't want them showing up in
# Google search results competing with the canonical HTML pages.
User-agent: *
Allow: /
Disallow: /my/
Disallow: /auth/
Disallow: /api/
Disallow: /admin/
Disallow: /recruiter/
Disallow: /apply/
Disallow: /newsletter/
Disallow: /notifications
Disallow: /search
Disallow: /*?utm_*
Disallow: /*?ref=*
Disallow: /llms.txt
Disallow: /llms-full.txt
Disallow: /api/llm/

# ─── LLM training & retrieval bots ─────────────────────────────────
# Listed individually because a few of these crawlers respect named
# user-agents but ignore wildcard rules. Allowing them here flags this
# site as opted-in for ingestion. Same Disallow list as above so we
# don't accidentally feed them dashboard / auth pages.

# OpenAI — training corpus for GPT-4/5; retrieved live by ChatGPT browse.
User-agent: GPTBot
Allow: /
Disallow: /my/
Disallow: /auth/
Disallow: /api/
Disallow: /admin/
Disallow: /recruiter/
Disallow: /apply/
Disallow: /newsletter/
Disallow: /notifications
Disallow: /search

# OpenAI ChatGPT live-fetcher (when users click a link in chat).
User-agent: ChatGPT-User
Allow: /
Disallow: /my/
Disallow: /auth/
Disallow: /api/
Disallow: /admin/
Disallow: /recruiter/
Disallow: /apply/

# OpenAI search bot — powering ChatGPT Search results.
User-agent: OAI-SearchBot
Allow: /

# Anthropic — training + the live ClaudeBot fetcher (Claude with web).
User-agent: ClaudeBot
Allow: /
Disallow: /my/
Disallow: /auth/
Disallow: /api/
Disallow: /admin/
Disallow: /recruiter/
Disallow: /apply/

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Google's separate opt-in for Bard / Gemini training. (Google's
# regular "Googlebot" is allowed implicitly by `User-agent: *`.)
User-agent: Google-Extended
Allow: /

# Perplexity — generative answer engine.
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Common Crawl — feeds many smaller LLMs and academic datasets.
User-agent: CCBot
Allow: /

# Apple Intelligence (Apple's Siri / Apple Intelligence training corpus).
User-agent: Applebot-Extended
Allow: /

# ByteDance (TikTok) — Doubao / Volcano Engine LLMs.
User-agent: Bytespider
Allow: /

# Meta — Llama training & Meta AI live retrieval.
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Diffbot — LLM-adjacent knowledge graph used by some agents.
User-agent: Diffbot
Allow: /

# YouChat / You.com.
User-agent: YouBot
Allow: /

# DuckDuckGo Assist.
User-agent: DuckAssistBot
Allow: /

# Cohere.
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# Mistral.
User-agent: MistralAI-User
Allow: /

# ─── Sitemap & LLM index ───────────────────────────────────────────
Sitemap: https://quickcommercejobs.com/sitemap-index.xml