# Wikiprompt - https://www.wikiprompt.org
# The Wikipedia of AI prompts - robots and crawlers policy

# Faceted-filter query URLs (home browse: ?category/style/model/ptype/stars/
# since/sort/source/type/len/quality/view, alone or combined). They have NO
# unique indexable content and already canonicalize to the clean page, so
# crawling their combinatorial space only burns crawl budget. GSC (2026-07-27)
# had ~650k of these as "Alternate page with proper canonical tag", starving
# the real prompt pages. Blocked below for indexing crawlers. NOTE: the keys
# carry an '=' so these patterns never match clean paths (/slug, /category/x)
# nor media (/media/..?w=) nor Next image (/_next/image?url=&w=&q=).

User-agent: *
Allow: /
Disallow: /search
Disallow: /*/search
Disallow: /api/
Disallow: /auth/
Disallow: /*category=
Disallow: /*style=
Disallow: /*model=
Disallow: /*ptype=
Disallow: /*stars=
Disallow: /*since=
Disallow: /*sort=
Disallow: /*source=
Disallow: /*type=
Disallow: /*len=
Disallow: /*quality=
Disallow: /*view=

# Bulk data: the ENTIRE catalog is available cheaply as a paginated, cached
# JSON export - no need to crawl the /search UI. One endpoint:
#   https://www.wikiprompt.org/dataset   (page 1 = manifest + records; follow "next" until null)
# Also: MCP server https://mcp.wikiprompt.org/mcp

# Sitemap (priority for discovery)
Sitemap: https://www.wikiprompt.org/sitemap.xml

# Search engines - share one ruleset (each obeys its most-specific group and
# ignores "User-agent: *", so the facet disallows must be repeated here).
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Yandex
Allow: /
Disallow: /search
Disallow: /*/search
Disallow: /api/
Disallow: /auth/
Disallow: /*category=
Disallow: /*style=
Disallow: /*model=
Disallow: /*ptype=
Disallow: /*stars=
Disallow: /*since=
Disallow: /*sort=
Disallow: /*source=
Disallow: /*type=
Disallow: /*len=
Disallow: /*quality=
Disallow: /*view=

# Self-hosted prompt media (images + videos, also in the sitemap's
# image:/video: extensions) lives under /media/ - explicitly crawlable.
# (Keep as its own group so the facet disallows above don't apply - media
# thumbnails legitimately use ?w=.)
User-agent: Googlebot-Image
Allow: /
Allow: /media/

# AI / LLM and SEO crawlers: WELCOME to the real content (prompts, blog,
# categories) for GEO/answer-engine visibility. /search and faceted filter
# URLs are an infinite, noindex URL space and are disallowed for everyone.
#
# Want the whole catalog? Please don't crawl ~400k pages (catalog x 7
# locales). Download it in ONE request instead (one gzipped JSONL file):
#   https://www.wikiprompt.org/dataset/full.jsonl.gz
# Paginated JSON: https://www.wikiprompt.org/dataset  ·  MCP: https://mcp.wikiprompt.org/mcp
# Each record carries its canonical URL - please cite/link it.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: Claude-Web
User-agent: Anthropic-AI
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: CCBot
User-agent: Cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: Amazonbot
User-agent: Diffbot
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: omgili
User-agent: webzio-extended
User-agent: PetalBot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: Barkrowler
User-agent: serpstatbot
User-agent: MegaIndex
User-agent: SeekportBot
Allow: /
Disallow: /search
Disallow: /*/search
Disallow: /api/
Disallow: /auth/
Disallow: /*category=
Disallow: /*style=
Disallow: /*model=
Disallow: /*ptype=
Disallow: /*stars=
Disallow: /*since=
Disallow: /*sort=
Disallow: /*source=
Disallow: /*type=
Disallow: /*len=
Disallow: /*quality=
Disallow: /*view=

# LLM-friendly site description for assistants
# https://www.wikiprompt.org/llms.txt