# ————————————————————————
# BuildPiper.io - robots.txt
# Updated: May 2026 | Optimised for SEO + GEO/AI Visibility
# ————————————————————————

# === GENERAL RULES ===
User-agent: *
Allow: /

# Core WordPress blocks
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# Search & internal pages
Disallow: /?s=
Disallow: /search/

# Tags & pagination (low SEO value)
Disallow: /tag/
Disallow: /page/
Disallow: /?page_id=

# RSS feeds (no SEO value)
Disallow: /feed/
Disallow: */feed/$
Disallow: /?feed=

# WordPress date archives (no SEO value)
Disallow: /2020/
Disallow: /2021/
Disallow: /2022/
Disallow: /2023/
Disallow: /2024/
Disallow: /2025/
Disallow: /2026/

# WooCommerce — block all product/shop pages
# (test store pages found in 404 list — sneakers, hoodies, clothes etc.)
Disallow: /product/
Disallow: /product-category/
Disallow: /product-tag/
Disallow: /shop/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?add-to-cart=

# Block query strings that generate duplicate/junk URLs
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?cat=
Disallow: /*?tag=
Disallow: /*?p=

# === GOOGLE ===
User-agent: Googlebot
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Google AI Training (Gemini/Vertex) — allow for GEO visibility
User-agent: Google-Extended
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

# === BING ===
# Note: Crawl-delay removed — Googlebot ignores it and flags a warning
User-agent: Bingbot
Disallow: /wp-admin/

# === OPENAI ===
# ChatGPT Search indexing
User-agent: OAI-SearchBot
Allow: /

# User-triggered browsing via ChatGPT
User-agent: ChatGPT-User
Allow: /

# GPT model training crawler
User-agent: GPTBot
Allow: /

# === ANTHROPIC (Claude) ===
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# === PERPLEXITY ===
User-agent: PerplexityBot
Allow: /

# === APPLE ===
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# === META ===
User-agent: FacebookBot
Allow: /

# === AMAZON ===
User-agent: Amazonbot
Allow: /

# === MISTRAL ===
User-agent: MistralAI-User
Allow: /

# === COHERE ===
User-agent: cohere-ai
Allow: /

# === DEEPSEEK ===
User-agent: DeepSeekBot
Allow: /

# === DUCKDUCKGO ===
User-agent: DuckAssistBot
Allow: /

# === COMMON CRAWL (feeds most open-source LLMs) ===
User-agent: CCBot
Allow: /

# ————————————————————————
# Sitemap
# ————————————————————————
Sitemap: https://buildpiper.io/sitemap_index.xml