# Graphene-BSM Ltd — robots.txt
# Allow normal search-engine crawling and AI search/retrieval.
# Block known AI training/model-development crawlers where
# the provider provides a separate control.
#
# Note: robots.txt is a crawler directive, not an access-control
# mechanism. Cloudflare/WAF/server rules must also permit the
# crawlers you want to reach.

User-agent: *
Allow: /

# ============================================================
# SEARCH ENGINES
# ============================================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

# ============================================================
# AI SEARCH / ANSWER RETRIEVAL
# ============================================================

# OpenAI — ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

# OpenAI — user-directed page retrieval
User-agent: ChatGPT-User
Allow: /

# OpenAI — advertising crawler
User-agent: OAI-AdsBot
Allow: /

# Anthropic — user-directed retrieval
User-agent: Claude-User
Allow: /

# Anthropic — search/retrieval
User-agent: Claude-SearchBot
Allow: /

# Perplexity — search/retrieval
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ============================================================
# AI TRAINING / MODEL-DEVELOPMENT CRAWLERS
# ============================================================

# Google-Extended:
# Disallow use of site content for Gemini model training /
# related Google generative-AI data use controlled by this token.
# This does not block Google Search crawling or indexing.

User-agent: Google-Extended
Allow: /

# OpenAI:
# GPTBot is used for model-development/training crawling.
# OAI-SearchBot and ChatGPT-User remain allowed above.

User-agent: GPTBot
Allow: /

# Anthropic:
# ClaudeBot is associated with model-development data collection.
# Claude-User and Claude-SearchBot remain allowed above.

User-agent: ClaudeBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Apple:
# Applebot-Extended is the separate control for AI-model training.
# Applebot itself remains allowed above for search/discovery.

User-agent: Applebot-Extended
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /

# ============================================================
# SITEMAP
# ============================================================

Sitemap: https://graphene-bsm.com/sitemap.xml