# chrisgetsyousold.com
#
# The transfer tax index is original research: 162 municipalities, each verified
# against that municipality's own published documents. Terms of use, including
# the attribution licence and the text-and-data-mining reservation, are at
# https://chrisgetsyousold.com/terms and machine-readable at
# https://chrisgetsyousold.com/.well-known/tdmrep.json
#
# The split below is deliberate. Search crawlers -- including AI search -- are
# welcome, because being cited is the whole point. Crawlers that exist to build
# training corpora are not, because they take the work without returning a
# reader. Several operators run BOTH under different user agents, which is why
# this file names them individually instead of blanket-allowing everything.

# ---------------------------------------------------------------- search: allow
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# OpenAI's SEARCH crawler -- powers ChatGPT search results, which cite and link.
# Distinct from GPTBot below.
User-agent: OAI-SearchBot
Allow: /

# Fetches a page because a user asked for it by name. That is a reader.
User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --------------------------------------------------------------- training: deny
# Reserved under the Terms of Use and under Article 4 of Directive (EU) 2019/790.

# OpenAI's TRAINING crawler. Distinct from OAI-SearchBot above.
User-agent: GPTBot
Disallow: /

# Anthropic's training crawler. Distinct from Claude-SearchBot above.
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google's AI training and grounding signal. Does NOT affect Search inclusion,
# and AI Overviews follow the Search index via Googlebot above.
User-agent: Google-Extended
Disallow: /

# Common Crawl feeds a large share of public training corpora.
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

# --------------------------------------------------------------- everyone else
User-agent: *
Allow: /

Sitemap: https://chrisgetsyousold.com/sitemap.xml

# build: auto-deploy verified 2026-07-24