# dubir.net — Dubir Group Limited, Cyprus.
#
# Open crawling. This is a marketing site: we want classic search visibility AND
# to be quotable by AI answer engines, so every group below is Allow: /.
#
# `Host:` was removed too: Yandex retired it in 2021 and nothing else ever read
# it. The canonical host is declared by the 301 from www and by <link canonical>.
#
# /embed/ is deliberately NOT disallowed. Those pages carry <meta name="robots"
# content="noindex,follow">, and a crawler has to be allowed to fetch a page to
# see that tag. Blocking them here would strand them as URL-only entries.
#
# The four Disallow lines repeated in every group hide Next's RSC payloads:
# `next build` emits /index.txt, /<route>/index.txt, /__next.*.txt and
# /<route>/__next.*.txt beside the HTML, each a text/plain copy of a page that
# the client router fetches during navigation. They must stay reachable —
# deleting them breaks client-side nav — but nothing should index a plaintext
# duplicate of a page we already rank.
# They are repeated because a named user-agent group replaces the * group
# outright; robots.txt has no inheritance and no include.

# --- Search engines --------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Google-InspectionTool
User-agent: Bingbot
User-agent: msnbot
User-agent: AdIdxBot
User-agent: Yandex
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Slurp
User-agent: SeznamBot
User-agent: Qwantbot
Allow: /
Disallow: /index.txt
Disallow: /*/index.txt
Disallow: /__next.
Disallow: /*/__next.

# --- AI answer engines: we want to be cited ---------------------------------
# Split into the two jobs these crawlers do. Retrieval bots fetch a page to
# answer a live question (a citation, a link, traffic). Training bots collect
# corpora. Both are yes here — a small Cyprus firm gains more from being
# in the weights than it loses.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: MistralAI-User
User-agent: cohere-ai
User-agent: YouBot
User-agent: Diffbot
User-agent: CCBot
Allow: /
Disallow: /index.txt
Disallow: /*/index.txt
Disallow: /__next.
Disallow: /*/__next.

# --- Link unfurlers ---------------------------------------------------------
User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Discordbot
User-agent: Slackbot-LinkExpanding
User-agent: redditbot
Allow: /
Disallow: /index.txt
Disallow: /*/index.txt
Disallow: /__next.
Disallow: /*/__next.

# --- Everyone else ----------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /index.txt
Disallow: /*/index.txt
Disallow: /__next.
Disallow: /*/__next.

Sitemap: https://dubir.net/sitemap.xml