# ============================================================= # SECTION 1 — AI SEARCH CRAWLERS (EXPLICITLY ALLOWED) # ------------------------------------------------------------- # Anthropic split ClaudeBot into THREE bots in early 2026: # ClaudeBot = training crawler # Claude-SearchBot = powers Claude's search/answer citations # Claude-User = live fetch when a Claude.ai user asks about a page # "anthropic-ai" and "Claude-Web" are DEPRECATED — Anthropic no # longer reads them, so they're removed (keeping them does nothing). # ============================================================= # OpenAI — trains + powers ChatGPT User-agent: GPTBot Allow: / # OpenAI — the crawler behind ChatGPT Search results User-agent: OAI-SearchBot Allow: / # OpenAI — live fetch when a ChatGPT user opens/browses a link User-agent: ChatGPT-User Allow: / # Anthropic — Claude's training crawler User-agent: ClaudeBot Allow: / # Anthropic — powers Claude's search/citation answers User-agent: Claude-SearchBot Allow: / # Anthropic — live fetch by Claude on user request User-agent: Claude-User Allow: / # Perplexity AI — answer engine citations User-agent: PerplexityBot Allow: / # Perplexity — live fetch when a user's query needs your page User-agent: Perplexity-User Allow: / # Google — controls use in Gemini + AI Overviews training # (does NOT affect normal Google Search indexing) User-agent: Google-Extended Allow: / # Apple — powers Apple Intelligence / Siri answers User-agent: Applebot-Extended Allow: / # Microsoft Bing — powers Copilot answers User-agent: Bingbot Allow: / # Common Crawl — open dataset used to train many LLMs User-agent: CCBot Allow: / # ============================================================= # SECTION 2 — SHOPIFY'S DYNAMIC DEFAULT RULES + SITEMAP # ------------------------------------------------------------- # DO NOT add a manual "Sitemap:" line anywhere else in this file. # This loop generates it automatically from the live sitemap.xml. # A duplicate manual Sitemap line previously pointed at a stale file. # ============================================================= User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://originalscbd.co.uk/sitemap.xml User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /services/login_with_shop User-agent: Nutch Disallow: / User-agent: AhrefsBot Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://originalscbd.co.uk/sitemap.xml User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://originalscbd.co.uk/sitemap.xml User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1 # ============================================================= # SECTION 3 — CUSTOM CRAWL-BUDGET RULES (ONE CONSOLIDATED GROUP) # ------------------------------------------------------------- # IMPORTANT: only ONE "User-agent: *" block for these custom rules. # Strict parsers only read the FIRST matching group for a given # user-agent name — a second "*" block risks being ignored. # ============================================================= User-agent: * # --- Shopify system & account paths --- Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /checkout/ Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /orders Disallow: /carts Disallow: /account Disallow: /account/logout Disallow: /a/ Disallow: /apps/ Disallow: /community/ Disallow: /tools/ Disallow: /services/login_with_shop Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products # --- Search results pages (thin/duplicate) --- Disallow: /search # --- Policies (allow only the privacy policy) --- Disallow: /policies/ Disallow: /*/policies/ Allow: /policies/privacy-policy # --- Collection sort/filter parameter variants (duplicate content) --- Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* # --- Blog parameter variants --- Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* # --- Blog tag archive pages (auto-generated thin content) --- Disallow: /blogs/*/tagged/* # --- Atom feeds (no SEO value) --- Disallow: /*.atom # --- Shopify search/filter parameter variants (keep clean canonical only) --- Disallow: /*?_pos=* Disallow: /*?*_pos=* Disallow: /*?_psq=* Disallow: /*?*_psq=* Disallow: /*?_ss=* Disallow: /*?*_ss=* Disallow: /*?_v=* Disallow: /*?*_v=* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* # --- Shopify "remote" product duplicate URLs --- Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote # ============================================================= # SECTION 4 — SEO TOOL & MISC CRAWLERS (RATE-LIMITED / BLOCKED) # ------------------------------------------------------------- # These are each their OWN named user-agent — safe to have as # separate groups, since the "one group per name" caution only # applies to duplicate/repeated identical names (like "*"). # ============================================================= # Ahrefs — your own audits; keep but rate-limit User-agent: AhrefsBot Crawl-delay: 10 Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /carts Disallow: /account Disallow: /search Disallow: /policies/ Disallow: /collections/*sort_by* Disallow: /*?_pos=* Disallow: /*?_ss=* Disallow: /blogs/*/tagged/* Disallow: /*.atom User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /admin Disallow: /cart Disallow: /checkout Disallow: /checkouts/ Disallow: /orders Disallow: /carts Disallow: /account Disallow: /search Disallow: /policies/ Disallow: /collections/*sort_by* Disallow: /*?_pos=* Disallow: /*?_ss=* Disallow: /blogs/*/tagged/* Disallow: /*.atom # Google Ads bot User-agent: adsbot-google Disallow: /checkout Disallow: /checkouts/ Disallow: /carts Disallow: /orders Disallow: /85752250716/checkouts Disallow: /85752250716/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop # Pinterest — allow, light rate-limit User-agent: Pinterest Crawl-delay: 1 # --- Aggressive/low-value bots: block entirely --- User-agent: Nutch Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Crawl-delay: 10