# ===========================================
# AI CRAWLER ACCESS - PRIORITY RULES
# ===========================================

# OpenAI Crawlers (ChatGPT, SearchGPT)
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Crawlers (Claude)
User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

# Google AI (Gemini/Bard)
User-agent: Google-Extended
Allow: /

# Apple AI (Siri, Spotlight)
User-agent: Applebot-Extended
Allow: /

# Amazon AI (Alexa)
User-agent: Amazonbot
Allow: /
Crawl-delay: 1

# Common Crawl
User-agent: CCBot
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ===========================================
# TRADITIONAL SEARCH ENGINES
# ===========================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ===========================================
# SEO TOOL CRAWLERS
# ===========================================

User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 2

# ===========================================
# SHOPIFY DEFAULT RULES
# ===========================================


User-agent: *

Disallow: /a/downloads/-/*

Disallow: /admin

Disallow: /cart

Disallow: /orders

Disallow: /checkouts/

Disallow: /checkout

Disallow: /1415938083/checkouts

Disallow: /1415938083/orders

Disallow: /carts

Disallow: /account

Disallow: /collections/*sort_by*

Disallow: /*/collections/*sort_by*

Disallow: /collections/*+*

Disallow: /collections/*%2B*

Disallow: /collections/*%2b*

Disallow: /*/collections/*+*

Disallow: /*/collections/*%2B*

Disallow: /*/collections/*%2b*

Disallow: */collections/*filter*&*filter*

Disallow: /blogs/*+*

Disallow: /blogs/*%2B*

Disallow: /blogs/*%2b*

Disallow: /*/blogs/*+*

Disallow: /*/blogs/*%2B*

Disallow: /*/blogs/*%2b*

Disallow: /*?*oseid=*

Disallow: /*preview_theme_id*

Disallow: /*preview_script_id*

Disallow: /policies/

Disallow: /*/policies/

Disallow: /*/*?*ls=*&ls=*

Disallow: /*/*?*ls%3D*%3Fls%3D*

Disallow: /*/*?*ls%3d*%3fls%3d*

Disallow: /search

Disallow: /apple-app-site-association

Disallow: /.well-known/shopify/monorail

Disallow: /cdn/wpm/*.js

Disallow: /recommendations/products

Disallow: /*/recommendations/products

Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote


Sitemap: https://cermatreatment.com/sitemap.xml



User-agent: adsbot-google

Disallow: /checkouts/

Disallow: /checkout

Disallow: /carts

Disallow: /orders

Disallow: /1415938083/checkouts

Disallow: /1415938083/orders

Disallow: /*?*oseid=*

Disallow: /*preview_theme_id*

Disallow: /*preview_script_id*

Disallow: /cdn/wpm/*.js

Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote




User-agent: Nutch

Disallow: /




User-agent: AhrefsBot

Crawl-delay: 10

Disallow: /a/downloads/-/*

Disallow: /admin

Disallow: /cart

Disallow: /orders

Disallow: /checkouts/

Disallow: /checkout

Disallow: /1415938083/checkouts

Disallow: /1415938083/orders

Disallow: /carts

Disallow: /account

Disallow: /collections/*sort_by*

Disallow: /*/collections/*sort_by*

Disallow: /collections/*+*

Disallow: /collections/*%2B*

Disallow: /collections/*%2b*

Disallow: /*/collections/*+*

Disallow: /*/collections/*%2B*

Disallow: /*/collections/*%2b*

Disallow: */collections/*filter*&*filter*

Disallow: /blogs/*+*

Disallow: /blogs/*%2B*

Disallow: /blogs/*%2b*

Disallow: /*/blogs/*+*

Disallow: /*/blogs/*%2B*

Disallow: /*/blogs/*%2b*

Disallow: /*?*oseid=*

Disallow: /*preview_theme_id*

Disallow: /*preview_script_id*

Disallow: /policies/

Disallow: /*/policies/

Disallow: /*/*?*ls=*&ls=*

Disallow: /*/*?*ls%3D*%3Fls%3D*

Disallow: /*/*?*ls%3d*%3fls%3d*

Disallow: /search

Disallow: /apple-app-site-association

Disallow: /.well-known/shopify/monorail

Disallow: /cdn/wpm/*.js

Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote


Sitemap: https://cermatreatment.com/sitemap.xml



User-agent: AhrefsSiteAudit

Crawl-delay: 10

Disallow: /a/downloads/-/*

Disallow: /admin

Disallow: /cart

Disallow: /orders

Disallow: /checkouts/

Disallow: /checkout

Disallow: /1415938083/checkouts

Disallow: /1415938083/orders

Disallow: /carts

Disallow: /account

Disallow: /collections/*sort_by*

Disallow: /*/collections/*sort_by*

Disallow: /collections/*+*

Disallow: /collections/*%2B*

Disallow: /collections/*%2b*

Disallow: /*/collections/*+*

Disallow: /*/collections/*%2B*

Disallow: /*/collections/*%2b*

Disallow: */collections/*filter*&*filter*

Disallow: /blogs/*+*

Disallow: /blogs/*%2B*

Disallow: /blogs/*%2b*

Disallow: /*/blogs/*+*

Disallow: /*/blogs/*%2B*

Disallow: /*/blogs/*%2b*

Disallow: /*?*oseid=*

Disallow: /*preview_theme_id*

Disallow: /*preview_script_id*

Disallow: /policies/

Disallow: /*/policies/

Disallow: /*/*?*ls=*&ls=*

Disallow: /*/*?*ls%3D*%3Fls%3D*

Disallow: /*/*?*ls%3d*%3fls%3d*

Disallow: /search

Disallow: /apple-app-site-association

Disallow: /.well-known/shopify/monorail

Disallow: /cdn/wpm/*.js

Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote

Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote


Sitemap: https://cermatreatment.com/sitemap.xml



User-agent: MJ12bot

Crawl-delay: 10




User-agent: Pinterest

Crawl-delay: 1





# ===========================================
# BLOCK DUPLICATE CONTENT
# ===========================================

User-agent: *
Disallow: /collections/*+*
Disallow: /collections/*?*sort_by*
Disallow: /collections/*?*filter*
Disallow: /search
Disallow: /search?*
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts
Disallow: /account
Disallow: /admin

# Block Shopify system endpoints (Added Feb 2026 v1.1)
Disallow: /.well-known/
Disallow: /services/
Disallow: /*/cart
Disallow: /*/recommendations/products

# ===========================================
# SITEMAP
# ===========================================

Sitemap: https://cermatreatment.com/sitemap.xml
Sitemap: https://llms.cermatreatment.com/sitemap.xml