# ==========================================
# LOOMLAN.COM - STRATEGIC AI-FORWARD ROBOTS
# Last Updated: June 2026
# CHANGE (Jun 2026): Added Disallow /a/s/ (search-app proxy) and
#   /*?*q=* (on-site search results) to the * and Googlebot blocks
#   to stop crawl-budget waste on non-indexable search URLs.
# ==========================================

User-agent: *

# ── Core Shopify system paths ──
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /apps
Disallow: /tools
Disallow: /a/downloads/-/*
Disallow: /cdn/wpm/*.js
Disallow: /.well-known/shopify/monorail
Disallow: /services/login*
Disallow: /products/option-set*
Disallow: /*.atom
Disallow: /*.rss
Disallow: /4268556360/checkouts
Disallow: /4268556360/orders

# ── Duplicate/low-value URL parameters ──
Disallow: /*?*variant=*
Disallow: /*&variant=*
Disallow: /*?*filter*
Disallow: /*&filter*
Disallow: /*?*sort_by*
Disallow: /*&sort_by*
Disallow: /collections/*?filter.p.m.custom.*
Disallow: /collections/*?*filter.p.product_type*
Disallow: /collections/*?*filter.p.vendor*
Disallow: /collections/*?*filter.v.availability*
Disallow: /collections/*/Custom
Disallow: /collections/*/custom
Disallow: /*?*utm_*
Disallow: /*&utm_*
Disallow: /*?*gclid=*
Disallow: /*&gclid=*
Disallow: /*?*fbclid=*
Disallow: /*&fbclid=*
Disallow: /*?*ref=*
Disallow: /*&ref=*
Disallow: /*?*campaign=*
Disallow: /*?*session_id=*
Disallow: /*?*_pos=*
Disallow: /*&_pos=*
Disallow: /*?*_sid=*
Disallow: /*&_sid=*
Disallow: /*?*_ss=*
Disallow: /*&_ss=*
Disallow: /*?*pr_prod_strat=*
Disallow: /*&pr_prod_strat=*
Disallow: /*?*pr_rec_id=*
Disallow: /*&pr_rec_id=*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /search*

# ── App search proxy & on-site search results ──
Disallow: /a/s/
Disallow: /*?*q=*

# ── Explicit allows for core content ──
Allow: /products/
Allow: /collections/
Allow: /pages/
Allow: /blogs/

# ==========================================
# GOOGLE
# FIX: Googlebot-specific blocks MUST be
# repeated here. User-agent: * does NOT
# carry over into named agent blocks —
# a named block fully overrides * for
# that agent. Without these Disallows,
# Googlebot can crawl checkout, admin,
# and all parameter URLs.
# ==========================================

User-agent: Googlebot
# ── Core Shopify system paths ──
Disallow: /admin
Disallow: /cart
Disallow: /checkout
Disallow: /checkouts/
Disallow: /orders
Disallow: /account
Disallow: /carts
Disallow: /apps
Disallow: /tools
Disallow: /a/downloads/-/*
Disallow: /cdn/wpm/*.js
Disallow: /.well-known/shopify/monorail
Disallow: /services/login*
Disallow: /products/option-set*
Disallow: /*.atom
Disallow: /*.rss
Disallow: /4268556360/checkouts
Disallow: /4268556360/orders
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# ── Duplicate/low-value URL parameters ──
Disallow: /*?*variant=*
Disallow: /*&variant=*
Disallow: /*?*filter*
Disallow: /*&filter*
Disallow: /*?*sort_by*
Disallow: /*&sort_by*
Disallow: /collections/*?filter.p.m.custom.*
Disallow: /collections/*?*filter.p.product_type*
Disallow: /collections/*?*filter.p.vendor*
Disallow: /collections/*?*filter.v.availability*
Disallow: /collections/*/Custom
Disallow: /collections/*/custom
Disallow: /*?*utm_*
Disallow: /*&utm_*
Disallow: /*?*gclid=*
Disallow: /*&gclid=*
Disallow: /*?*fbclid=*
Disallow: /*&fbclid=*
Disallow: /*?*ref=*
Disallow: /*&ref=*
Disallow: /*?*campaign=*
Disallow: /*?*session_id=*
Disallow: /*?*_pos=*
Disallow: /*&_pos=*
Disallow: /*?*_sid=*
Disallow: /*&_sid=*
Disallow: /*?*pr_prod_strat=*
Disallow: /*&pr_prod_strat=*
Disallow: /*?*pr_rec_id=*
Disallow: /*&pr_rec_id=*
Disallow: /*?*oseid=*
Disallow: /search*

# ── App search proxy & on-site search results ──
Disallow: /a/s/
Disallow: /*?*q=*

# ── Explicit allows for core content ──
Allow: /products/
Allow: /collections/
Allow: /pages/
Allow: /blogs/

User-agent: Googlebot-Image
Allow: /

User-agent: AdsBot-Google
Allow: /products/
Allow: /collections/
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders

# ==========================================
# BING & OTHERS
# ==========================================

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# ==========================================
# AI ENGINES — ALLOW ALL (CRITICAL FOR GEO)
# ==========================================

# OpenAI training crawler
User-agent: GPTBot
Allow: /

# OpenAI live web browsing (ChatGPT citations)
# FIX ADDED: OAI-SearchBot was missing — this is
# the crawler that powers real-time ChatGPT answers.
# Without it, ChatGPT Browse cannot cite loomlan.com.
User-agent: OAI-SearchBot
Allow: /

# ChatGPT user-initiated browsing
User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: Anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl (feeds many AI training datasets)
User-agent: CCBot
Allow: /

# Google Gemini / AI Overviews training
User-agent: Google-Extended
Allow: /

# Apple Siri + Spotlight
User-agent: Applebot
Allow: /

# ==========================================
# SOCIAL MEDIA
# ==========================================

User-agent: Pinterest
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: FacebookBot
Allow: /

# NOTE: Bytespider = ByteDance/TikTok scraper.
# Keeping crawl-delay rather than blocking since
# TikTok social referral traffic is a channel.
# Change to Disallow: / if you want to block it.
User-agent: Bytespider
Allow: /

# ==========================================
# SEO TOOLS
# ==========================================

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

# ==========================================
# BLOCK: Aggressive scrapers & competitor AI
# ==========================================

User-agent: Amazonbot
Disallow: /

User-agent: Amazon-AI
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ==========================================
# SITEMAP
# ==========================================

Sitemap: https://loomlan.com/sitemap.xml