# Rule ordering matters here (restructured 2026-08-08).
#
# Google resolves conflicts by LONGEST match, so order is irrelevant to it. Bing and
# most AI crawlers use FIRST match. Under first-match the old layout was wrong twice:
# a blanket "Allow: /" sat on line 2, which shadowed every Disallow beneath it (so
# /admin/ and /portal/ were effectively open), and "Disallow: /api/" sat AFTER the
# three specific /api/ Allows it was meant to be narrowed by.
#
# The portable form, honoured by both strategies: specific Allow rules first, broad
# Disallow rules after, and no blanket "Allow: /" (a path with no matching Disallow
# is already allowed).

User-agent: *
Allow: /api/quote
Allow: /api/product-feed.json
Allow: /api/payment-link
Allow: /.well-known/
Allow: /llms-commerce.txt
Disallow: /admin/
Disallow: /portal/
Disallow: /api/
Disallow: /cost-segregation/_city_template
Disallow: /cost-segregation/_state_template
Disallow: /examples/_template
# Images hosted only for partner email newsletters — no page links to them.
Disallow: /email/

# AI crawlers — explicitly allowed, but NOT into the admin/portal/API surface.
# One group with many User-agent lines is spec-compliant and keeps these rules from
# drifting apart. Previously each agent had its own block containing only "Allow: /",
# which fully replaced the * group for that agent and therefore handed every named AI
# crawler the admin and customer-portal paths.
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Bytespider
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: Amazonbot
Allow: /api/product-feed.json
Allow: /llms-commerce.txt
Disallow: /admin/
Disallow: /portal/
Disallow: /api/

# Training-only crawlers — no search visibility benefit
User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

# LLM information: https://costsegsmart.com/llms.txt
# Sitemap (Astro @astrojs/sitemap emits a sitemap-index.xml pointing to
# sitemap-0.xml). Declaring the index directly eliminates a redirect hop
# vs. /sitemap.xml (which 301s here anyway).
Sitemap: https://costsegsmart.com/sitemap-index.xml