# =========================================
# GoGuides Robots Policy
# https://www.goguides.com/
#
# Public pages and public machine-readable
# trust resources are crawlable.
# Expensive search/API surfaces are limited.
# =========================================

# ---- Default policy ----
# Applies to Googlebot, Bingbot, OAI-SearchBot,
# GPTBot, ChatGPT-User, ClaudeBot,
# Claude-SearchBot, Claude-User, Applebot,
# Amazonbot and other compliant crawlers.

User-agent: *
Allow: /

# Public machine-readable trust surfaces
Allow: /machine-use.json
Allow: /docs/openapi.yaml

# GG_ACN_WELL_KNOWN_DISCOVERY_V1
# Exact machine-discovery surfaces
Allow: /.well-known/goguides.json
Allow: /.well-known/openapi.yaml
Allow: /.well-known/goguides-authority.json
Allow: /.well-known/goguides-authority-openapi.yaml

# Authority Clearing Network public discovery
Allow: /authority-clearing-network/
Allow: /api/v1/status
Allow: /api/v1/public-key
Allow: /api/v1/public-keys
Allow: /docs/request-url.html
Allow: /signal.json
Allow: /broadcast
Allow: /broadcast/
Allow: /ai-discovery
Allow: /ai-discovery/
Allow: /ai-discovery/before/

# Permit site-specific search checks
Allow: /search?q=site:
Allow: /search?q=site%3A
Allow: /search.php?q=site:
Allow: /search.php?q=site%3A

# Expensive / non-public crawl surfaces
Disallow: /search.php
Disallow: /ask-ai.php
Disallow: /api/

# Block bulk query scraping
Disallow: /*?q=
Disallow: /*&q=
Disallow: /*search=
Disallow: /*query=

# ---- Commercial SEO scrapers ----

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# ---- Sitemaps ----

Sitemap: https://www.goguides.com/sitemap.xml
Sitemap: https://www.goguides.com/sitemap-verified-text-1.xml
Sitemap: https://www.goguides.com/sitemap-verified-text-library.xml
Sitemap: https://www.goguides.com/sitemap-history.xml