# VisuStock Robots.txt
# Last updated: 2026-02-04
# Optimized for Google Search Console indexing

User-agent: *
Allow: /
Allow: /marketplace
Allow: /products/
Allow: /s/products/
Allow: /s/categories/
Allow: /s/categories
Allow: /s/collections/
Allow: /s/collections
Allow: /about
Allow: /contact
Allow: /support
Allow: /licenses
Allow: /packages-pricing
Allow: /ai-image-generator
Allow: /buy-credits
Allow: /en/
Allow: /fr/
Allow: /es/
Allow: /de/
Allow: /pt/
Allow: /blog
Allow: /blog/

# Disallow private/authenticated routes
Disallow: /admin
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard/
Disallow: /seller-dashboard
Disallow: /seller-dashboard/
Disallow: /buyer-dashboard
Disallow: /buyer-dashboard/
Disallow: /auth
Disallow: /auth/
Disallow: /checkout
Disallow: /checkout/
Disallow: /cart
Disallow: /cart/
Disallow: /file-upload
Disallow: /file-upload/
Disallow: /product-management
Disallow: /product-management/
Disallow: /become-seller
Disallow: /become-seller/
Disallow: /payment-success
Disallow: /payment-cancelled
Disallow: /subscription-success
Disallow: /seller-registration-success
Disallow: /seller-registration-cancelled
Disallow: /portfolio
Disallow: /test-accounts
Disallow: /create-test-accounts
Disallow: /studio-ai

# Block low-value filter/search pages (crawl trap prevention)
# These return 200 but have duplicate or thin content
Disallow: /marketplace?*price=*
Disallow: /marketplace?*format=*
Disallow: /marketplace?*license=*
Disallow: /marketplace?*orientation=*
Disallow: /marketplace?*duration=*
Disallow: /marketplace?*search=*
Disallow: /marketplace?*sort=*
Disallow: /marketplace?*page=*
Disallow: /marketplace?*theme=*

# Block SEO-unfriendly search routes
Disallow: /videos/*
Disallow: /photos/*
Disallow: /audio/*
Disallow: /ebooks/*

# Block legacy UUID-based product URLs (redirects exist)
Disallow: /product/

# Allow only clean category facets
Allow: /marketplace?category=*

# ===== AI Crawlers (Allowed) =====

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# Google / Gemini
User-agent: Googlebot
Allow: /
User-agent: Google-Extended
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /

# xAI / Grok
User-agent: GrokBot
Allow: /
User-agent: xAI-Grok
Allow: /

# Perplexity & CommonCrawl
User-agent: PerplexityBot
Allow: /
User-agent: CCBot
Allow: /

# Sitemaps - split by type for scalability
Sitemap: https://visustock.com/sitemap.xml