# robots.txt for newatoms.com
# Last Updated: 2026-08-04
# Optimized for search engines and AI crawlers

# AI-SPECIFIC CRAWLERS - GENEROUS ACCESS
User-agent: Claude-Web
Allow: /
Crawl-Delay: 1
Request-Rate: 1/s

User-agent: GPTBot
Allow: /
Crawl-Delay: 1
Request-Rate: 1/s

User-agent: GoogleBot
Allow: /
Crawl-Delay: 1
Request-Rate: 1/s

User-agent: Bard
Allow: /
Crawl-Delay: 1

User-agent: Meta-Externalagent
Allow: /
Crawl-Delay: 1

User-agent: Copilot
Allow: /
Crawl-Delay: 1

User-agent: PerplexityBot
Allow: /
Crawl-Delay: 1

User-agent: AppleBot-Extended
Allow: /
Crawl-Delay: 1

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Bingbot
Allow: /

# FALLBACK FOR ALL OTHER BOTS
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /customer-accounts/
Disallow: /user-profiles/
Disallow: /order-history/
Disallow: /payment-info/
Disallow: /personal-data/
Disallow: /search/
Disallow: /?s=
Disallow: /?p=
Disallow: /?*
Disallow: /*?
Disallow: /*.pdf
Disallow: /private/
Disallow: /confidential/
Disallow: /wp-json/
Disallow: /api/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /trackback/
Disallow: /cgi-bin/

Crawl-Delay: 2
Request-Rate: 1/2

# SITEMAPS
Sitemap: https://newatoms.com/sitemap.xml
Sitemap: https://newatoms.com/sitemap-products.xml
Sitemap: https://newatoms.com/sitemap-brands.xml
Sitemap: https://newatoms.com/sitemap-blog.xml

# SPECIFIC DIRECTORY RULES

# Allow product and shop directories
Allow: /products/
Allow: /shop/
Allow: /category/
Allow: /brand/
Allow: /brands/

# Allow content directories
Allow: /blog/
Allow: /guides/
Allow: /resources/
Allow: /faq/
Allow: /about/
Allow: /articles/

# Allow static assets
Allow: /wp-content/uploads/
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$

# CRAWL DELAY FOR PERFORMANCE
# Default crawl delay: 2 seconds for regular bots
# AI crawlers have 1 second delay for better indexing

# REQUEST RATE
# Limit to 30 requests per minute for safety

# COMMENTS POLICY
# Disallow indexing of comments in RSS to avoid duplicate content
Disallow: /*/comments/feed/

# AVOID CRAWLING SESSION/TEMPORARY FILES
Disallow: *?
Disallow: *.php?
Disallow: */trackback/
Disallow: */feed/atom/

# E-COMMERCE SPECIFIC
# Disallow checkout and cart processes
Disallow: /shop/cart/
Disallow: /shop/checkout/
Disallow: /wc-api/
Disallow: /wp-admin/admin-ajax.php

# QUERY STRING PARAMETERS TO AVOID
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=
Disallow: /*?ref=

# ALLOW SPECIFIC PARAMETERS FOR E-COMMERCE
Allow: /*?product_cat=
Allow: /*?product_tag=

# PROGRESSIVE WEB APP
Allow: /manifest.json
Allow: /service-worker.js

# ACCESSIBILITY
Allow: /accessibility/
Allow: /wcag/

# NOTE: This robots.txt is designed to balance:
# 1. AI crawler access for training and indexing
# 2. Search engine optimization
# 3. Protection of sensitive user data
# 4. Server resource protection
# 5. E-commerce platform integrity

# POLICY URL
# Full crawl policy available at: https://newatoms.com/crawler-policy
# Questions or issues: info@shftx.tech