# START ROBOTS.TXT
User-agent: *

# Block WordPress admin and core files for security
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# Allow necessary AJAX functionality
Allow: /wp-admin/admin-ajax.php

# Block theme, plugin, and cache folders
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/

# Block thin content and duplicate content URLs
Disallow: /search/
Disallow: /*?s=
Disallow: /feed/
Disallow: */feed/
Disallow: */embed/
Disallow: */page/

Disallow: /go/

# Sitemap location - IMPORTANT
Sitemap: https://bestopschainai.com/sitemap_index.xml

# END ROBOTS.TXT