# robots.txt for Quality Standard Ohio
# https://qualitystandardohio.com/

# Allow legitimate search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Applebot
Allow: /

User-agent: AhrefsBot
Allow: /

# Block known malicious bots and crawlers

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: 360Spider
Disallow: /

# Block common scraping tools
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Xaldon WebSpider
Disallow: /

# Main rules for all other bots
User-agent: *
Allow: /
Allow: /blog/
Allow: /areas/
Allow: /residential/
Allow: /commercial/
Allow: /specialized/

# Protect sensitive areas
Disallow: /admin/
Disallow: /includes/
Disallow: /layouts/
Disallow: /data/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/
Disallow: /search
Disallow: /feed/
Disallow: /trackback/
Disallow: /comments/
Disallow: /*?
Disallow: /*?*
Disallow: /*.php$
Disallow: /*.json$
Disallow: /*.xml$
Disallow: /cgi-bin/
Disallow: /tmp/

# Media and resource protection
Allow: /img/*.avif$ 
Allow: /img/*.jpg$
Allow: /img/*.png$
Disallow: /css/
Disallow: /js/
Disallow: /fonts/

# Critical files
Allow: /sitemap.xml
Allow: /feed.xml

# Sitemaps
Sitemap: https://qualitystandardohio.com/sitemap.xml

# Additional Security Headers (add these to your .htaccess or server config)
# X-Robots-Tag: noarchive, noimageindex
# X-Frame-Options: SAMEORIGIN
# X-Content-Type-Options: nosniff
# Referrer-Policy: strict-origin-when-cross-origin