User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /*~
Disallow: /*.bak
Disallow: /*.sql
Disallow: /*.log
Disallow: /trackback/
Disallow: /feed/
Disallow: /xmlrpc.php

# Block  
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: wget
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Bytedance
Disallow: /
User-agent: Unknown robot (identified by hit on robots.txt)
Disallow: /
User-agent: Unknown robot identified by bot*
Disallow: /
User-agent: Unknown robot identified by *bot   
Disallow: /

# Prioritize
User-agent: Googlebot
Crawl-delay: 5
Allow: /
User-agent: Mediapartners-Google
Crawl-delay: 5
Allow: /
User-agent: Bingbot
Crawl-delay: 30
Allow: /


# Limit crawling   
User-agent: *
Crawl-delay: 60 
Disallow: /*?s=  
Disallow: /*&s=  

Sitemap: https://www.myengineers.com.ng/sitemap_index.xml