#
#Sitemaps
Sitemap: https://www.labunlimited.com/sitemapindex.xml

##--##

#####################
#SEO Crawlers

#BLEXBOT http://webmeup-crawler.com/ - SEO
User-agent: BLEXBot
Crawl-delay: 10

#https://ahrefs.com/robot - SEO
User-agent: AhrefsBot
Crawl-Delay: 10

#Majestic http://mj12bot.com/ - SEO
User-agent: MJ12bot
Crawl-Delay: 10

#http://www.semrush.com/bot.html - SEO 
User-agent: SemrushBot
Crawl-Delay: 10

# https://megaindex.com/crawler - SEO
# Has a maximum crawl delay of 5
User-agent: megaindex.com
Crawl-Delay: 5

#https://opensiteexplorer.org/dotbot - SEO
User-agent: dotbot
Crawl-delay: 10

#https://www.screamingfrog.co.uk/seo-spider - SEO
#Doesn't respect crawl-delay
User-agent: Screaming Frog SEO Spider
Disallow: /

#####################
#Agressive Search Engines

#http://www.bing.com/bingbot.htm - Bing Search
User-agent: Bingbot
Crawl-delay: 1

#https://aspiegel.com/petalbot - Petal Search (Huawei Assistant and AI Search services)
#Doesn't respect crawl-delay
User-agent: PetalBot
Disallow: /

#http://yandex.com/bots - Yandex Search Engine (Russian)
#Doesn't respect crawl-delay
User-agent: Yandex
Disallow: /

#https://developer.amazon.com/support/amazonbot - Alexa indexing
#Doesn't respect crawl-delay
User-agent: Amazonbot
Disallow: /

#https://bot.seekport.com - German Search Engine
User-agent: SeekportBot
Crawl-delay: 10

# bytedance spiders
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sogou inst spider
Disallow: /

#chatgpt bot
User-agent: GPTBot
Disallow: /