# Block bad actors and unneeded SEO tools and crawlers User-agent: MJ12bot Disallow: / Crawl-delay: 25 User-agent: DataForSeoBot Disallow: / Crawl-delay: 25 User-agent: DotBot Disallow: / Crawl-delay: 25 User-agent: BLEXBot Disallow: / Crawl-delay: 25 User-agent: AhrefsBot Disallow: / Crawl-delay: 25 User-agent: PetalBot Disallow: / Crawl-delay: 25 User-agent: ClaudeBot Disallow: / Crawl-delay: 25 User-agent: PerplexityBot Disallow: / Crawl-delay: 25 User-agent: GeedoProductSearch Disallow: / Crawl-delay: 25 User-agent: Barkrowler Disallow: / Crawl-delay: 25 User-agent: Amazonbot Disallow: / Crawl-delay: 25 User-agent: jetmon Disallow: / Crawl-delay: 25 User-agent: SeekportBot Disallow: / Crawl-delay: 25 User-agent: UptimeRobot Disallow: / Crawl-delay: 25 # Block or limit AI and data harvesters User-agent: OAI-SearchBot Crawl-delay: 25 User-agent: MetaBot Disallow: / Crawl-delay: 25 User-agent: Meta-ExternalAgent Disallow: / Crawl-delay: 25 User-agent: Applebot Disallow: / Crawl-delay: 25 # WordPress paths and crawl rules User-agent: * Disallow: /wp-admin/ Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /?s= Allow: /wp-admin/admin-ajax.php Allow: /wp-content/uploads/ Allow: /wp-content/themes/ Allow: /wp-content/plugins/ Allow: /wp-includes/js/ # Sitemap Sitemap: https://www.tensabarrieronline.com/sitemap_index.xml # START DARK VISITORS BLOCK # --------------------------- # AI Data Scraper # https://knownagents.com/agents/ai2bot-dolma User-agent: Ai2Bot-Dolma Disallow: / # AI Data Scraper # https://knownagents.com/agents/amazonbot User-agent: Amazonbot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/apifybot User-agent: ApifyBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/apifywebsitecontentcrawler User-agent: ApifyWebsiteContentCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/applebot-extended User-agent: Applebot-Extended Disallow: / # AI Data Scraper # https://knownagents.com/agents/bytespider User-agent: Bytespider Disallow: / # AI Data Scraper # https://knownagents.com/agents/ccbot User-agent: CCBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/chatglm-spider User-agent: ChatGLM-Spider Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/claude-web User-agent: Claude-Web Disallow: / # AI Data Scraper # https://knownagents.com/agents/claudebot User-agent: ClaudeBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/cloudvertexbot User-agent: CloudVertexBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/cohere-ai User-agent: cohere-ai Disallow: / # AI Data Scraper # https://knownagents.com/agents/cohere-training-data-crawler User-agent: cohere-training-data-crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/cotoyogi User-agent: Cotoyogi Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/crawl4ai User-agent: Crawl4AI Disallow: / # AI Data Scraper # https://knownagents.com/agents/datenbank-crawler User-agent: Datenbank Crawler Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/deepseekbot User-agent: DeepSeekBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/diffbot User-agent: Diffbot Disallow: / # AI Data Scraper # https://knownagents.com/agents/facebookbot User-agent: FacebookBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/firecrawlagent User-agent: FirecrawlAgent Disallow: / # AI Data Scraper # https://knownagents.com/agents/google-extended User-agent: Google-Extended Disallow: / # AI Data Scraper # https://knownagents.com/agents/googleother User-agent: GoogleOther Disallow: / # AI Data Scraper # https://knownagents.com/agents/gptbot User-agent: GPTBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskbot User-agent: iAskBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskspider User-agent: iaskspider Disallow: / # AI Data Scraper # https://knownagents.com/agents/icc-crawler User-agent: ICC-Crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/imagespider User-agent: imageSpider Disallow: / # AI Data Scraper # https://knownagents.com/agents/kangaroo-bot User-agent: Kangaroo Bot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/kunatocrawler User-agent: KunatoCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/laion-huggingface-processor User-agent: laion-huggingface-processor Disallow: / # AI Data Scraper # https://knownagents.com/agents/lcc User-agent: LCC Disallow: / # AI Data Scraper # https://knownagents.com/agents/meta-externalagent User-agent: meta-externalagent Disallow: / # AI Data Scraper # https://knownagents.com/agents/netestate-imprint-crawler User-agent: netEstate Imprint Crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/omgili User-agent: omgili Disallow: / # AI Data Scraper # https://knownagents.com/agents/pangubot User-agent: PanguBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/sbintuitionsbot User-agent: SBIntuitionsBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/spider User-agent: Spider Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/tavilybot User-agent: TavilyBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/terracotta User-agent: TerraCotta Disallow: / # AI Data Scraper # https://knownagents.com/agents/timpibot User-agent: Timpibot Disallow: / # AI Data Scraper # https://knownagents.com/agents/velenpublicwebcrawler User-agent: VelenPublicWebCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/webzio-extended User-agent: webzio-extended Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/wrtnbot User-agent: WRTNBot Disallow: / # --------------------------- # END DARK VISITORS BLOCK # START KNOWN AGENTS BLOCK # --------------------------- # AI Data Scraper # https://knownagents.com/agents/ai2bot-dolma User-agent: Ai2Bot-Dolma Disallow: / # AI Data Scraper # https://knownagents.com/agents/amazonbot User-agent: Amazonbot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / # AI Data Scraper # https://knownagents.com/agents/applebot-extended User-agent: Applebot-Extended Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/aranet-searchbot User-agent: Aranet-SearchBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/bytespider User-agent: Bytespider Disallow: / # AI Data Scraper # https://knownagents.com/agents/ccbot User-agent: CCBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/chatglm-spider User-agent: ChatGLM-Spider Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/claude-web User-agent: Claude-Web Disallow: / # AI Data Scraper # https://knownagents.com/agents/claudebot User-agent: ClaudeBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/cloudvertexbot User-agent: CloudVertexBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/cohere-ai User-agent: cohere-ai Disallow: / # AI Data Scraper # https://knownagents.com/agents/cohere-training-data-crawler User-agent: cohere-training-data-crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/cotoyogi User-agent: Cotoyogi Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/crawl4ai User-agent: Crawl4AI Disallow: / # AI Data Scraper # https://knownagents.com/agents/datenbank-crawler User-agent: Datenbank Crawler Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/deepseekbot User-agent: DeepSeekBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/facebookbot User-agent: FacebookBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/google-extended User-agent: Google-Extended Disallow: / # AI Data Scraper # https://knownagents.com/agents/googleother User-agent: GoogleOther Disallow: / # AI Data Scraper # https://knownagents.com/agents/gptbot User-agent: GPTBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskbot User-agent: iAskBot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/iaskspider User-agent: iaskspider Disallow: / # AI Data Scraper # https://knownagents.com/agents/icc-crawler User-agent: ICC-Crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/imagespider User-agent: imageSpider Disallow: / # AI Data Scraper # https://knownagents.com/agents/kangaroo-bot User-agent: Kangaroo Bot Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/kunatocrawler User-agent: KunatoCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/laion-huggingface-processor User-agent: laion-huggingface-processor Disallow: / # AI Data Scraper # https://knownagents.com/agents/lcc User-agent: LCC Disallow: / # AI Data Scraper # https://knownagents.com/agents/meta-externalagent User-agent: meta-externalagent Disallow: / # AI Data Scraper # https://knownagents.com/agents/netestate-imprint-crawler User-agent: netEstate Imprint Crawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/omgili User-agent: omgili Disallow: / # AI Data Scraper # https://knownagents.com/agents/pangubot User-agent: PanguBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/sbintuitionsbot User-agent: SBIntuitionsBot Disallow: / # AI Data Scraper # https://knownagents.com/agents/spider User-agent: Spider Disallow: / # AI Data Scraper # https://knownagents.com/agents/timpibot User-agent: Timpibot Disallow: / # AI Data Scraper # https://knownagents.com/agents/velenpublicwebcrawler User-agent: VelenPublicWebCrawler Disallow: / # AI Data Scraper # https://knownagents.com/agents/webzio-extended User-agent: webzio-extended Disallow: / # Undocumented AI Agent # https://knownagents.com/agents/wrtnbot User-agent: WRTNBot Disallow: / # --------------------------- # END KNOWN AGENTS BLOCK