# Block specific AI + archival crawlers
      User-agent: AddSearchBot
      Disallow: /
User-agent: AI2Bot
      Disallow: /
User-agent: AI2Bot-DeepResearchEval
      Disallow: /
User-agent: Ai2Bot-Dolma
      Disallow: /
User-agent: aiHitBot
      Disallow: /
User-agent: amazon-kendra
      Disallow: /
User-agent: AmazonBuyForMe
      Disallow: /
User-agent: Andibot
      Disallow: /
User-agent: Anomura
      Disallow: /
User-agent: anthropic-ai
      Disallow: /
User-agent: Applebot-Extended
      Disallow: /
User-agent: bedrockbot
      Disallow: /
User-agent: bigsur.ai
      Disallow: /
User-agent: Bravebot
      Disallow: /
User-agent: BuddyBot
      Disallow: /
User-agent: Bytespider
      Disallow: /
User-agent: CCBot
      Disallow: /
User-agent: Channel3Bot
      Disallow: /
User-agent: ChatGLM-Spider
      Disallow: /
User-agent: ChatGPT Agent
      Disallow: /
User-agent: ChatGPT-User
      Disallow: /
User-agent: Claude-SearchBot
      Disallow: /
User-agent: Claude-User
      Disallow: /
User-agent: Claude-Web
      Disallow: /
User-agent: ClaudeBot
      Disallow: /
User-agent: Cloudflare-AutoRAG
      Disallow: /
User-agent: CloudVertexBot
      Disallow: /
User-agent: cohere-ai
      Disallow: /
User-agent: cohere-training-data-crawler
      Disallow: /
User-agent: Cotoyogi
      Disallow: /
User-agent: Crawl4AI
      Disallow: /
User-agent: Crawlspace
      Disallow: /
User-agent: Datenbank Crawler
      Disallow: /
User-agent: DeepSeekBot
      Disallow: /
User-agent: Devin
      Disallow: /
User-agent: Diffbot
      Disallow: /
User-agent: DuckAssistBot
      Disallow: /
User-agent: FirecrawlAgent
      Disallow: /
User-agent: FriendlyCrawler
      Disallow: /
User-agent: Gemini-Deep-Research
      Disallow: /
User-agent: Google-CloudVertexBot
      Disallow: /
User-agent: Google-Extended
      Disallow: /
User-agent: Google-NotebookLM
      Disallow: /
User-agent: GoogleAgent-Mariner
      Disallow: /
User-agent: GPTBot
      Disallow: /
User-agent: iAskBot
      Disallow: /
User-agent: iaskspider
      Disallow: /
User-agent: IbouBot
      Disallow: /
User-agent: imageSpider
      Disallow: /
User-agent: ImagesiftBot
      Disallow: /
User-agent: img2dataset
      Disallow: /
User-agent: Kangaroo Bot
      Disallow: /
User-agent: KlaviyoAIBot
      Disallow: /
User-agent: KunatoCrawler
      Disallow: /
User-agent: laion-huggingface-processor
      Disallow: /
User-agent: LAIONDownloader
      Disallow: /
User-agent: LCC
      Disallow: /
User-agent: LinerBot
      Disallow: /
User-agent: Linguee Bot
      Disallow: /
User-agent: LinkupBot
      Disallow: /
User-agent: Manus-User
      Disallow: /
User-agent: meta-externalagent
      Disallow: /
User-agent: Meta-ExternalAgent
      Disallow: /
User-agent: meta-externalfetcher
      Disallow: /
User-agent: Meta-ExternalFetcher
      Disallow: /
User-agent: meta-webindexer
      Disallow: /
User-agent: MistralAI-User
      Disallow: /
User-agent: MistralAI-User/1.0
      Disallow: /
User-agent: MyCentralAIScraperBot
      Disallow: /
User-agent: NotebookLM
      Disallow: /
User-agent: NovaAct
      Disallow: /
User-agent: OAI-SearchBot
      Disallow: /
User-agent: omgili
      Disallow: /
User-agent: omgilibot
      Disallow: /
User-agent: OpenAI
      Disallow: /
User-agent: Operator
      Disallow: /
User-agent: PanguBot
      Disallow: /
User-agent: Panscient
      Disallow: /
User-agent: panscient.com
      Disallow: /
User-agent: Perplexity-User
      Disallow: /
User-agent: PerplexityBot
      Disallow: /
User-agent: PhindBot
      Disallow: /
User-agent: Poggio-Citations
      Disallow: /
User-agent: Poseidon Research Crawler
      Disallow: /
User-agent: QuillBot
      Disallow: /
User-agent: quillbot.com
      Disallow: /
User-agent: SBIntuitionsBot
      Disallow: /
User-agent: ShapBot
      Disallow: /
User-agent: TavilyBot
      Disallow: /
User-agent: TerraCotta
      Disallow: /
User-agent: Thinkbot
      Disallow: /
User-agent: TwinAgent
      Disallow: /
User-agent: WARDBot
      Disallow: /
User-agent: webzio-extended
      Disallow: /
User-agent: Webzio-Extended
      Disallow: /
User-agent: WRTNBot
      Disallow: /
User-agent: YaK
      Disallow: /
User-agent: YouBot
      Disallow: /
User-agent: ZanistaBot
      Disallow: /
User-agent: archive.org_bot
      Disallow: /
User-agent: ia_archiver
      Disallow: /
User-agent: ia_archiver-web.archive.org
      Disallow: /
User-agent: special_archiver
      Disallow: /

# Allow all other crawlers  
User-agent: *
Allow: /

# Host
Host: www.semissourian.com

# Sitemaps
Sitemap: https://www.semissourian.com/sitemap.xml