User-agent: *
Disallow: /typo3/
Disallow: /typo3_src/
Allow: /typo3/sysext/frontend/Resources/Public/*
Sitemap: https://institut-agro-dijon.fr//sitemap.xml

# Crawl Delay for AI bots
User-agent: AddSearchBot
Crawl-delay: 30
User-agent: AI2Bot
Crawl-delay: 30
User-agent: Ai2Bot-Dolma
Crawl-delay: 30
User-agent: aiHitBot
Crawl-delay: 30
User-agent: Amazonbot
Crawl-delay: 30
User-agent: Andibot
Crawl-delay: 30
User-agent: anthropic-ai
Crawl-delay: 30
User-agent: Applebot
Crawl-delay: 30
User-agent: Applebot-Extended
Crawl-delay: 30
User-agent: Awario
Crawl-delay: 30
User-agent: bedrockbot
Crawl-delay: 30
User-agent: bigsur.ai
Crawl-delay: 30
User-agent: Brightbot 1.0
Crawl-delay: 30
User-agent: Bytespider
Crawl-delay: 30
User-agent: CCBot
Crawl-delay: 30
User-agent: ChatGPT Agent
Crawl-delay: 30
User-agent: Claude-SearchBot
Crawl-delay: 30
User-agent: Claude-Web
Crawl-delay: 30
User-agent: ClaudeBot
Crawl-delay: 30
User-agent: CloudVertexBot
Crawl-delay: 30
User-agent: cohere-ai
Crawl-delay: 30
User-agent: cohere-training-data-crawler
Crawl-delay: 30
User-agent: Cotoyogi
Crawl-delay: 30
User-agent: Crawlspace
Crawl-delay: 30
User-agent: Datenbank Crawler
Crawl-delay: 30
User-agent: Devin
Crawl-delay: 30
User-agent: Diffbot
Crawl-delay: 30
User-agent: DuckAssistBot
Crawl-delay: 30
User-agent: Echobot Bot
Crawl-delay: 30
User-agent: EchoboxBot
Crawl-delay: 30
User-agent: FacebookBot
Crawl-delay: 30
User-agent: facebookexternalhit
Crawl-delay: 30
User-agent: Factset_spyderbot
Crawl-delay: 30
User-agent: FirecrawlAgent
Crawl-delay: 30
User-agent: FriendlyCrawler
Crawl-delay: 30
User-agent: Gemini-Deep-Research
Crawl-delay: 30
User-agent: Google-CloudVertexBot
Crawl-delay: 30
User-agent: Google-Extended
Crawl-delay: 30
User-agent: Google-Firebase
Crawl-delay: 30
User-agent: GoogleAgent-Mariner
Crawl-delay: 30
User-agent: GoogleOther
Crawl-delay: 30
User-agent: GoogleOther-Image
Crawl-delay: 30
User-agent: GoogleOther-Video
Crawl-delay: 30
User-agent: GPTBot
Crawl-delay: 30
User-agent: iaskspider/2.0
Crawl-delay: 30
User-agent: ICC-Crawler
Crawl-delay: 30
User-agent: ImagesiftBot
Crawl-delay: 30
User-agent: img2dataset
Crawl-delay: 30
User-agent: ISSCyberRiskCrawler
Crawl-delay: 30
User-agent: Kangaroo Bot
Crawl-delay: 30
User-agent: LinerBot
Crawl-delay: 30
User-agent: meta-externalagent
Crawl-delay: 30
User-agent: Meta-ExternalAgent
Crawl-delay: 30
User-agent: meta-externalfetcher
Crawl-delay: 30
User-agent: Meta-ExternalFetcher
Crawl-delay: 30
User-agent: MyCentralAIScraperBot
Crawl-delay: 30
User-agent: netEstate Imprint Crawler
Crawl-delay: 30
User-agent: NovaAct
Crawl-delay: 30
User-agent: OAI-SearchBot
Crawl-delay: 30
User-agent: omgili
Crawl-delay: 30
User-agent: omgilibot
Crawl-delay: 30
User-agent: OpenAI
Crawl-delay: 30
User-agent: Operator
Crawl-delay: 30
User-agent: PanguBot
Crawl-delay: 30
User-agent: Panscient
Crawl-delay: 30
User-agent: panscient.com
Crawl-delay: 30
User-agent: PerplexityBot
Crawl-delay: 30
User-agent: PetalBot
Crawl-delay: 30
User-agent: PhindBot
Crawl-delay: 30
User-agent: Poseidon Research Crawler
Crawl-delay: 30
User-agent: QualifiedBot
Crawl-delay: 30
User-agent: QuillBot
Crawl-delay: 30
User-agent: quillbot.com
Crawl-delay: 30
User-agent: SBIntuitionsBot
Crawl-delay: 30
User-agent: Scrapy
Crawl-delay: 30
User-agent: SemrushBot-OCOB
Crawl-delay: 30
User-agent: SemrushBot-SWA
Crawl-delay: 30
User-agent: ShapBot
Crawl-delay: 30
User-agent: Sidetrade indexer bot
Crawl-delay: 30
User-agent: Thinkbot
Crawl-delay: 30
User-agent: TikTokSpider
Crawl-delay: 30
User-agent: Timpibot
Crawl-delay: 30
User-agent: VelenPublicWebCrawler
Crawl-delay: 30
User-agent: WARDBot
Crawl-delay: 30
User-agent: Webzio-Extended
Crawl-delay: 30
User-agent: wpbot
Crawl-delay: 30
User-agent: YaK
Crawl-delay: 30
User-agent: YandexAdditional
Crawl-delay: 30
User-agent: YandexAdditionalBot
Crawl-delay: 30
User-agent: YouBot