# dermHA robots.txt # Last updated: 2026-08-05 # Canonical host: https://dermhaskincare.com # ---------- Major search engines ---------- User-agent: Googlebot Allow: / Disallow: /admin Disallow: /admin/ Disallow: /cart Disallow: /checkout Disallow: /order-confirmation Disallow: /reset-password Disallow: /.lovable/ Disallow: /slider-audit User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Bingbot Allow: / Crawl-delay: 1 Disallow: /admin Disallow: /admin/ Disallow: /cart Disallow: /checkout Disallow: /order-confirmation Disallow: /reset-password Disallow: /.lovable/ User-agent: DuckDuckBot Allow: / User-agent: YandexBot Allow: / Crawl-delay: 1 User-agent: Baiduspider Allow: / # ---------- Social previews ---------- User-agent: Twitterbot Allow: / User-agent: facebookexternalhit Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterestbot Allow: / # ---------- AI search crawlers (explicitly allowed, full discovery) ---------- User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: Google-Extended Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: Amazonbot Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / User-agent: cohere-ai Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / User-agent: YouBot Allow: / User-agent: CCBot Allow: / User-agent: Bytespider Allow: / User-agent: Diffbot Allow: / User-agent: ImagesiftBot Allow: / User-agent: Omgilibot Allow: / User-agent: MistralAI-User Allow: / User-agent: DuckAssistBot Allow: / # ---------- Default ---------- # Polite crawl-delay for generic bots that honour it. AI discovery agents # and Googlebot are handled above without throttling so they can index # llms.txt, ai.txt, and the knowledge-base at full throughput. User-agent: * Allow: / Crawl-delay: 1 Disallow: /admin Disallow: /admin/ Disallow: /cart Disallow: /checkout Disallow: /order-confirmation Disallow: /reset-password Disallow: /.lovable/ Disallow: /slider-audit # ---------- Canonical host ---------- # Note: "Host:" is a Yandex-only directive and is treated as an unknown # directive by Google. Canonical host is enforced via . # Host: https://dermhaskincare.com # ---------- Sitemap index ---------- Sitemap: https://dermhaskincare.com/sitemap.xml # ---------- AI discovery files ---------- # These are not robots.txt directives; they are discoverable at: # - https://dermhaskincare.com/llms.txt # - https://dermhaskincare.com/llms-full.txt # - https://dermhaskincare.com/ai.txt # - https://dermhaskincare.com/humans.txt # - https://dermhaskincare.com/knowledge-base # ---------- IndexNow key ---------- # https://dermhaskincare.com/d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6.txt