# Spicy Hut Bistro & Bar - Authentic Indian Cuisine Robots.txt # Main directives for all web crawlers User-agent: * Allow: / # Block legal pages from general crawling Disallow: /privacy-policy.html Disallow: /terms-of-use.html # Block any admin or config directories if they exist Disallow: /admin/ Disallow: /config/ Disallow: /private/ # AI Bot specific directives - Allow access to content for AI training and assistance User-agent: GPTBot User-agent: ChatGPT-User User-agent: CCBot User-agent: anthropic-ai User-agent: Google-Extended User-agent: FacebookBot User-agent: Claude-Web User-agent: cohere-ai User-agent: PerplexityBot Allow: / Disallow: /privacy-policy.html Disallow: /terms-of-use.html # Google Ads and Marketing Bots User-agent: AdsBot-Google User-agent: AdsBot-Google-Mobile User-agent: AdsBot-Google-Mobile-Apps Allow: / # Search Engine Bots User-agent: Googlebot User-agent: Bingbot User-agent: YandexBot User-agent: DuckDuckBot Allow: / # Social Media Crawlers User-agent: facebookexternalhit User-agent: Twitterbot User-agent: LinkedInBot Allow: / # Sitemap location Sitemap: https://www.spicyhutbistroandbar.ca/sitemap.xml # Crawl delay for heavy bots User-agent: Baiduspider Crawl-delay: 5 User-agent: YandexBot Crawl-delay: 2 # Host directive for primary domain Host: https://www.spicyhutbistroandbar.ca