# ===========================================================
# robots.txt — PRO SECURE EDITION (Deny-by-Default)          
# Version: 2025-11                                           
# Purpose: Maximum protection + allowlist-only crawling      
# ===========================================================
                                                             
# ===========================================================
# ======================= SITEMAP ===========================
# ===========================================================
Sitemap: https://www.immoges.es/sitemap.xml
                                                             
# ===========================================================
# ============= MAIN SEARCH ENGINES (ALLOWED) ===============
# ===========================================================
User-agent: Googlebot                                        
User-agent: Googlebot-Image                                  
User-agent: Googlebot-Mobile                                 
User-agent: Google-Extended                                  
User-agent: GoogleOther                                      
User-agent: Bingbot                                          
User-agent: Slurp                                            
User-agent: DuckDuckBot                                      
User-agent: Applebot                                         
User-agent: Applebot-Image                                   
User-agent: LinkedInBot                                      
Allow: /                                                     
                                                             
# ===========================================================
# ========== AI & MACHINE LEARNING CRAWLERS (ALLOWED) =======
# ===========================================================
User-agent: GPTBot                                           
User-agent: OAI-SearchBot                                    
User-agent: ChatGPT-User                                     
User-agent: DeepSeek                                         
User-agent: anthropic-ai                                     
User-agent: claude-web                                       
User-agent: claudebot                                        
User-agent: Claude-SearchBot                                 
User-agent: Claude-User                                      
User-agent: CohereAI                                         
User-agent: AI2Bot                                           
User-agent: DuckAssistBot                                    
User-agent: CCBot                                            
User-agent: xAI-Bot                                          
User-agent: YouBot                                           
User-agent: MistralBot                                       
User-agent: Diffbot                                          
User-agent: PhindBot                                         
User-agent: KagiBot                                          
User-agent: MetaBot                                          
User-agent: ForefrontBot                                     
User-agent: AndiBot                                          
User-agent: SerperBot                                        
User-agent: GrokBot                                          
User-agent: GrokBot/1.0                                      
User-agent: xAI-Grok                                         
User-agent: xAI-Grok/1.0                                     
User-agent: Grok-DeepSearch                                  
User-agent: Grok-DeepSearch/1.0                              
User-agent: Grok                                             
User-agent: xAI                                              
User-agent: x.AI                                             
Allow: /                                                     
                                                             
# ===========================================================
# =========== SOCIAL & MESSAGING PREVIEWS (ALLOWED) =========
# ===========================================================
User-agent: Twitterbot                                       
User-agent: FacebookBot                                      
User-agent: facebookexternalhit                              
User-agent: facebookcatalog                                  
User-agent: TelegramBot                                      
User-agent: Pinterestbot                                     
User-agent: Slackbot-LinkExpanding                           
User-agent: Discordbot                                       
User-agent: WhatsApp                                         
User-agent: ByteAI                                           
Allow: /                                                     
                                                             
# ===========================================================
# ============= DEFAULT POLICY (BLOCK EVERYTHING) ===========
# ===========================================================
User-agent: *                                                
Disallow: /                                                  
# ===========================================================
# ============= ADDITIONAL BLOCKING FOR INDEXING ============
# ===========================================================
Disallow: /regards
Disallow: /includes/youtube-publication-section.php