Sitemap: https://www.calferu.com/sitemap.xml

# --- Bots no deseados: bloqueo total ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MauiBot
Disallow: /

# --- Crawler de IA de Meta: bloqueo total (no afecta a las previews al
# --- compartir en Facebook/Instagram, que las hace facebookexternalhit)
User-agent: meta-externalagent
Disallow: /

User-agent: meta-webindexer
Disallow: /

# --- Buscadores legitimos (Google, Bing...) ---
User-agent: *
Disallow: /*?*order=
Disallow: /*?*search=
Disallow: /*?*attrib=
Disallow: /*?*tags=
Disallow: /*?*ppg=
Disallow: /shop/cart
Disallow: /shop/checkout
Disallow: /shop/payment
Disallow: /shop/confirmation
Disallow: /web/login
Disallow: /web/signup
Disallow: /my/
Disallow: /website/info
Disallow: /page/product_brands

# --- Combinaciones de etiquetas de blog: URLs infinitas que solo devuelven
# --- 302 y se comian el presupuesto de rastreo
Disallow: /blog/tag/*,*
Disallow: /*/blog/tag/*,*

# --- Parametros de filtros y paginacion: la misma pagina con otra URL
# --- (2.300 rastreos en 14 dias solo en las fechas del blog)
Disallow: /*?*date_begin=
Disallow: /*?*date_end=
Disallow: /*?*min_price=
Disallow: /*?*max_price=
Disallow: /*?*page=
Disallow: /*?*category=