#=========================================================== # All crawlers #=========================================================== User-Agent: * Allow: / # URLs from public folders --------------------------------- Disallow: /static/ Disallow: /var/ # Non-localized URLs from routing -------------------------- Disallow: /admin/ Disallow: /api/ Disallow: /translations # Localized URLs from routing ------------------------------ # Login pages should be crawlable/indexable Allow: /de/schnellbestellung/login$ Allow: /en/quickorder/login$ Allow: /fr/commande-rapide/login$ Allow: /it/ordine-rapido/login$ Allow: /es/pedido-rapido/login$ Allow: /nl/snelle-bestelling/login$ Disallow: /*/checkout-free-samples/ Disallow: /*/quickorder/ Disallow: /*/checkout/ Allow: /*/checkout/sample-selection Disallow: /*/pdf/generate/ Disallow: /*/user/login/ Disallow: /*/*/login # Non-localized URLs from CMS ------------------------------ # Depending on the user agent, paths may be case-sensitive. # Those which can be defined in the CMS, should be set up # for different cases... Disallow: /PDF-Templates/ Disallow: /pdf-templates/ Disallow: /Templates/ Disallow: /templates/ # URLs from CMS (with arbitrary locale) -------------------- # See the comment on case-sensitivity above. Disallow: /*/Email/ Disallow: /*/email/ Disallow: /*/Links/ Disallow: /*/links/ Disallow: /*/PDF-Templates/ Disallow: /*/pdf-templates/ Disallow: /*/Snippets/ Disallow: /*/snippets/ Disallow: /*/Templates/ Disallow: /*/templates/ Disallow: /*/Vorlagen/ Disallow: /*/vorlagen/ # Above, we blocked any documents inside a Links/links # folder. If there is a dot in the path, we don't want to # block the URL. This is intended to allow indexing of # assets but will also work for documents the path of which # contains a dot, e.g., "/de/links/.hidden/page". Allow: /*/Links/*.* Allow: /*/links/*.* # Localized URLs from routing (de) ------------------------- Disallow: /de/bestellung-gratis-probe/ Disallow: /de/schnellbestellung/ Disallow: /de/bestellung/ Allow: /de/bestellung/proben-auswahl # Localized URLs from CMS (en) ----------------------------- Disallow: /en/EN%20Snippets/ # Localized URLs from routing (es) ------------------------- Disallow: /es/pedido-muestra-gratis/ Disallow: /es/pedido-rapido/ Disallow: /es/pedido/ Allow: /es/pedido/selecci%C3%B3n-muestras # Localized URLs from routing (fr) ------------------------- Disallow: /fr/commande-%C3%A9chantillon-gratuit/ Disallow: /fr/commande-rapide/ Disallow: /fr/commande/ Allow: /fr/commande/s%C3%A9lection-%C3%A9chantillon # Localized URLs from routing (it) ------------------------- Disallow: /it/ordine-campioni-gratuiti/ Disallow: /it/ordine-rapido/ Disallow: /it/ordine/ Allow: /it/ordine/selezione-campioni # Localized URLs from routing (nl) ------------------------- Disallow: /nl/bestelling-gratis-proef/ Disallow: /nl/snelle-bestelling/ Disallow: /nl/bestelling/ Allow: /nl/bestelling/monsterselectie #=========================================================== # Special crawlers #=========================================================== User-Agent: AdsBot-Google Allow: / #=========================================================== # Sitemap (not related to any User-Agent) #=========================================================== Sitemap: https://www.crb-gmbh.com/sitemap.xml