# For all robots
User-agent: *
# Block access to specific groups of pages
Disallow: */checkout
Disallow: */account
Disallow: */stocklocatorpage
Disallow: */stocknotification
Disallow: */wishlist
Disallow: */api/v1/*
Disallow: */minibasket
Disallow: */suggest
Disallow: *q=*
Disallow: *sort=*
# Allow Googlebot & Google Images
User-agent: Googlebot
Disallow:
Disallow: */checkout
Disallow: */account
Disallow: */stocklocatorpage
Disallow: */stocknotification
Disallow: */wishlist
Disallow: */api/v1/*
Disallow: */minibasket
Disallow: */suggest
Disallow: *q=*
Disallow: *sort=*
User-agent: Googlebot-Image
Disallow:
Disallow: */checkout
Disallow: */account
Disallow: */stocklocatorpage
Disallow: */stocknotification
Disallow: */wishlist
Disallow: */api/v1/*
Disallow: */minibasket
Disallow: */suggest
Disallow: *q=*
Disallow: *sort=*
# Allow search crawlers to discover the sitemap
Sitemap: https://www.g-star.com/sitemap/index.xml
# Block CazoodleBot as it does not present correct accept content headers
User-agent: CazoodleBot
Disallow: /
# Block dotbot as it cannot parse base urls properly
User-agent: dotbot/1.0
Disallow: /
# Block Gigabot
User-agent: Gigabot
Disallow: /
#fail-safes, allow the ads-bot checkers everything.
#(Disallow: means, allow everything)
User-agent: adsbot-google
Disallow:
User-agent: Mediapartners-Google
Disallow:
#Bing Ads
User-agent: AdIdxBot
Disallow:
# Public key for AMP api
Disallow: /.well-known/amphtml/apikey.pub