User-agent: *

Disallow: /home/
Disallow: /service-unavailable/
Disallow: /enable-cookies/
Disallow: /testtemplate/

Disallow: *?___store

# FACET RULES - disallow any URL filter strings
Disallow: *?*bike_collection
Disallow: *?*where_you_ride
Disallow: *?*manufacturer
Disallow: *?*price
Disallow: *?*dir
Disallow: *?*limit
Disallow: *?*discipline
Disallow: *?*gender
Disallow: *?*battery_size
Disallow: *?*product_color
Disallow: *?*frame_size
Disallow: *?*headset_fitment
Disallow: *?*material
Disallow: *?*capacity
Disallow: *?*bottom_brackets
Disallow: *?*speed
Disallow: *?*teeth
Disallow: *?*chainring_teeth
Disallow: *?*bolt_circle_diameter
Disallow: *?*crank_length
Disallow: *?*chainset_ratio
Disallow: *?*clothing_size
Disallow: *?*steerer_diameter
Disallow: *?*band_width
Disallow: *?*handlebar_width
Disallow: *?*clamp_diameter
Disallow: *?*helmet_sizes
Disallow: *?*spoke_holes
Disallow: *?*inner_tube_width
Disallow: *?*valve_type
Disallow: *?*valve_length
Disallow: *?*wheel_size
Disallow: *?*cage_length
Disallow: *?*seatpost_diameter
Disallow: *?*seatpost_length
Disallow: *?*shoe_size
Disallow: *?*stem_length
Disallow: *?*terrain
Disallow: *?*tyre_width
Disallow: *?*tubeless_compatible
Disallow: *?*tyre_profile
Disallow: *?*wetsuit_sizes
Disallow: *?*type
Disallow: *?*rim_material
Disallow: *?*hub_type

Disallow: /*?___SID
Disallow: /*?SID

# LIMITING ACCESSIBLE URLS AND DEPTH
Disallow: /*/clearance/*

# BLOCK GENERAL AREAS - don't need to be crawled
Disallow: /wishlist
Disallow: /catalog
Disallow: /catalogsearch/result/?q=
Disallow: /destination
Disallow: /directory
Disallow: /merchandising
Disallow: /checkout
Disallow: /admin
Disallow: /productalert

Allow: /*.jpg
Allow: /*.gif
Allow: /*.jpeg
Allow: /*.png

# Disallow filters on brands
Allow: /brand/*/
Allow: /brand/*/*
Disallow: /brand/*/?
Disallow: /brand/*/*/?

Disallow: /request.asp

Disallow: */build/
Disallow: /bikes/bikebuilder/*

# UNWANTED BOTS - disallow

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

# Throttle crawling for Bingot
User-agent: msnbot
Crawl-delay: 1

# Throttle crawling for Baiduspider
User-agent: Baiduspider
Crawl-delay: 1

# Throttle crawling for Rogerbot
User-agent: rogerbot
Crawl-delay: 1

# SITEMAP REFERENCES

Sitemap: https://www.ribblecycles.com/au/sitemap/www.ribblecycles.com/au/images.product.sitemap.xml
Sitemap: https://www.ribblecycles.com/au/sitemap/www.ribblecycles.com/au/images.category.sitemap.xml
Sitemap: https://www.ribblecycles.com/au/sitemap/www.ribblecycles.com/au/product.sitemap.xml
Sitemap: https://www.ribblecycles.com/au/sitemap/www.ribblecycles.com/au/category.sitemap.xml
Sitemap: https://www.ribblecycles.com/au/sitemap/www.ribblecycles.com/au/brand.sitemap.xml