User-agent: * #Crawl-delay: 10 # #Request-rate: 1/5 # maximum rate is one page every 5 seconds #Visit-time: 0600-0845 # only visit between 6:00 AM and 8:45 AM UT (GMT) #Allow: /something/ Disallow: /compatibility/ Disallow: /images/ Disallow: /login/ Disallow: /common_images/ #Disallow: /*ItemNumber= # excludes page scripts with specified query string from crawling Disallow: /handsetview.asp$ # excludes page scripts without a query string from crawling Disallow: /itemview.asp$