# robots.txt for http://www.taburestaurant.com # normal bot arguments User-agent: * Crawl-delay: 60 # Set directories & files to be Disallow/Allow # NOTE: directives for Disallow/Allow are case-sensitive! # Use $ to anchor the match to the end of a URL string # As in disallowing or allowing files of a particular extension Disallow: /ads/ Disallow: /js/ Disallow: /images/ Disallow: /about.asp Disallow: /confirm.asp Disallow: /contact.asp Disallow: /gallery.asp Disallow: /gift_cert.asp Disallow: /menu.asp Disallow: /reservations.asp # Extended Parameters # Under Proposal, not universally recognized commands Request-rate: 1/10 # maximum rate of 1 page every 10 seconds Visit-time: 0900-1400 # only visit between 9:00 AM and 2:00 PM UT (1:00-6:00AM PST) # Go away arguments to be used to keep all robots # from accessing and indexing your web-site. # Comment out all lines above and # uncomment out below lines to enable. #User-agent: * #Disallow: /