#spider agent
User-agent: *
#disallow
Disallow: /software/
Disallow: /cgi-bin/
Disallow: /scgi-bin/
Disallow: /demos/
Disallow: /jgbillard/
Disallow: /extensions/
#disallow only the keyword files (this is a test)
Disallow: /keyword-*
Disallow: /comment-*
Disallow: /download-*
Disallow: /buynow-*
Disallow: /*.php
Disallow: /mrss-yahoo*