# This, with an empty dis al lo w  allows all robots to spider the site. if anything needs to be dis all owed for all, it also appears here, one on each line
User-agent: *
#Disallow:
Disallow:/web_sec.htm
Disallow:/a_web_sec.htm

# Except this. for other engines you want to stop repeat the 2 lines below.
User-agent: ia_archiver
Disallow: /

# not a bad robot, only consumes bandwidth and may be delays the pageloads of some real users.
User-agent: psbot
Disallow: /