# $Id: robots.txt,v 1.1 1997/10/27 09:00:00 fielding Exp $
# robots.txt for http://www.colby.edu/

# This version of robots.txt allows indexing of commonspot directories. It is
# copied to /web/prod/colby every evening at 10PM.

#--- this section is for the Colby search engine:
User-agent: ColbySpider
Disallow: /cgi-bin/
Disallow: /info.tech/stats/

User-agent: Colby-gsa-crawler
Disallow: /cgi-bin/
Disallow: /info.tech/stats/
Disallow: /echo/

#--- this section is for all other search engines:
User-agent: *                   # All other spiders should avoid 
Disallow: /commonspot/          # Commonspot installation
Disallow: /cgi-bin/             #     Script files
Disallow: /classof/2001.summer/
Disallow: /colby.mag/issues/2004/spring/alumni/		#	Spring 2004 Magazine | Alumni Section
Disallow: /college/             #     internal
Disallow: /cpa/             #     unfinished club
Disallow: /faculty/
Disallow: /focus/             #     nonextant club
Disallow: /fusetalk/            # fusetalk forums
Disallow: /echo/
Disallow: /i/
Disallow: /info.tech/stats/
Disallow: /music/
Disallow: /security/             #     unfinished dept.