# robots.txt for http://www.sil.si.edu/


User-agent: *

Disallow: /about/
Disallow: /ad/
Disallow: /Amnesty/
Disallow: /AnnualReport/
Disallow: /BCAProject/
Disallow: /BHL/
Disallow: /Branches/
Disallow: /DigitalCollections/Anthropology/
Disallow: /DigitalCollections/BCA/
Disallow: /DigitalCollections/bca/
Disallow: /DigitalCollections/BHLCollections/
Disallow: /digitalcollections/bae/
Disallow: /DigitalCollections/BAE/
Disallow: /DigitalCollections/BHLCollections/
Disallow: /DigitalCollections/fulltext/
Disallow: /DigitalCollections/HistoryCultureCollections/
Disallow: /digitalcollections/HistoryCultureCollections/
Disallow: /DigitalCollections/herbals/ #removing per request 5/2016
Disallow: /DigitalCollections/Herbals/
Disallow: /DigitalCollections/HST/Besson/
Disallow: /DigitalCollections/HST/Brahe/
Disallow: /DigitalCollections/HST/Huygens/
Disallow: /DigitalCollections/HST/Landauer/
Disallow: /DigitalCollections/hst/Landauer/
Disallow: /DigitalCollections/hst/Brahe/
Disallow: /DigitalCollections/hst/Besson/
Disallow: /digitalcollections/hst/besson/
Disallow: /DigitalCollections/hst/Huygens/
Disallow: /DigitalCollections/hst/huygens/
Disallow: /DigitalCollections/NHRareBooks/Hedwig/
Disallow: /DigitalCollections/NHRareBooks/Larreategui/
Disallow: /DigitalCollections/NHRareBooks/Martinet/
Disallow: /DigitalCollections/NHRareBooks/Spalowsky/
Disallow: /DigitalCollections/NHRareBooks/Spix/
Disallow: /DigitalCollections/NHRareBooks/Sturm/
Disallow: /digitalcollections/nhrarebooks/rothschild/
Disallow: /DigitalCollections/Trade-Literature/NYCoachMakersMagazine/
Disallow: /digitalcollections/trade-literature/samples/
Disallow: /digitalcollections/trade-literature/NYCoachMakersMagazine/
Disallow: /eresources/ # added 4/2015 many pages have 303s but not all
Disallow: /flash/
Disallow: /forms/
Disallow: /Galaxy/
Disallow: /Giving/
Disallow: /Graphics/
Disallow: /libqual/
Disallow: /libraries/ #added 3/2016
Disallow: /Newsletter/
Disallow: /onlineopinionF3r/
Disallow: /Permissions/ # this has a redirect on it
Disallow: /PDF/
Disallow: /ProjectAccess/
Disallow: /ResearchIntern/
Disallow: /research/ # added 4/2015 many pages have 303s but not all
Disallow: /Resources/
Disallow: /RSVP/ # this can probably just be deleted
Disallow: /RSS/
Disallow: /Search/ # check references - not for main site. seems to use old search engine
Disallow: /seasonal/
Disallow: /SILPublications/Translation/
Disallow: /SILGalaxy/ #test pages I think
Disallow: /SmithsonianContributions/ # added 2/2015
Disallow: /smithsoniancontributions/ # added 4/2016
Disallow: /SpecialCollections/
Disallow: /Subject-Guide/
Disallow: /subject-guide/
Disallow: /TradeLiterature/
Disallow: /USExExProject/
Disallow: /voyages/ #nothing in here

#updated 20160608 deleted directories: Information-Files, ILL, Link, Link_Checking,emailtoc,forms,staff
#all deleted directories are available in S:/DPI/WSD/SILWEB

User-agent: Smithsonian-public-search

# try to not index images to reduce number of pages indexed TEST
Disallow: /*.jpg
Disallow: /*.gif
# by directory
Disallow: /404/
Disallow: /Cataloging/
# the following are in the Digital Collections dir
# in BAE 
Disallow: /1000/   
Disallow: /600/
# in BCA
Disallow: /bca_00_00_00/
Disallow: /bca_01_00_00/
Disallow: /bca_02_00_00/
Disallow: /bca_03_01_00/
Disallow: /bca_03_02_00/
Disallow: /bca_03_03_00/
Disallow: /bca_03_04_00/
Disallow: /bca_04_00_00/
Disallow: /bca_05_00_00/
Disallow: /bca_06_00_00/
Disallow: /bca_07_00_00/
Disallow: /bca_08_00_00/
Disallow: /bca_09_00_00/
Disallow: /bca_10_00_00/
Disallow: /bca_11_00_00/
Disallow: /bca_12_01_01/
Disallow: /bca_12_01_02/
Disallow: /bca_12_02_01/
Disallow: /bca_12_02_02/
Disallow: /bca_12_03_01/
Disallow: /bca_12_03_02/
Disallow: /bca_12_04_01/
Disallow: /bca_12_04_02/
Disallow: /bca_12_04_03/
Disallow: /bca_12_04_04/
Disallow: /bca_12_04_05/
Disallow: /bca_12_04_06/
Disallow: /bca_12_04_07/
Disallow: /bca_12_05_00/
Disallow: /bca_12_06_01/
Disallow: /bca_12_06_01s/
Disallow: /bca_12_06_02/
Disallow: /bca_12_07_00/
Disallow: /bca_13_01_00/
Disallow: /bca_13_02_00/
Disallow: /bca_13_03_00/
Disallow: /bca_14_01_00/
Disallow: /bca_14_02_00/
Disallow: /bca_14_03_00/
Disallow: /bca_15_01_00/
Disallow: /bca_15_02_00/
Disallow: /bca_15_03_00/
Disallow: /bca_15_04_00/
Disallow: /bca_16_01_00/
Disallow: /bca_16_02_00/
Disallow: /bca_16_03_00/
Disallow: /bca_17_01_00/
Disallow: /bca_17_02_00/
Disallow: /bca_18_01_00/
Disallow: /bca_18_02_01/
Disallow: /bca_18_02_02/
Disallow: /bca_19_00_00/
Disallow: /bca_20_01_00/
Disallow: /bca_20_02_00/
Disallow: /bca_21_01_00/
Disallow: /bca_21_02_00/
Disallow: /bca_21_03_00/
Disallow: /bca_21_04_00/
Disallow: /bca_21_05_00/
# in bhlcollections and HSTcollections just pngs 
Disallow: /pages/ 
# in HST 
Disallow: /Cooper/PDF/
Disallow: /Lewton/high/
Disallow: /Lewton/low/
Disallow: /Scientific-Identity/pdf/
Disallow: /Incunabula/fullsize/
Disallow: /IndexAnimalium/volumes/
Allow: /IndexAnimalium/volumes/index.cfm
# NHRareBooks not sure about this dir 
Disallow: /NYCoachMakersMagazine/PDFs/
Disallow: /Trade-Literature/samples/
Disallow: /Scientific-Instruments/pdf/
Disallow: /Scientific-Instruments/files/
Disallow: /Sewing-Machines/sil/
Disallow: /Sewing-Machines/graphics/
Disallow: /Sewing-Machines/nmahtex/
Disallow: /Sewing-Machines/PDF/
Disallow: /USExEx/19-01/
Disallow: /USExEx/19-02/
Disallow: /USExEx/19-03/
Disallow: /USExEx/19-04/
Disallow: /USExEx/19-05/
Disallow: /USExEx/19-07/
Disallow: /USExEx/19-08/
Disallow: /USExEx/19-09/
Disallow: /USExEx/19-10/
Disallow: /USExEx/19-11/
Disallow: /USExEx/19-12/
Disallow: /USExEx/19-13/
Disallow: /USExEx/19-14/
Disallow: /USExEx/19-15/
Disallow: /USExEx/19-16/
Disallow: /USExEx/19-17/
Disallow: /USExEx/19-18/
Disallow: /USExEx/19-19/
Disallow: /USExEx/19-20/
Disallow: /USExEx/19-21/
Disallow: /USExEx/19-22/
Disallow: /USExEx/19-23/
Disallow: /USExEx/19-24/
Disallow: /USExEx/19-24a/
Disallow: /USExEx/19-26/
Disallow: /USExEx/19-28/
Disallow: /USExEx/19-29a/
Disallow: /USExEx/19-29b/
Disallow: /USExEx/19-30/
Disallow: /USExEx/19-31/
Disallow: /USExEx/19-34/
Disallow: /USExEx/Cultural_objects/
Disallow: /USExEx/narrative_images/
Disallow: /USExEx/objects/
Disallow: /USExEx/pdf/
Disallow: /USExEx/Scientific_images/
Disallow: /eresources/CIRLAEFlowChart_files/
Disallow: /Chasing-Venus/full-size/
Disallow: /Chasing-Venus/mp3/
Disallow: /Chasing-Venus/Henry-Statue/
Disallow: /Odyssey/school-librarians/
Disallow: /Exhibitions/Odyssey/OdyBanners/
Disallow: /Underwater-Web/raw-images/
Disallow: /Underwater-Web/sounds/
Disallow: /1926-New-Guinea/62916/
Disallow: /1926-New-Guinea/video/
Allow: /1926-New-Guinea/video/index.cfm
Disallow: /1926-New-Guinea/Photographs/
Disallow: /ImageGalaxy/verity/
Disallow: /ISD/
Disallow: /Newsletter/graphics/
Disallow: /Newsletter/PDF/
Disallow: /OnDisplay/Flight/baseimages/
Disallow: /OnDisplay/Flight/archivesimages/
Disallow: /OnDisplay/Flight/audio/
Disallow: /OnDisplay/Hirshhorn/jpgs/
Disallow: /OnDisplay/NestsandEggs/zoom/
Disallow: /OnDisplay/Ramelli/zoom/
Disallow: /OnDisplay/Shoppers/photos/
Disallow: /onlineopinionF3r/
Disallow: /ProjectAccess/

# not sure of implications of this
Disallow: /RSS/

Disallow: /Dibner-Library-Lectures/25/plates/
Disallow: /Caneva lecture-2nd version/
Disallow: /Grinnell/sources/
Disallow: /SILPublications/HMSG/
Allow: /SILPublications/HMSG/HMSG_audiotapes.htm
Disallow: /StyleSheets/
Disallow: /USExExProject/samples/
Disallow: /USExExProject/files/
Disallow: /USExExProject/conversion/

# within various directories only images 
Disallow: /images/
Disallow: /flash/
Disallow: /graphics/
Disallow: /thumbnails/
Disallow: /thumb/ #this is the thumb file for contributions
Disallow: /fullsize/
Disallow: /_vti_cnf/

Sitemap: http://www.sil.si.edu/SITEMAPS_INDEX_www_sil_si_edu.xml

# End of robots.txt file