User-agent: *
Disallow: /draft/	# not official even if visible
Disallow: /Public/1.1-Update/	# obsolete data
Disallow: /Public/2.1-Update2/	# obsolete data
Disallow: /Public/3.0-Update/	# obsolete data
Disallow: /Public/2.0-Update/	# obsolete data
Disallow: /Public/2.1-Update3/	# obsolete data
Disallow: /Public/2.1-Update/	# obsolete data
Disallow: /Public/2.1-Update4/	# obsolete data
Disallow: /Public/3.0-Update1/ # obsolete data
Disallow: /Public/3.1-Update/  # obsolete data
Disallow: /Public/3.1-Update1/ # obsolete data
Disallow: /Public/3.2-Update/ # obsolete data
Disallow: /Public/4.0-Update/ # obsolete data
Disallow: /Public/4.0-Update1/ # obsolete data
Disallow: /Public/4.1.0/	# obsolete data
Disallow: /Public/5.0.0/	# obsolete data
Disallow: /Public/5.1.0/	# obsolete data
Disallow: /Public/5.2.0/	# obsolete data
Disallow: /Public/6.0.0/	# obsolete data
Disallow: /Public/6.1.0/	# obsolete data
Disallow: /Public/6.2.0/	# obsolete data
Disallow: /Public/6.3.0/	# obsolete data
Disallow: /Public/7.0.0/	# obsolete data
Disallow: /Public/8.0.0/	# obsolete data
Disallow: /Public/9.0.0/	# obsolete data
Disallow: /Public/10.0.0/	# obsolete data
Disallow: /Public/11.0.0/	# obsolete data
Disallow: /Public/12.0.0/	# obsolete data
Disallow: /fr/			# obsolete pages and charts
Disallow: /versions/Unicode2.0.0/	# obsolete version
Disallow: /versions/Unicode2.1.0/	# obsolete version
Disallow: /versions/Unicode3.0.0/	# obsolete version
Disallow: /versions/Unicode3.0.1/	# obsolete version
Disallow: /versions/Unicode3.1.0/	# obsolete version
Disallow: /versions/Unicode3.1.1/	# obsolete version
Disallow: /versions/Unicode3.2.0/	# obsolete version
Disallow: /versions/Unicode4.0.0/	# obsolete version
Disallow: /versions/Unicode4.0.1/	# obsolete version
Disallow: /versions/Unicode4.1.0/	# obsolete version
Disallow: /versions/Unicode5.0.0/	# obsolete version
Disallow: /versions/Unicode5.1.0/	# obsolete version
Disallow: /versions/Unicode5.2.0/	# obsolete version
Disallow: /versions/Unicode6.0.0/	# obsolete version
Disallow: /versions/Unicode6.1.0/	# obsolete version
Disallow: /versions/Unicode6.2.0/	# obsolete version
Disallow: /versions/Unicode6.3.0/	# obsolete version
Disallow: /versions/Unicode7.0.0/	# obsolete version
Disallow: /versions/Unicode8.0.0/	# obsolete version
Disallow: /versions/Unicode9.0.0/	# obsolete version
Disallow: /versions/Unicode10.0.0/	# obsolete version
Disallow: /versions/Unicode11.0.0/	# obsolete version
Disallow: /charts/PDF/Unicode-3.1	# obsolete version
Disallow: /charts/PDF/Unicode-3.2	# obsolete version
Disallow: /charts/PDF/Unicode-4.0	# obsolete version
Disallow: /charts/PDF/Unicode-4.1	# obsolete version
Disallow: /charts/PDF/Unicode-5.0	# obsolete version
Disallow: /charts/PDF/Unicode-5.1	# obsolete version
Disallow: /charts/PDF/Unicode-5.2	# obsolete version
Disallow: /charts/PDF/Unicode-6.0	# obsolete version
Disallow: /charts/PDF/Unicode-6.1	# obsolete version
Disallow: /charts/PDF/Unicode-6.2	# obsolete version
Disallow: /charts/PDF/Unicode-6.3	# obsolete version
Disallow: /charts/PDF/Unicode-7.0	# obsolete version
Disallow: /charts/PDF/Unicode-8.0	# obsolete version
Disallow: /charts/PDF/Unicode-9.0	# obsolete version
Disallow: /charts/PDF/Unicode-10.0	# obsolete version
Disallow: /charts/PDF/Unicode-11.0	# obsolete version
Disallow: /charts/PDF/Unicode-12.0	# obsolete version
Disallow: /reports/tr1.html	# obsolete TR
Disallow: /reports/tr2.html	# obsolete TR
Disallow: /reports/tr3.html	# obsolete TR
Disallow: /reports/tr1/	# obsolete TR
Disallow: /reports/tr2/	# obsolete TR
Disallow: /reports/tr3/	# obsolete TR
Disallow: /reports/tr7/	# obsolete TR
Disallow: /reports/tr12/	# obsolete TR
Disallow: /reports/tr13/	# obsolete TR
Disallow: /reports/tr19/	# obsolete TR
Disallow: /reports/tr21/	# obsolete TR
Disallow: /reports/tr30/	# obsolete TR
Disallow: /reports/tr32/	# obsolete TR
Disallow: /reports/tr40/	# obsolete TR
Disallow: /reports/tr47/	# obsolete TR
Disallow: /reports/tr49/	# obsolete TR
Disallow: /reports/tr52/	# obsolete TR
Disallow: /anon-ftp/	# same as Public, different path
Disallow: /Public/MAPPINGS/OBSOLETE/ # obsolete data
Disallow: /unicode/members/
Disallow: /members/
Disallow: /repository/	# dynamic
Disallow: /cldr/repository/	# dynamic
Disallow: /cldr/utility/	# dynamic
Disallow: /cgi-bin/	# dynamic
Disallow: /cldr/data/diff/	# dynamic
Disallow: /cldr/trac/	# dynamic
Disallow: /edcom/bugtrack/	# dynamic
Disallow: /uli/trac/	# dynamic
Disallow: /~srloomis/ut/trac/	#dynamic
Disallow: /repos/	# dynamic
Disallow: /repos/cldr-tmp/	# dynamic
Disallow: /utility/trac/	# dynamic
Disallow: /utility/	# dynamic
Disallow: /forum/	# dynamic
Disallow: /forum/viewtopic.php	# dynamic, discontinued
Allow: /cldr/data/common/
Allow: /cldr/data/docs/
Allow: /cldr/data/tools/
Disallow: /cldr/data/   # everything else
Disallow: /cldr/dropbox/	# not really public
Disallow: /~ecartis	# not really public
Disallow: /unibook/oldversions/	# obsolete data
Disallow: /reports/tr46/tr46-4.html
Disallow: /reports/tr46/tr46-3.html
Disallow: /reports/tr46/tr46-2.html
Disallow: /reports/tr46/tr46-1.html
Disallow: /reports/tr15/tr15-24.html
Disallow: /reports/tr15/tr15-23.html
Disallow: /reports/tr15/tr15-31.html
Disallow: /reports/tr14/tr14-15.html
Disallow: /reports/dtr14-03.html
Disallow: /reports/tr14-4/index.html
Disallow: /reports/tr14-5/index.html
Disallow: /reports/tr9/tr9-21.html
Disallow: /reports/tr51/tr51-1.html
Disallow: /reports/tr51/tr51-1-archive.html
Disallow: /reports/tr51/tr51-2.html
Disallow: /reports/tr34/tr34-1.html
Disallow: /reports/tr34/tr34-2.html
Disallow: /reports/tr34/tr34-3.html
Disallow: /reports/tr34/tr34-4.html
Disallow: /reports/tr34/tr34-5.html
Disallow: /reports/tr34/tr34-6.html
Disallow: /reports/tr34/tr34-7.html
Disallow: /reports/tr34/tr34-8.html
Disallow: /reports/tr34/tr34-9.html
Disallow: /reports/tr34/tr34-10.html
Disallow: /reports/tr34/tr34-11.html
Disallow: /reports/tr34/tr34-12.html
Disallow: /reports/tr34/tr34-13.html
Disallow: /reports/tr34/tr34-14.html
Disallow: /reports/tr34/tr34-15.html
Disallow: /reports/tr34/tr34-16.html
Disallow: /reports/tr34/tr34-17.html
Disallow: /reports/tr34/tr34-18.html
Disallow: /reports/tr25/tr25-1.html
Disallow: /reports/tr25/tr25-2.html
Disallow: /reports/tr25/tr25-3.html
Disallow: /reports/tr25/tr25-4.html
Disallow: /reports/tr25/tr25-5.html
Disallow: /reports/tr25/tr25-6.html
Disallow: /reports/tr25/tr25-7.html
Disallow: /reports/tr25/tr25-8.html
Disallow: /reports/tr25/tr25-9.html
Disallow: /reports/tr25/tr25-10.pdf
Disallow: /reports/tr25/tr25-11.pdf
Disallow: /reports/tr25/tr25-12.pdf
Disallow: /reports/tr25/tr25-13.pdf
Disallow: /reports/tr25/tr25-14.pdf
User-Agent: AhrefsBot
Crawl-Delay: 60
User-Agent: Baiduspider
Crawl-Delay: 60
User-agent: CCBot
Crawl-Delay: 60
User-agent: Gigabot
Disallow: /
User-agent: betaBot
Disallow: /
User-agent: CCBot
Crawl-Delay: 60
User-agent: AhrefsBot
Crawl-Delay: 60 
User-agent: dotbot
Crawl-Delay: 60 
User-agent: MJ12bot
Crawl-Delay: 60 
User-agent: msnbot
Crawl-delay: 60
User-agent: BLEXBot
Crawl-delay: 60
User-agent: SeznamBot
Request-rate: 1/1m
Crawl-delay: 60