User-agent: *
Disallow: /draft/	# not official even if visible
Disallow: /Public/1.1-Update/	# obsolete data
Disallow: /Public/2.1-Update2/	# obsolete data
Disallow: /Public/3.0-Update/	# obsolete data
Disallow: /Public/3.0-Update/	# obsolete data
Disallow: /Public/2.1-Update3/	# obsolete data
Disallow: /Public/2.1-Update/	# obsolete data
Disallow: /Public/2.1-Update4/	# obsolete data
Disallow: /Public/3.0-Update1/ # obsolete data
Disallow: /Public/3.1-Update/  # obsolete data
Disallow: /Public/3.1-Update1/ # obsolete data
Disallow: /Public/3.2-Update/ # obsolete data
Disallow: /Public/4.0-Update/ # obsolete data
Disallow: /Public/4.0-Update1/ # obsolete data
Disallow: /Public/4.1.0/	# obsolete data
Disallow: /Public/5.0.0/	# obsolete data
Disallow: /Public/5.1.0/	# obsolete data
Disallow: /Public/5.2.0/	# obsolete data
Disallow: /Public/6.0.0/	# obsolete data
Disallow: /Public/6.1.0/	# obsolete data
Disallow: /Public/6.2.0/	# obsolete data
Disallow: /Public/6.3.0/	# obsolete data
Disallow: /Public/7.0.0/	# obsolete data
Disallow: /Public/8.0.0/	# obsolete data
Disallow: /Public/9.0.0/	# obsolete data
Disallow: /Public/10.0.0/	# obsolete data
Disallow: /Public/11.0.0/	# obsolete data
Disallow: /Public/12.0.0/	# obsolete data
Disallow: /Public/13.0.0/	# obsolete data
Disallow: /Public/14.0.0/	# obsolete data
Disallow: /Public/15.0.0/	# obsolete data
Disallow: /fr/			# obsolete pages and charts
Disallow: /versions/Unicode2.0.0/	# obsolete version
Disallow: /versions/Unicode2.1.0/	# obsolete version
Disallow: /versions/Unicode3.0.0/	# obsolete version
Disallow: /versions/Unicode3.0.1/	# obsolete version
Disallow: /versions/Unicode3.1.0/	# obsolete version
Disallow: /versions/Unicode3.1.1/	# obsolete version
Disallow: /versions/Unicode3.2.0/	# obsolete version
Disallow: /versions/Unicode4.0.0/	# obsolete version
Disallow: /versions/Unicode4.0.1/	# obsolete version
Disallow: /versions/Unicode4.1.0/	# obsolete version
Disallow: /versions/Unicode5.0.0/	# obsolete version
Disallow: /versions/Unicode5.1.0/	# obsolete version
Disallow: /versions/Unicode5.2.0/	# obsolete version
Disallow: /versions/Unicode6.0.0/	# obsolete version
Disallow: /versions/Unicode6.1.0/	# obsolete version
Disallow: /versions/Unicode6.2.0/	# obsolete version
Disallow: /versions/Unicode6.3.0/	# obsolete version
Disallow: /versions/Unicode7.0.0/	# obsolete version
Disallow: /versions/Unicode8.0.0/	# obsolete version
Disallow: /versions/Unicode9.0.0/	# obsolete version
Disallow: /versions/Unicode10.0.0/	# obsolete version
Disallow: /versions/Unicode11.0.0/	# obsolete version
Disallow: /versions/Unicode12.0.0/	# obsolete version
Disallow: /versions/Unicode13.0.0/	# obsolete version
Disallow: /versions/Unicode14.0.0/	# obsolete version
Disallow: /versions/Unicode15.0.0/	# obsolete version
Disallow: /charts/PDF/Unicode-3.1	# obsolete version
Disallow: /charts/PDF/Unicode-3.2	# obsolete version
Disallow: /charts/PDF/Unicode-4.0	# obsolete version
Disallow: /charts/PDF/Unicode-4.1	# obsolete version
Disallow: /charts/PDF/Unicode-5.0	# obsolete version
Disallow: /charts/PDF/Unicode-5.1	# obsolete version
Disallow: /charts/PDF/Unicode-5.2	# obsolete version
Disallow: /charts/PDF/Unicode-6.0	# obsolete version
Disallow: /charts/PDF/Unicode-6.1	# obsolete version
Disallow: /charts/PDF/Unicode-6.2	# obsolete version
Disallow: /charts/PDF/Unicode-6.3	# obsolete version
Disallow: /charts/PDF/Unicode-7.0	# obsolete version
Disallow: /charts/PDF/Unicode-8.0	# obsolete version
Disallow: /charts/PDF/Unicode-9.0	# obsolete version
Disallow: /charts/PDF/Unicode-10.0	# obsolete version
Disallow: /charts/PDF/Unicode-11.0	# obsolete version
Disallow: /charts/PDF/Unicode-12.0	# obsolete version
Disallow: /charts/PDF/Unicode-13.0	# obsolete version
Disallow: /charts/PDF/Unicode-14.0	# obsolete version
Disallow: /charts/beta/			# not permanent content
Disallow: /reports/tr1.html	# obsolete TR
Disallow: /reports/tr2.html	# obsolete TR
Disallow: /reports/tr3.html	# obsolete TR
Disallow: /reports/tr6-10.html	# obsolete TR draft
Disallow: /reports/tr1/	# obsolete TR
Disallow: /reports/tr2/	# obsolete TR
Disallow: /reports/tr3/	# obsolete TR
Disallow: /reports/tr7/	# obsolete TR
Disallow: /reports/tr12/	# obsolete TR
Disallow: /reports/tr13/	# obsolete TR
Disallow: /reports/tr19/	# obsolete TR
Disallow: /reports/tr21/	# obsolete TR
Disallow: /reports/tr30/	# obsolete TR
Disallow: /reports/tr32/	# obsolete TR
Disallow: /reports/tr40/	# obsolete TR
Disallow: /reports/tr47/	# obsolete TR
Disallow: /reports/tr49/	# obsolete TR
Disallow: /reports/tr52/	# obsolete TR
Disallow: /anon-ftp/	# same as Public, different path
Disallow: /Public/MAPPINGS/OBSOLETE/ # obsolete data
Disallow: /unicode/members/
Disallow: /members/
Disallow: /repository/	# dynamic
Disallow: /cldr/repository/	# dynamic
Disallow: /cldr/utility/	# dynamic
Disallow: /cgi-bin/	# dynamic
Disallow: /cldr/data/diff/	# dynamic
Disallow: /cldr/trac/	# dynamic
Disallow: /edcom/bugtrack/	# dynamic
Disallow: /uli/trac/	# dynamic
Disallow: /~srloomis/ut/trac/	#dynamic
Disallow: /repos/	# dynamic
Disallow: /repos/cldr-tmp/	# dynamic
Disallow: /utility/trac/	# dynamic
Disallow: /utility/	# dynamic
Disallow: /forum/	# dynamic
Disallow: /forum/viewtopic.php	# dynamic, discontinued
Allow: /cldr/data/common/
Allow: /cldr/data/docs/
Allow: /cldr/data/tools/
Disallow: /cldr/data/   # everything else
Disallow: /cldr/dropbox/	# not really public
Disallow: /~ecartis	# not really public
Disallow: /unibook/oldversions/	# obsolete data
Disallow: /reports/tr46/tr46-1.html
Disallow: /reports/tr46/tr46-2.html
Disallow: /reports/tr46/tr46-3.html
Disallow: /reports/tr46/tr46-4.html
Disallow: /reports/tr46/tr46-5.html
Disallow: /reports/tr46/tr46-6.html
Disallow: /reports/tr46/tr46-7.html
Disallow: /reports/tr46/tr46-8.html
Disallow: /reports/tr46/tr46-9.html
Disallow: /reports/tr46/tr46-10.html
Disallow: /reports/tr46/tr46-11.html
Disallow: /reports/tr46/tr46-12.html
Disallow: /reports/tr46/tr46-13.html
Disallow: /reports/tr46/tr46-14.html
Disallow: /reports/tr46/tr46-15.html
Disallow: /reports/tr46/tr46-16.html
Disallow: /reports/tr46/tr46-17.html
Disallow: /reports/tr46/tr46-18.html
Disallow: /reports/tr46/tr46-19.html
Disallow: /reports/tr46/tr46-20.html
Disallow: /reports/tr46/tr46-21.html
Disallow: /reports/tr46/tr46-22.html
Disallow: /reports/tr46/tr46-23.html
Disallow: /reports/tr46/tr46-24.html
Disallow: /reports/tr46/tr46-25.html
Disallow: /reports/tr46/tr46-26.html
Disallow: /reports/tr46/tr46-27.html
Disallow: /reports/tr46/tr46-28.html
Disallow: /reports/tr46/tr46-29.html
Disallow: /reports/tr46/tr46-30.html
Disallow: /reports/tr15/tr15-7.html
Disallow: /reports/tr15/tr15-8.html
Disallow: /reports/tr15/tr15-9.html
Disallow: /reports/tr15/tr15-10.html
Disallow: /reports/tr15/tr15-11.html
Disallow: /reports/tr15/tr15-12.html
Disallow: /reports/tr15/tr15-13.html
Disallow: /reports/tr15/tr15-14.html
Disallow: /reports/tr15/tr15-15.html
Disallow: /reports/tr15/tr15-16.html
Disallow: /reports/tr15/tr15-17.html
Disallow: /reports/tr15/tr15-18.html
Disallow: /reports/tr15/tr15-19.html
Disallow: /reports/tr15/tr15-20.html
Disallow: /reports/tr15/tr15-21.html
Disallow: /reports/tr15/tr15-22.html
Disallow: /reports/tr15/tr15-23.html
Disallow: /reports/tr15/tr15-24.html
Disallow: /reports/tr15/tr15-25.html
Disallow: /reports/tr15/tr15-26.html
Disallow: /reports/tr15/tr15-20.html
Disallow: /reports/tr15/tr15-21.html
Disallow: /reports/tr15/tr15-22.html
Disallow: /reports/tr15/tr15-23.html
Disallow: /reports/tr15/tr15-24.html
Disallow: /reports/tr15/tr15-25.html
Disallow: /reports/tr15/tr15-26.html
Disallow: /reports/tr15/tr15-27.html
Disallow: /reports/tr15/tr15-28.html
Disallow: /reports/tr15/tr15-29.html
Disallow: /reports/tr15/tr15-30.html
Disallow: /reports/tr15/tr15-31.html
Disallow: /reports/tr15/tr15-32.html
Disallow: /reports/tr15/tr15-33.html
Disallow: /reports/tr15/tr15-34.html
Disallow: /reports/tr15/tr15-35.html
Disallow: /reports/tr15/tr15-36.html
Disallow: /reports/tr15/tr15-37.html
Disallow: /reports/tr15/tr15-38.html
Disallow: /reports/tr15/tr15-39.html
Disallow: /reports/tr15/tr15-40.html
Disallow: /reports/tr15/tr15-40.html
Disallow: /reports/tr15/tr15-41.html
Disallow: /reports/tr15/tr15-42.html
Disallow: /reports/tr15/tr15-43.html
Disallow: /reports/tr15/tr15-44.html
Disallow: /reports/tr15/tr15-45.html
Disallow: /reports/tr15/tr15-46.html
Disallow: /reports/tr15/tr15-47.html
Disallow: /reports/tr15/tr15-48.html
Disallow: /reports/tr15/tr15-49.html
Disallow: /reports/tr15/tr15-50.html
Disallow: /reports/tr15/tr15-51.html
Disallow: /reports/tr15/tr15-52.html
Disallow: /reports/tr15/tr15-53.html
Disallow: /reports/tr14/tr14-15.html
Disallow: /reports/dtr14-03.html
Disallow: /reports/tr14-4/index.html
Disallow: /reports/tr14-5/index.html
Disallow: /reports/tr9/tr9-10.html
Disallow: /reports/tr9/tr9-11.html
Disallow: /reports/tr9/tr9-12.html
Disallow: /reports/tr9/tr9-13.html
Disallow: /reports/tr9/tr9-14.html
Disallow: /reports/tr9/tr9-15.html
Disallow: /reports/tr9/tr9-16.html
Disallow: /reports/tr9/tr9-17.html
Disallow: /reports/tr9/tr9-18.html
Disallow: /reports/tr9/tr9-19.html
Disallow: /reports/tr9/tr9-20.html
Disallow: /reports/tr9/tr9-21.html
Disallow: /reports/tr9/tr9-22.html
Disallow: /reports/tr9/tr9-23.html
Disallow: /reports/tr9/tr9-24.html
Disallow: /reports/tr9/tr9-25.html
Disallow: /reports/tr9/tr9-26.html
Disallow: /reports/tr9/tr9-20.html
Disallow: /reports/tr9/tr9-21.html
Disallow: /reports/tr9/tr9-22.html
Disallow: /reports/tr9/tr9-23.html
Disallow: /reports/tr9/tr9-24.html
Disallow: /reports/tr9/tr9-25.html
Disallow: /reports/tr9/tr9-26.html
Disallow: /reports/tr9/tr9-27.html
Disallow: /reports/tr9/tr9-28.html
Disallow: /reports/tr9/tr9-29.html
Disallow: /reports/tr9/tr9-30.html
Disallow: /reports/tr9/tr9-31.html
Disallow: /reports/tr9/tr9-32.html
Disallow: /reports/tr9/tr9-33.html
Disallow: /reports/tr9/tr9-34.html
Disallow: /reports/tr9/tr9-35.html
Disallow: /reports/tr9/tr9-36.html
Disallow: /reports/tr9/tr9-37.html
Disallow: /reports/tr9/tr9-38.html
Disallow: /reports/tr9/tr9-39.html
Disallow: /reports/tr9/tr9-40.html
Disallow: /reports/tr9/tr9-40.html
Disallow: /reports/tr9/tr9-41.html
Disallow: /reports/tr9/tr9-42.html
Disallow: /reports/tr9/tr9-43.html
Disallow: /reports/tr9/tr9-44.html
Disallow: /reports/tr9/tr9-45.html
Disallow: /reports/tr9/tr9-46.html
Disallow: /reports/tr9/tr9-47.html
Disallow: /reports/tr51/tr51-1.html
Disallow: /reports/tr51/tr51-1-archive.html
Disallow: /reports/tr51/tr51-2.html
Disallow: /reports/tr51/tr51-3.html
Disallow: /reports/tr51/tr51-4.html
Disallow: /reports/tr51/tr51-5.html
Disallow: /reports/tr51/tr51-6.html
Disallow: /reports/tr51/tr51-7.html
Disallow: /reports/tr51/tr51-8.html
Disallow: /reports/tr51/tr51-9.html
Disallow: /reports/tr51/tr51-10.html
Disallow: /reports/tr51/tr51-11.html
Disallow: /reports/tr51/tr51-12.html
Disallow: /reports/tr51/tr51-13.html
Disallow: /reports/tr51/tr51-14.html
Disallow: /reports/tr51/tr51-15.html
Disallow: /reports/tr51/tr51-16.html
Disallow: /reports/tr51/tr51-17.html
Disallow: /reports/tr51/tr51-18.html
Disallow: /reports/tr51/tr51-19.html
Disallow: /reports/tr51/tr51-20.html
Disallow: /reports/tr51/tr51-21.html
Disallow: /reports/tr51/tr51-22.html
Disallow: /reports/tr34/tr34-1.html
Disallow: /reports/tr34/tr34-2.html
Disallow: /reports/tr34/tr34-3.html
Disallow: /reports/tr34/tr34-4.html
Disallow: /reports/tr34/tr34-5.html
Disallow: /reports/tr34/tr34-6.html
Disallow: /reports/tr34/tr34-7.html
Disallow: /reports/tr34/tr34-8.html
Disallow: /reports/tr34/tr34-9.html
Disallow: /reports/tr34/tr34-10.html
Disallow: /reports/tr34/tr34-11.html
Disallow: /reports/tr34/tr34-12.html
Disallow: /reports/tr34/tr34-13.html
Disallow: /reports/tr34/tr34-14.html
Disallow: /reports/tr34/tr34-15.html
Disallow: /reports/tr34/tr34-16.html
Disallow: /reports/tr34/tr34-17.html
Disallow: /reports/tr34/tr34-18.html
Disallow: /reports/tr34/tr34-19.html
Disallow: /reports/tr34/tr34-20.html
Disallow: /reports/tr34/tr34-21.html
Disallow: /reports/tr34/tr34-22.html
Disallow: /reports/tr34/tr34-23.html
Disallow: /reports/tr34/tr34-24.html
Disallow: /reports/tr34/tr34-25.html
Disallow: /reports/tr34/tr34-26.html
Disallow: /reports/tr34/tr34-27.html
Disallow: /reports/tr35/tr35-1.html	# obsolete version
Disallow: /reports/tr35/tr35-2.html	# obsolete version
Disallow: /reports/tr35/tr35-3.html	# obsolete version
Disallow: /reports/tr35/tr35-4.html	# obsolete version
Disallow: /reports/tr35/tr35-5.html	# obsolete version
Disallow: /reports/tr35/tr35-6.html	# obsolete version
Disallow: /reports/tr35/tr35-7.html	# obsolete version
Disallow: /reports/tr35/tr35-8.html	# obsolete version
Disallow: /reports/tr35/tr35-9.html	# obsolete version
Disallow: /reports/tr35/tr35-10.html	# obsolete version
Disallow: /reports/tr35/tr35-11.html	# obsolete version
Disallow: /reports/tr35/tr35-12.html	# obsolete version
Disallow: /reports/tr35/tr35-13.html	# obsolete version
Disallow: /reports/tr35/tr35-14d1.html	# obsolete version
Disallow: /reports/tr35/tr35-15.html	# obsolete version
Disallow: /reports/tr35/tr35-16.html	# obsolete version
Disallow: /reports/tr35/tr35-17.html	# obsolete version
Disallow: /reports/tr35/tr35-18.html	# obsolete version
Disallow: /reports/tr35/tr35-19.html	# obsolete version
Disallow: /reports/tr35/tr35-20.html	# obsolete version
Disallow: /reports/tr35/tr35-21.html	# obsolete version
Disallow: /reports/tr35/tr35-22.html	# obsolete version
Disallow: /reports/tr35/tr35-23.html	# obsolete version
Disallow: /reports/tr35/tr35-25.html	# obsolete version
Disallow: /reports/tr35/tr35-27.html	# obsolete version
Disallow: /reports/tr35/tr35-29.html	# obsolete version
Disallow: /reports/tr35/tr35-30/	# obsolete version
Disallow: /reports/tr35/tr35-31/	# obsolete version
Disallow: /reports/tr35/tr35-32/	# obsolete version
Disallow: /reports/tr35/tr35-33/	# obsolete version
Disallow: /reports/tr35/tr35-34/	# obsolete version
Disallow: /reports/tr35/tr35-35/	# obsolete version
Disallow: /reports/tr35/tr35-36/	# obsolete version
Disallow: /reports/tr35/tr35-37/	# obsolete version
Disallow: /reports/tr35/tr35-38/	# obsolete version
Disallow: /reports/tr35/tr35-39/	# obsolete version
Disallow: /reports/tr35/tr35-40/	# obsolete version
Disallow: /reports/tr35/tr35-41/	# obsolete version
Disallow: /reports/tr35/tr35-42/	# obsolete version
Disallow: /reports/tr35/tr35-43/	# obsolete version
Disallow: /reports/tr35/tr35-44/	# obsolete version
Disallow: /reports/tr35/tr35-45/	# obsolete version
Disallow: /reports/tr35/tr35-46/	# obsolete version
Disallow: /reports/tr35/tr35-47/	# obsolete version
Disallow: /reports/tr35/tr35-48/	# obsolete version
Disallow: /reports/tr35/tr35-49/	# obsolete version
Disallow: /reports/tr35/tr35-50/	# obsolete version
Disallow: /reports/tr35/tr35-51/	# obsolete version
Disallow: /reports/tr35/tr35-52/	# obsolete version
Disallow: /reports/tr35/tr35-53/	# obsolete version
Disallow: /reports/tr35/tr35-54/	# obsolete version
Disallow: /reports/tr35/tr35-55/	# obsolete version
Disallow: /reports/tr35/tr35-56/	# obsolete version
Disallow: /reports/tr35/tr35-57/	# obsolete version
Disallow: /reports/tr35/tr35-58/	# obsolete version
Disallow: /reports/tr35/tr35-59/	# obsolete version
Disallow: /reports/tr35/tr35-60/	# obsolete version
Disallow: /reports/tr35/tr35-61/	# obsolete version
Disallow: /reports/tr35/tr35-62/	# obsolete version
Disallow: /reports/tr35/tr35-63/	# obsolete version
Disallow: /reports/tr35/tr35-64/	# obsolete version
Disallow: /reports/tr35/tr35-65/	# obsolete version
Disallow: /reports/tr35/tr35-66/	# obsolete version
Disallow: /reports/tr35/tr35-67/	# obsolete version
Disallow: /reports/tr35/tr35-68/	# obsolete version
Disallow: /reports/tr25/tr25-1.html
Disallow: /reports/tr25/tr25-2.html
Disallow: /reports/tr25/tr25-3.html
Disallow: /reports/tr25/tr25-4.html
Disallow: /reports/tr25/tr25-5.html
Disallow: /reports/tr25/tr25-6.html
Disallow: /reports/tr25/tr25-7.html
Disallow: /reports/tr25/tr25-8.html
Disallow: /reports/tr25/tr25-9.html
Disallow: /reports/tr25/tr25-10.pdf
Disallow: /reports/tr25/tr25-11.pdf
Disallow: /reports/tr25/tr25-12.pdf
Disallow: /reports/tr25/tr25-13.pdf
Disallow: /reports/tr25/tr25-14.pdf
User-Agent: AhrefsBot
Crawl-Delay: 60
User-Agent: Baiduspider
Crawl-Delay: 60
User-agent: CCBot
Crawl-Delay: 60
User-agent: Gigabot
Disallow: /
User-agent: betaBot
Disallow: /
User-agent: CCBot
Crawl-Delay: 60
User-agent: AhrefsBot
Crawl-Delay: 60 
User-agent: dotbot
Crawl-Delay: 60 
User-agent: MJ12bot
Crawl-Delay: 60 
User-agent: msnbot
Crawl-delay: 60
User-agent: BLEXBot
Crawl-delay: 60
User-agent: SeznamBot
Request-rate: 1/1m
Crawl-delay: 60
User-agent: Viber/18.3.2.0
Disallow: /