# 1) this filename (robots.txt) must stay lowercase
# 2) this file must be in the servers root directory
# 	ex: http://www.mydomain.com/pliggsubfolder/ -- you must move the robots.txt from
# 	/pliggsubfolder/ to the root folder for http://www.mydomain.com/
# 	you must then add your subfolder to each 'Disallow' below
# 	ex:  Disallow: /cache/  becomes  Disallow: /pliggsubfolder/cache/
User-agent: *
Crawl-delay: 2
Disallow:
Disallow: /admin/
Disallow: /3rdparty/
Disallow: /backup/
Disallow: /cache/
Disallow: /install/
Disallow: /internal/
Disallow: /libs/
Disallow: /plugins/
Disallow: /cache/templates_c/
Disallow: /search.php
Disallow: /admin_index.php
Disallow: /search/
Disallow: /searchurl/
Disallow: /tag/
Disallow: /user/view/shaken/
Disallow: /user/view/history/
Disallow: /user/view/published/
Disallow: /user/view/commented/
Disallow: /user/view/voted/
Disallow: /recent/
Disallow: /week/
Disallow: /month/
Disallow: /year/
Disallow: /upcoming/recent/
Disallow: /upcoming/week/
Disallow: /upcoming/month/
Disallow: /upcoming/year/
Disallow: /*?login=*
Disallow: /blog/wp-login.php
Disallow: /blog/wp-register.php
Disallow: /blog/images/
Disallow: /blog/send-to/
Disallow: /blog/page/
Disallow: /blog/*?*
Disallow: /blog/wp-login.php
Disallow: /blog/wp-register.php
Disallow: /blog/images/
Disallow: /blog/send-to/
Disallow: /blog/page/
Disallow: /blog/*?*
Disallow: /blog/tag/
Disallow: /blog/author/

User-agent: Mediapartners-Google
Disallow:

User-agent: bingbot
Crawl-delay: 20

User-agent: MSNBot
Crawl-delay: 20

User-agent: Slurp
Crawl-delay: 3400

User-agent: Yahoo! Slurp
Crawl-delay: 3400

User-agent: Yahoo! Slurp China
Disallow: / 
Crawl-delay: 3400

User-agent: Yahoo! DE Slurp
Crawl-delay: 3400

User-agent: Yahoo-MMCrawler
Crawl-delay: 3400


User-agent: YandexBot
User-agent: YandexMedia
User-agent: YandexImages
User-agent: YandexCatalog
User-agent: YaDirectFetcher
User-agent: YandexBlogs
User-agent: YandexNews
User-agent: YandexPagechecker
User-agent: YandexMetrika
User-agent: YandexMarket
User-agent: YandexCalendar
Crawl-delay: 360

User-agent: YahooMobile
User-agent: YahooCacheSystem
User-agent: Yahoo! Slurp/Site Explorer
User-agent: Mozilla/4.05 [en]
User-agent: LTI/LemurProject
User-agent: Yahoo-Blogs
User-agent: Yahoo-Blogs/v3.9
User-agent: Yahoo-MMCrawler
User-agent: Yahoo-MMCrawler/3.x
User-agent: YahooYSMcm
User-agent: YahooYSMcm/2.0.0
User-agent: Yahoo-Test
User-agent: Yahoo! Mindset
User-agent: Y!J-BSC
User-agent: Y!J-BSC/1.0
User-agent: Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
User-agent: y!j-bsc
User-agent: y!j-bsc/1.0
User-agent: y!j-bsc/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
User-agent: Y!J
User-agent: Y!J/1.0
User-agent: Y!J/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
User-agent: y!j
User-agent: y!j/1.0
User-agent: y!j/1.0 (http://help.yahoo.co.jp/help/jp/search/indexing/indexing-15.html)
User-agent: Mozilla/4.0 (compatible; Y!J; for robot study; keyoshid)
User-agent: Mozilla/4.0 (compatible; y!j; for robot study; keyoshid)
User-agent: Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
User-agent: Mozilla/5.0 (compatible; Yahoo! DE Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
User-agent: Mozilla/5.0 (Yahoo-Test/4.0 mailto:vertical-crawl-support@yahoo-inc.com) 
Crawl-delay: 3400