1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
|
User-Agent: *
Disallow: /*.html*mode=reply
Disallow: /*.html*replyto
Disallow: /data/foaf/
Disallow: /tag/
Disallow: /friendstimes
Disallow: /d*
Disallow: /ratings/users
User-Agent: Mediapartners-Google*
Allow: /
User-Agent: Yandex
Allow: /
Disallow: /data/foaf/
Disallow: /ratings/users
Clean-param: nojs&tag&mode&thread
Host: https://ohnotheydidnt.livejournal.com
Crawl-delay: 10
User-Agent: GoogleBot
Allow: /
Disallow: /ratings/users
Disallow: /data/foaf/
User-Agent: spbot
Disallow: /
User-agent: AhrefsBot
Disallow: /
Sitemap: https://ohnotheydidnt.livejournal.com/sitemap.xml
|