1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
# robots.txt for https://www.gamespot.com/
User-agent: *
Allow: /api/$
Disallow: /api/*
Disallow: /notifications/
Disallow: /search/
Disallow: *login=*
Disallow: /postRender
Disallow: /forums/*/flag/
Disallow: /forums/*/delete/
Disallow: /forums/*/edit/
Disallow: /forums/*/lock/
Disallow: /forums/*/anchor/
Disallow: /forums/*/add-to-favorite/
Disallow: /forums/*/best-answer/*/
Disallow: /*/forums/new-topic/
Disallow: /jsonsearch/
Disallow: /chat/
Disallow: /latest-amp-article/
|
