1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
# $Id: robots.txt,v 1.81 2018/12/13 19:15:03 jliao Exp $
#
# This is a file retrieved by webwalkers a.k.a. spiders that
# conform to a defacto standard.
# See <URL:http://www.robotstxt.org/wc/exclusion.html#robotstxt>
#
# Comments to the webmaster should be posted at <URL:http://www.ibm.com/contact>
#
# Format is:
# User-agent: <name of spider>
# Disallow: <nothing> | <path>
# ------------------------------------------------------------------------------
User-agent: *
Disallow: //
Disallow: /account/registration
Disallow: /account/mypro
Disallow: /account/myint
Disallow: /Admin
Disallow: /cgi-
Disallow: /contact/employees/servlets
Disallow: /data
Disallow: /db2s
Disallow: /developerworks/*-pdf.pdf$
Disallow: /developerworks/forums/servlet
Disallow: /developerworks/forums/abuse
Disallow: /developerworks/forums/post
Disallow: /fcgi-
Disallow: /fscripts
# Disallow: /homepage
Disallow: /image
Disallow: /mashupmaker
Disallow: /partnerworld/portal
Disallow: /portal/home
Disallow: /products/finder
Disallow: /profiles
Disallow: /scripts
Disallow: /Scripts
Disallow: /search/?
Disallow: /Search
Disallow: /smartmarket/*/productListInitial.action?
Disallow: /standards
Disallow: /support/knowledgecenter/api
Disallow: /support/knowledgecenter/v1
Disallow: /survey
Disallow: /tmp
Disallow: /vrm/bin/sendThisPage
Disallow: /webmaster
Disallow: /zx
Disallow: /zz
Disallow: /*.ssi$
Disallow: /account/myibm/InterestsEdit.do
Disallow: /wcs
Disallow: /wcsstore
Disallow: /webapp
Disallow: /web/portal/software/websphere
Disallow: /common/austin-summit
Disallow: /link
Disallow: /links
Disallow: /blog/
Disallow: /web/portal/commerce
Disallow: /industries/clients
Disallow: /standards
Disallow: /internal
Disallow: /test
Disallow: /test-kick-tires
Disallow: /product-detail
Allow: /common/ssi
Allow: /data-responsibility
sitemap: https://www.ibm.com/homepage_sitemap.xml
sitemap: https://www.ibm.com/sitemap-cms.xml
sitemap: https://www.ibm.com/connect/ibm/sitemap.xml
sitemap: https://www.ibm.com/marketplace/storefront-sitemap-index.xml
sitemap: https://www.ibm.com/downloads/cas/sitemap/sitemap.xml
User-agent: Fast corporate crawler
Disallow: //
Disallow: /account/registration
Disallow: /account/mypro
Disallow: /account/myint
Disallow: /Admin
Disallow: /cgi-
Disallow: /common
Disallow: /contact/employees/servlets
Disallow: /data
Disallow: /db2s
Disallow: /developerworks
Disallow: /fcgi-
Disallow: /fscripts
Disallow: /homepage
Disallow: /i/
Disallow: /image
Disallow: /investor
Disallow: /link
Disallow: /mashupmaker
Disallow: /portal/home
Disallow: /products/finder
Disallow: /profiles
Disallow: /SametimeWebApp
Disallow: /scripts
Disallow: /Scripts
Disallow: /search/?
Disallow: /Search
Disallow: /survey
Disallow: /tmp
Disallow: /vrm/bin/sendThisPage
Disallow: /webmaster
Disallow: /zx
Disallow: /zz
User-agent: 008
Disallow: /smartmarket
# Added for IN4173782 on 7Aug2013
User-agent: magpie-crawler
Disallow: /connections/
# Added for IN4177562 on 8Aug2013
User-agent: magpie-crawler/1.1
Disallow: /connections/
# Added for IN4177562 on 8Aug2013
User-agent: MagpieRSS
Disallow: /connections/
User-agent: psnsearch
Disallow: /developerworks/wikis/login.action
# Added to block site mirroring
User-agent: HTTrack
Disallow: /
#
|
