1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
\name{readHTMLTable}
\alias{readHTMLTable}
\alias{readHTMLTable,character-method}
\alias{readHTMLTable,HTMLInternalDocument-method}
\alias{readHTMLTable,XMLInternalElementNode-method}
\alias{coerce,character,FormattedInteger-method}
\alias{coerce,character,FormattedNumber-method}
\alias{coerce,character,Percent-method}
\alias{coerce,character,Currency-method}
\alias{FormattedInteger-class}
\alias{FormattedNumber-class}
\alias{Percent-class}
\title{Read data from one or more HTML tables}
\description{
This function and its methods provide somewhat robust methods for
extracting data from HTML tables in an HTML document.
One can read all the tables in a document given by filename or (\code{http:}
or \code{ftp:}) URL,
or having already parsed the document via \code{\link{htmlParse}}.
Alternatively, one can specify an individual \code{<table>}
node in the document.
The methods attempt to do some heuristic computations to determine
the header labels for the columns, the name of the table, etc.
}
% xmlName(node) == "table" && ("thead" \%in\% names(node) || length(getNodeSet(node, "./tr[1]/th")) > 0)
\usage{
readHTMLTable(doc, header = NA,
colClasses = NULL, skip.rows = integer(), trim = TRUE,
elFun = xmlValue, as.data.frame = TRUE, which = integer(),
...)
}
\arguments{
\item{doc}{the HTML document which can be a file name or a URL
or an already parsed \code{HTMLInternalDocument}, or
an HTML node of class \code{XMLInternalElementNode},
or a character vector containing the HTML content to parse and process.}
\item{header}{either a logical value indicating whether the table has
column labels, e.g. the first row or a \code{thead}, or alternatively
a character vector giving the names to use for the resulting columns.
This can be a logical vector and the individual values will be used
in turn for the different tables. This allows the caller to control
whether individual tables are processed as having column names.
Alternatively, one can read a specific table via the \code{which}
parameter and control how that is processed with a single scalar logical.
}
\item{colClasses}{either a list or a vector that gives the names of
the data types for the different columns in the table, or
alternatively a function used to convert the string values to the
appropriate type. A value of \code{NULL} means that we should drop
that column from the result.
Note that currently the conversion occurs before the
vectors are converted to a data frame (if \code{as.data.frame} is
\code{TRUE}).
As a result, to ensure that character vectors remain as characters
and not factors, use \code{stringsAsFactors = FALSE}.
This typically applies only to an individual table and so for the
method applied to a \code{XMLInternalElementNode} object.
In addition to the usual "integer", "numeric", "logical", "character", etc.
names of R data types, one can use
"FormattedInteger", "FormattedNumber" and "Percent" to specify that
format of the values are numbers possibly with commas (,) separating
groups of digits or a number followed by a percent sign (\%).
This mechanism allows one to introduce new classes and specify these
as targets in \code{colClasses}.
}
\item{skip.rows}{an integer vector indicating which rows to ignore.}
\item{trim}{a logical value indicating whether to remove leading and
trailing white space from the content cells.}
\item{elFun}{a function which, if specified, is called when converting
each cell. Currently, only the node is specified. In the future, we might
additionally pass the index of the column so that the function has
some context, e.g. whether the value is a row label or a regular
value, or if the caller knows the type of columns.
}
\item{as.data.frame}{a logical value indicating whether to turn the
resluting table(s) into data frames or leave them as matrices.
}
\item{which}{an integer vector identifying which tables to return from
within the document. This applies to the method for the document, not
individual tables.}
\item{\dots}{currently additional parameters that are passed on to
\code{\link{as.data.frame}} if \code{as.data.frame} is \code{TRUE}.
We may change this to use these as additional arguments for calls to
\code{elFun}.}
}
\value{
If the document (either by name or parsed tree) is specified,
the return vale is a list of data frames or matrices.
If a single HTML node is provided
}
\references{
HTML4.0 specification
}
\author{Duncan Temple Lang}
\seealso{
\code{\link{htmlParse}}
\code{\link{getNodeSet}}
\code{\link{xpathSApply}}
}
\examples{
\dontrun{
## This changed to using https: in June 2015, and that is unsupported.
# u = "http://en.wikipedia.org/wiki/World_population"
u = "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population"
tables = readHTMLTable(u)
names(tables)
tables[[2]]
# Print the table. Note that the values are all characters
# not numbers. Also the column names have a preceding X since
# R doesn't allow the variable names to start with digits.
tmp = tables[[2]]
# Let's just read the second table directly by itself.
doc = htmlParse(u)
tableNodes = getNodeSet(doc, "//table")
tb = readHTMLTable(tableNodes[[2]])
# Let's try to adapt the values on the fly.
# We'll create a function that turns a th/td node into a val
tryAsInteger = function(node) {
val = xmlValue(node)
ans = as.integer(gsub(",", "", val))
if(is.na(ans))
val
else
ans
}
tb = readHTMLTable(tableNodes[[2]], elFun = tryAsInteger)
tb = readHTMLTable(tableNodes[[2]], elFun = tryAsInteger,
colClasses = c("character", rep("integer", 9)))
}
zz =
readHTMLTable("https://www.inflationdata.com/Inflation/Consumer_Price_Index/HistoricalCPI.aspx")
if(any(i <- sapply(zz, function(x) if(is.null(x)) 0 else ncol(x)) == 14)) {
# guard against the structure of the page changing.
zz = zz[[which(i)[1]]] # 4th table
# convert columns to numeric. Could use colClasses in the call to readHTMLTable()
zz[-1] = lapply(zz[-1], function(x) as.numeric(gsub(".* ", "", as.character(x))))
matplot(1:12, t(zz[-c(1, 14)]), type = "l")
}
# From Marsh Feldman on R-help, possibly
# https://stat.ethz.ch/pipermail/r-help/2010-March/232586.html
# That site was non-responsive in June 2015,
# and this does not do a good job on the current table.
\dontrun{
doc <- "http://www.nber.org/cycles/cyclesmain.html"
# The main table is the second one because it's embedded in the page table.
tables <- getNodeSet(htmlParse(doc), "//table")
xt <- readHTMLTable(tables[[2]],
header = c("peak","trough","contraction",
"expansion","trough2trough","peak2peak"),
colClasses = c("character","character","character",
"character","character","character"),
trim = TRUE, stringsAsFactors = FALSE
)
}
if(FALSE) {
# Here is a totally different way of reading tables from HTML documents.
# The data are formatted using PRE and so can be read via read.table
u = "http://tidesonline.nos.noaa.gov/data_read.shtml?station_info=9414290+San+Francisco,+CA"
h = htmlParse(u)
p = getNodeSet(h, "//pre")
con = textConnection(xmlValue(p[[2]]))
tides = read.table(con)
}
\dontrun{
## This is not accessible without authentication ...
u = "https://www.omegahat.net/RCurl/testPassword/table.html"
if(require(RCurl) && url.exists(u)) {
tt = getURL(u, userpwd = "bob:duncantl")
readHTMLTable(tt)
}}
}
\keyword{IO}
\keyword{data}
|
