1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
// Copyright 2010 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: jdtang@google.com (Jonathan Tang)
//
// This contains an implementation of a tokenizer for HTML5. It consumes a
// buffer of UTF-8 characters, and then emits a stream of tokens.
#ifndef GUMBO_TOKENIZER_H_
#define GUMBO_TOKENIZER_H_
#include <stdbool.h>
#include <stddef.h>
#include "gumbo.h"
#include "token_type.h"
#include "tokenizer_states.h"
#ifdef __cplusplus
extern "C" {
#endif
struct GumboInternalParser;
// Struct containing all information pertaining to doctype tokens.
typedef struct GumboInternalTokenDocType {
const char* name;
const char* public_identifier;
const char* system_identifier;
bool force_quirks;
// There's no way to tell a 0-length public or system ID apart from the
// absence of a public or system ID, but they're handled different by the
// spec, so we need bool flags for them.
bool has_public_identifier;
bool has_system_identifier;
} GumboTokenDocType;
// Struct containing all information pertaining to start tag tokens.
typedef struct GumboInternalTokenStartTag {
GumboTag tag;
GumboVector /* GumboAttribute */ attributes;
bool is_self_closing;
} GumboTokenStartTag;
// A data structure representing a single token in the input stream. This
// contains an enum for the type, the source position, a GumboStringPiece
// pointing to the original text, and then a union for any parsed data.
typedef struct GumboInternalToken {
GumboTokenType type;
GumboSourcePosition position;
GumboStringPiece original_text;
bool is_injected;
union {
GumboTokenDocType doc_type;
GumboTokenStartTag start_tag;
GumboTag end_tag;
const char* text; // For comments.
int character; // For character, whitespace, null, and EOF tokens.
} v;
} GumboToken;
// Initializes the tokenizer state within the GumboParser object, setting up a
// parse of the specified text.
void gumbo_tokenizer_state_init(
struct GumboInternalParser* parser, const char* text, size_t text_length);
// Destroys the tokenizer state within the GumboParser object, freeing any
// dynamically-allocated structures within it.
void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
// Sets the tokenizer state to the specified value. This is needed by some
// parser states, which alter the state of the tokenizer in response to tags
// seen.
void gumbo_tokenizer_set_state(
struct GumboInternalParser* parser, GumboTokenizerEnum state);
// Flags whether the current node is a foreign content element. This is
// necessary for the markup declaration open state, where the tokenizer must be
// aware of the state of the parser to properly tokenize bad comment tags.
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
void gumbo_tokenizer_set_is_current_node_foreign(
struct GumboInternalParser* parser, bool is_foreign);
// Lexes a single token from the specified buffer, filling the output with the
// parsed GumboToken data structure. Returns true for a successful
// tokenization, false if a parse error occurs.
//
// Example:
// struct GumboInternalParser parser;
// GumboToken output;
// gumbo_tokenizer_state_init(&parser, text, strlen(text));
// while (gumbo_lex(&parser, &output)) {
// ...do stuff with output.
// gumbo_token_destroy(&parser, &token);
// }
// gumbo_tokenizer_state_destroy(&parser);
bool gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
// Frees the internally-allocated pointers within an GumboToken. Note that this
// doesn't free the token itself, since oftentimes it will be allocated on the
// stack. A simple call to free() (or GumboParser->deallocator, if
// appropriate) can handle that.
//
// Note that if you are handing over ownership of the internal strings to some
// other data structure - for example, a parse tree - these do not need to be
// freed.
void gumbo_token_destroy(GumboToken* token);
#ifdef __cplusplus
}
#endif
#endif // GUMBO_TOKENIZER_H_
|
