File: JSON_Tokenizer.hh

package info (click to toggle)
eclipse-titan 6.5.0-1
  • links: PTS
  • area: main
  • in suites: buster
  • size: 101,128 kB
  • sloc: cpp: 259,139; ansic: 47,560; yacc: 22,554; makefile: 14,074; sh: 12,630; lex: 9,101; xml: 5,362; java: 4,849; perl: 3,784; awk: 48; php: 32; python: 13
file content (189 lines) | stat: -rw-r--r-- 8,178 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
/******************************************************************************
 * Copyright (c) 2000-2018 Ericsson Telecom AB
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v2.0
 * which accompanies this distribution, and is available at
 * https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.html
 *
 * Contributors:
 *   Balasko, Jeno
 *   Baranyi, Botond
 *   Szabo, Bence Janos
 *
 ******************************************************************************/

#ifndef JSON_TOKENIZER_HH
#define	JSON_TOKENIZER_HH

#include <cstddef>

/** JSON token types */
enum json_token_t {
  JSON_TOKEN_ERROR = 0,     // not actually a token, used when get_next_token() fails
  JSON_TOKEN_NONE,          // not actually a token, used for initializing
  JSON_TOKEN_OBJECT_START,  // "{"
  JSON_TOKEN_OBJECT_END,    // "}"
  JSON_TOKEN_ARRAY_START,   // "["
  JSON_TOKEN_ARRAY_END,     // "]"
  JSON_TOKEN_NAME,          // field name (key) in a JSON object, followed by ":"
  JSON_TOKEN_NUMBER,        // JSON number value
  JSON_TOKEN_STRING,        // JSON string value
  JSON_TOKEN_LITERAL_TRUE,  // "true" value
  JSON_TOKEN_LITERAL_FALSE, // "false" value
  JSON_TOKEN_LITERAL_NULL   // "null" value
};
  
/** A class for building and processing JSON documents. Stores the document in a buffer.
  * Can build JSON documents by inserting tokens into an empty buffer.
  * Can extract tokens from an existing JSON document. */
class JSON_Tokenizer {
  
private:
  
  /** The buffer that stores the JSON document 
    * This is a buffer with exponential allocation (expstring), only uses expstring
    * memory operations from memory.h (ex.: mputstr, mputprintf) */
  char* buf_ptr;
  
  /** Number of bytes currently in the buffer */
  size_t buf_len;
  
  /** Current position in the buffer */
  size_t buf_pos;
  
  /** Current depth in the JSON document (only used if pretty printing is set */
  unsigned int depth;
  
  /** Stores the previous JSON token inserted by put_next_token() */
  json_token_t previous_token;
  
  /** Activates or deactivates pretty printing
    * If active, put_next_token() and put_separator() will add extra newlines 
    * and indenting to the JSON code to make it more readable for you humans,
    * otherwise it will be compact (no white spaces). */
  bool pretty;
  
  /** Initializes the properties of the tokenizer. 
    * The buffer is initialized with the parameter data (unless it's empty). */
  void init(const char* p_buf, const size_t p_buf_len);
  
  /** Inserts a character to the end of the buffer */
  void put_c(const char c);
  
  /** Inserts a null-terminated string to the end of the buffer */
  void put_s(const char* s);
  
  /** Indents a new line in JSON code depending on the current depth.
    * If the maximum depth is reached, the code is not indented further.
    * Used only if pretty printing is set. */
  void put_depth();
  
  /** Skips white spaces until a non-white-space character is found.
    * Returns false if the end of the buffer is reached before a non-white-space
    * character is found, otherwise returns true. */
  bool skip_white_spaces();
  
  /** Attempts to find a JSON string at the current buffer position. 
    * Returns true if a valid string is found before the end of the buffer
    * is reached, otherwise returns false. */
  bool check_for_string();

  /** Checks if the current character in the buffer is a valid JSON separator.
    * Separators are: commas (,), colons (:) and curly and square brackets ({}[]).
    * This function also steps over the separator if it's a comma.
    * Returns true if a separator is found, otherwise returns false. */
  bool check_for_separator();
  
  /** Attempts to find a specific JSON literal at the current buffer position.
    * Returns true if the literal is found, otherwise returns false.
    * @param p_literal [in] Literal value to find */
  bool check_for_literal(const char* p_literal);
  
  /** Adds a separating comma (,) if the previous token is a value, or an object or
    * array end mark. */
  void put_separator();
  
  /** No copy constructor. Implement if needed. */
  JSON_Tokenizer(const JSON_Tokenizer&);
  
  /** No assignment operator. Implement if needed. */
  JSON_Tokenizer& operator=(const JSON_Tokenizer&);
  
public:
  /** Constructs a tokenizer with an empty buffer.
    * Use put_next_token() to build a JSON document and get_buffer()/get_buffer_length() to retrieve it */
  JSON_Tokenizer(bool p_pretty = false) : pretty(p_pretty) { init(0, 0); }
  
  /** Constructs a tokenizer with the buffer parameter.
    * Use get_next_token() to read JSON tokens and get_pos()/set_pos() to move around in the buffer */
  JSON_Tokenizer(const char* p_buf, const size_t p_buf_len) : pretty(false) { init(p_buf, p_buf_len); }
  
  /** Destructor. Frees the buffer. */
  ~JSON_Tokenizer();
  
  /** Reinitializes the tokenizer with a new buffer. */
  inline void set_buffer(const char* p_buf, const size_t p_buf_len) { init(p_buf, p_buf_len); }
  
  /** Retrieves the buffer containing the JSON document. */
  inline const char* get_buffer() { return buf_ptr; }
  
  /** Retrieves the length of the buffer containing the JSON document. */
  inline size_t get_buffer_length() { return buf_len; }
  
  /** Extracts a JSON token from the current buffer position.
    * @param p_token [out] Extracted token type, or JSON_TOKEN_ERROR if no token
    * could be extracted, or JSON_TOKEN_NONE if the buffer end is reached
    * @param p_token_str [out] A pointer to the token data (if any):
    * the name of a JSON object field (without quotes), or the string representation
    * of a JSON number, or a JSON string (with quotes and double-escaped).
    * @param p_str_len [out] The character length of the token data (if there is data)
    * @return The number of characters extracted 
    * @note The token data is not copied, *p_token_str will point to the start of the 
    * data in the tokenizer's buffer. */
  size_t get_next_token(json_token_t* p_token, char** p_token_str, size_t* p_str_len);
  
  /** Gets the current read position in the buffer.
    * This is where get_next_token() will read from next. */
  inline size_t get_buf_pos() { return buf_pos; }
  
  /** Sets the current read position in the buffer.
    * This is where get_next_buffer() will read from next. */
  inline void set_buf_pos(const size_t p_buf_pos) { buf_pos = p_buf_pos; }
  
  /** Adds the specified JSON token to end of the buffer. 
    * @param p_token [in] Token type
    * @param p_token_str [in] The name of a JSON object field (without quotes), or
    * the string representation of a JSON number, or a JSON string (with quotes 
    * and double-escaped). For all the other tokens this parameter will be ignored.
    * @return The number of characters added to the JSON document */
  int put_next_token(json_token_t p_token, const char* p_token_str = 0);
  
  /** Adds raw data to the end of the buffer.
    * @param p_data [in] Pointer to the beginning of the data
    * @param p_len [in] Length of the data in bytes */
  void put_raw_data(const char* p_data, size_t p_len);
  
  /** Attempts to find a JSON number at the current buffer position.
  * For number format see http://json.org/.
  * Returns true if a valid number is found before the end of the buffer
  * is reached, otherwise returns false.
  * is_float variable is true if the number is a float. The result
  * should be used only if this function returns true. */
  bool check_for_number(bool* is_float = NULL);
  
}; // class JSON_Tokenizer

// A dummy JSON tokenizer, use when there is no actual JSON document
static JSON_Tokenizer DUMMY_BUFFER;

/** Converts a string into a JSON string by replacing all control characters
  * with JSON escape sequences, if available, or with the \uHHHH escape sequence.
  * The string is also wrapped inside a set of double quotes and all double quotes
  * and backslash characters are double-escaped.
  *
  * Returns an expstring, that needs to be freed. */
extern char* convert_to_json_string(const char* str);


#endif	/* JSON_TOKENIZER_HH */