File: StringTable.cpp

package info (click to toggle)
freeorion 0.5.1-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 194,940 kB
  • sloc: cpp: 186,508; python: 40,969; ansic: 1,164; xml: 719; makefile: 32; sh: 7
file content (360 lines) | stat: -rw-r--r-- 17,042 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
#include "StringTable.h"

#include "Logger.h"
#include "Directories.h"
#include "../parse/Parse.h"
#include "ranges.h"

#include <boost/filesystem/fstream.hpp>
#include <boost/xpressive/xpressive.hpp>
#include <boost/algorithm/string/replace.hpp>

#include <iostream>
#include <atomic>

#if defined(_MSC_VER) && _MSC_VER >= 1930
struct IUnknown; // Workaround for "combaseapi.h(229,21): error C2760: syntax error: 'identifier' was unexpected here; expected 'type specifier'"
#endif

// define needed on Windows due to conflict with windows.h and std::min and std::max
#ifndef NOMINMAX
#  define NOMINMAX
#endif
// define needed in GCC
#ifndef _GNU_SOURCE
#  define _GNU_SOURCE
#endif

#include <boost/stacktrace.hpp>


namespace {
#if defined(__cpp_lib_constexpr_string) && ((!defined(__GNUC__) || (__GNUC__ > 12) || (__GNUC__ == 12 && __GNUC_MINOR__ >= 2))) && ((!defined(_MSC_VER) || (_MSC_VER >= 1934))) && ((!defined(__clang_major__) || (__clang_major__ >= 17)))
    constexpr std::string EMPTY_STRING;
#else
    const std::string EMPTY_STRING;
#endif
}

StringTable::StringTable(std::string filename, std::shared_ptr<const StringTable> fallback):
    m_filename(std::move(filename))
{ Load(std::move(fallback)); }

#if BOOST_VERSION >= 107900
bool StringTable::StringExists(const std::string& key) const
{ return m_strings.contains(key); }

bool StringTable::StringExists(const std::string_view key) const
{ return m_strings.contains(key); }

bool StringTable::StringExists(const char* key) const
{ return m_strings.contains(key); }

std::pair<bool, const std::string&> StringTable::CheckGet(const std::string& key) const {
    auto it = m_strings.find(key);
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}

std::pair<bool, const std::string&> StringTable::CheckGet(const std::string_view key) const {
    auto it = m_strings.find(key);
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}

std::pair<bool, const std::string&> StringTable::CheckGet(const char* key) const {
    auto it = m_strings.find(key);
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}

#else

bool StringTable::StringExists(const std::string& key) const
{ return m_strings.find(key, m_strings.hash_function(), m_strings.key_eq()) != m_strings.end(); }

bool StringTable::StringExists(const std::string_view key) const
{ return m_strings.find(key, m_strings.hash_function(), m_strings.key_eq()) != m_strings.end(); }

bool StringTable::StringExists(const char* key) const
{ return m_strings.find(key, m_strings.hash_function(), m_strings.key_eq()) != m_strings.end(); }

std::pair<bool, const std::string&> StringTable::CheckGet(const std::string& key) const {
    auto it = m_strings.find(key, m_strings.hash_function(), m_strings.key_eq());
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}

std::pair<bool, const std::string&> StringTable::CheckGet(const std::string_view key) const {
    auto it = m_strings.find(key, m_strings.hash_function(), m_strings.key_eq());
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}

std::pair<bool, const std::string&> StringTable::CheckGet(const char* key) const {
    auto it = m_strings.find(key, m_strings.hash_function(), m_strings.key_eq());
    bool found_string = it != m_strings.end();
    return {found_string, found_string ? it->second : EMPTY_STRING};
}
#endif

const std::string& StringTable::Add(std::string key, std::string value)
{ return m_strings.emplace(std::move(key), std::move(value)).first->second; }

namespace {
    std::string_view MatchLookupKey(const boost::xpressive::smatch& match, std::size_t idx) {
        //return match[idx].str(); // constructs a std::string, which should be avoidable for lookup purposes...
        const auto& m{match[idx]};
        return {&*m.first, static_cast<std::size_t>(std::max(0, static_cast<int>(m.length())))};
    }

    using namespace boost::xpressive;

    const sregex IDENTIFIER = +_w;
    const sregex COMMENT = '#' >> *(~_n) >> _n;
    const sregex KEY = IDENTIFIER;
    const sregex SINGLE_LINE_VALUE = *(~_n);
    const sregex MULTI_LINE_VALUE = -*_;

    const sregex ENTRY =
        keep(*(space | keep(+COMMENT))) >>
        KEY >> *blank >> (_n | COMMENT) >>
        (("'''" >> MULTI_LINE_VALUE >> "'''" >> *space >> _n) | SINGLE_LINE_VALUE >> _n);

    const sregex TRAILING_WS =
        *(space | COMMENT);

    const sregex REFERENCE =
        keep("[[" >> (s1 = IDENTIFIER) >> +space >> (s2 = IDENTIFIER) >> "]]");

    const sregex KEYEXPANSION =
        keep("[[" >> (s1 = IDENTIFIER) >> "]]");
}

void StringTable::Load(std::shared_ptr<const StringTable> fallback) {
    if (fallback && !fallback->m_initialized) {
        [[unlikely]]
        // this prevents deadlock if two stringtables were to be loaded
        // simultaneously with eachother as fallback tables
        ErrorLogger() << "StringTable::Load given uninitialized stringtable as fallback. Ignoring.";
        fallback = nullptr;
    }

    auto path = FilenameToPath(m_filename);
    std::string file_contents;

    bool read_success = ReadFile(path, file_contents);
    if (!read_success) {
        [[unlikely]]
        ErrorLogger() << "StringTable::Load failed to read file at path: " << m_filename;
        //m_initialized intentionally left false
        return;
    }
    // add newline at end to avoid errors when one is left out, but is expected by parsers
    file_contents += "\n";

    parse::file_substitution(file_contents, path.parent_path(), m_filename);

    decltype(fallback->m_strings) fallback_lookup_strings;
    std::string fallback_table_file;
    if (fallback) {
        fallback_table_file = fallback->Filename();
        fallback_lookup_strings = fallback->m_strings; //.insert(fallback->m_strings.begin(), fallback->m_strings.end());
    }

    using boost::xpressive::smatch;

    // parse input text stream
    auto it = file_contents.begin();
    const auto end = file_contents.end();

    smatch matches;
    bool well_formed = false;
    std::string key, prev_key;

    try {
        // grab first line of file, which should be the name of this language
        well_formed = regex_search(it, end, matches, SINGLE_LINE_VALUE, regex_constants::match_continuous);
        it = end - matches.suffix().length();
        if (well_formed)
            m_language = matches.str(0);

        // match series of key-value entries to store as stringtable
        while (well_formed) {
            well_formed = regex_search(it, end, matches, ENTRY, regex_constants::match_continuous);
            it = end - matches.suffix().length();

            if (well_formed) {
                for (const auto& match : matches.nested_results()) {
                    if (match.regex_id() == KEY.regex_id()) {
                        key = match.str();

                    } else if (match.regex_id() == SINGLE_LINE_VALUE.regex_id() ||
                               match.regex_id() == MULTI_LINE_VALUE.regex_id())
                    {
                        assert(key != "");
                        const bool has_key = [](const auto& strings, const auto& key) {
                            if constexpr (requires { strings.contains(key); })
                                return strings.contains(key);
                            else
                                return strings.count(key) != 0;
                        }(m_strings, key);

                        if (!has_key) {
                            m_strings[key] = match.str();
                            boost::algorithm::replace_all(m_strings[key], "\\n", "\n");
                        } else {
                            ErrorLogger() << "Duplicate string ID found: '" << key
                                          << "' in file: '" << m_filename
                                          << "'.  Ignoring duplicate.";
                        }
                        prev_key = std::move(key);
                        key.clear();
                    }
                }
            }
        }

        regex_search(it, end, matches, TRAILING_WS, regex_constants::match_continuous);
        it = end - matches.suffix().length();

        well_formed = it == end;

    } catch (const std::exception& e) {
        ErrorLogger() << "Exception caught regex parsing Stringtable: " << e.what();
        ErrorLogger() << "Last and prior keys matched: " << key << ", " << prev_key;
        std::cerr << "Exception caught regex parsing Stringtable: " << e.what() << std::endl;
        std::cerr << "Last and prior keys matched: " << key << ", " << prev_key << std::endl;
        m_initialized = true;
        return;
    }

    if (well_formed) {
        // recursively expand keys -- replace [[KEY]] by the text resulting from expanding everything in the definition for KEY
        for (auto& [loop_key, user_read_entry] : m_strings) {
            //DebugLogger() << "Checking key expansion for: " << key;
            std::size_t position = 0; // position in the definition string, past the already processed part
            smatch match;
            std::map<std::string, std::size_t> cyclic_reference_check;
            cyclic_reference_check[loop_key] = user_read_entry.length();
            std::string rawtext = user_read_entry;
            std::string cumulative_subsititions;

            while (regex_search(user_read_entry.begin() + position, user_read_entry.end(), match, KEYEXPANSION)) {
                position += match.position();
                //DebugLogger() << "checking next internal keyword match: " << match[1] << " with matchlen " << match.length();
                if (match[1].length() != match.length() - 4)
                    ErrorLogger() << "Positional error in key expansion: " << match[1] << " with length: "
                                  << match[1].length() << "and matchlen: " << match.length();
                // clear out any keywords that have been fully processed
                for (auto ref_check_it = cyclic_reference_check.begin();
                     ref_check_it != cyclic_reference_check.end(); )
                {
                    if (ref_check_it->second <= position) {
                        //DebugLogger() << "Popping from cyclic ref check: " << ref_check_it->first;
                        ref_check_it = cyclic_reference_check.erase(ref_check_it);
                    } else if (ref_check_it->second < position + match.length()) {
                        ErrorLogger() << "Expansion error in key expansion: [[" << ref_check_it->first << "]] having end " << ref_check_it->second;
                        ErrorLogger() << "         currently at expansion text position " << position << " with match length: " << match.length();
                        ErrorLogger() << "         of current expansion text: " << user_read_entry;
                        ErrorLogger() << "         from keyword "<< loop_key << " with raw text: " << rawtext;
                        ErrorLogger() << "         and cumulative substitions: " << cumulative_subsititions;
                        // will also trigger further error logging below
                        ++ref_check_it;
                    } else
                        ++ref_check_it;
                }
                if (!cyclic_reference_check.contains(match[1])) {
                    //DebugLogger() << "Pushing to cyclic ref check: " << match[1];
                    cyclic_reference_check[match[1]] = position + match.length();

#if BOOST_VERSION >= 107900
                    auto map_lookup_it = m_strings.find(MatchLookupKey(match, 1u));
#else
                    auto map_lookup_it = m_strings.find(MatchLookupKey(match, 1u), m_strings.hash_function(),
                                                        m_strings.key_eq());
#endif
                    bool foundmatch = map_lookup_it != m_strings.end();
                    if (!foundmatch && !fallback_lookup_strings.empty()) {
                        DebugLogger() << "Key expansion: " << match[1] << " not found in primary stringtable: " << m_filename
                                      << "; checking in fallback file: " << fallback_table_file;
#if BOOST_VERSION >= 107900
                        map_lookup_it = fallback_lookup_strings.find(MatchLookupKey(match, 1u));
#else
                        map_lookup_it = fallback_lookup_strings.find(
                            MatchLookupKey(match, 1u), m_strings.hash_function(), m_strings.key_eq());
#endif
                        foundmatch = map_lookup_it != fallback_lookup_strings.end();
                    }
                    if (foundmatch) {
                        const std::string& substitution = map_lookup_it->second;
                        cumulative_subsititions += substitution + "|**|";
                        user_read_entry.replace(position, match.length(), substitution);
                        std::size_t added_chars = substitution.length() - match.length();
                        for (auto& ref_check : cyclic_reference_check)
                            ref_check.second += added_chars;
                        // replace recursively -- do not skip past substitution
                    } else {
                        ErrorLogger() << "Unresolved key expansion: " << match[1] << " in: " << m_filename << ".";
                        position += match.length();
                    }
                } else {
                    ErrorLogger() << "Cyclic key expansion: " << match[1] << " in: " << m_filename << "."
                                  << "         at expansion text position " << position;
                    ErrorLogger() << "         of current expansion text: " << user_read_entry;
                    ErrorLogger() << "         from keyword "<< loop_key << " with raw text: " << rawtext;
                    ErrorLogger() << "         and cumulative substitions: " << cumulative_subsititions;
                    position += match.length();
                }
            }
        }

        // nonrecursively replace references -- convert [[type REF]] to <type REF>string for REF</type>
        for (auto& user_read_entry : m_strings | range_values) {
            std::size_t position = 0; // position in the definition string, past the already processed part
            smatch match;
            while (regex_search(user_read_entry.begin() + position, user_read_entry.end(), match, REFERENCE)) {
                position += match.position();
#if BOOST_VERSION >= 107900
                auto map_lookup_it = m_strings.find(MatchLookupKey(match, 2u));
#else
                auto map_lookup_it = m_strings.find(MatchLookupKey(match, 2u), m_strings.hash_function(),
                                                    m_strings.key_eq());
#endif
                bool foundmatch = map_lookup_it != m_strings.end();
                if (!foundmatch && !fallback_lookup_strings.empty()) {
                    DebugLogger() << "Key reference: " << match[2] << " not found in primary stringtable: " << m_filename
                                  << "; checking in fallback file: " << fallback_table_file;
#if BOOST_VERSION >= 107900
                    map_lookup_it = fallback_lookup_strings.find(MatchLookupKey(match, 2u));
#else
                    map_lookup_it = fallback_lookup_strings.find(
                        MatchLookupKey(match, 2u), m_strings.hash_function(), m_strings.key_eq());
#endif
                    foundmatch = map_lookup_it != fallback_lookup_strings.end();
                }
                if (foundmatch) {
                    const std::string substitution =
                        '<' + match[1].str() + ' ' + match[2].str() + '>' +
                        map_lookup_it->second + "</" + match[1].str() + '>';
                    user_read_entry.replace(position, match.length(), substitution);
                    position += substitution.length();
                } else {
                    if (match[1] == "value") {
                        TraceLogger() << "Unresolved optional value reference: " << match[2] << " in: " << m_filename << ".";
                        const std::string substitution = "<value " + match[2].str() + "></value>";
                        user_read_entry.replace(position, match.length(), substitution);
                        position += substitution.length();
                    } else {
                        ErrorLogger() << "Unresolved reference: " << match[2] << " in: " << m_filename << ".";
                        position += match.length();
                    }
                }
            }
        }
    } else {
        ErrorLogger() << "StringTable file \"" << m_filename << "\" is malformed around line " << std::count(file_contents.begin(), it, '\n');
    }

    m_initialized = true;
}