File: construct.hpp

package info (click to toggle)
seqan3 3.0.2%2Bds-9
  • links: PTS, VCS
  • area: main
  • in suites: bullseye
  • size: 16,052 kB
  • sloc: cpp: 144,641; makefile: 1,288; ansic: 294; sh: 228; xml: 217; javascript: 50; python: 27; php: 25
file content (281 lines) | stat: -rw-r--r-- 9,225 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
// Copyright (c) 2016, the SDSL Project Authors.  All rights reserved.
// Please see the AUTHORS file for details.  Use of this source code is governed
// by a BSD license that can be found in the LICENSE file.
/*! \file construct.hpp
    \brief construct.hpp contains methods to construct indexes (compressed suffix arrays and trees).
	\author Simon Gog
*/

#ifndef INCLUDED_SDSL_CONSTRUCT
#define INCLUDED_SDSL_CONSTRUCT

#include "sdsl_concepts.hpp"
#include "int_vector.hpp"
#include "construct_lcp.hpp"
#include "construct_bwt.hpp"
#include "construct_sa.hpp"
#include <string>

namespace sdsl {

template <class int_vector>
bool contains_no_zero_symbol(const int_vector& text, const std::string& file)
{
	for (int_vector_size_type i = 0; i < text.size(); ++i) {
		if ((uint64_t)0 == text[i]) {
			throw std::logic_error(std::string("Error: File \"") + file +
								   "\" contains zero symbol.");
			return false;
		}
	}
	return true;
}

template <class int_vector>
void append_zero_symbol(int_vector& text)
{
	text.resize(text.size() + 1);
	text[text.size() - 1] = 0;
}


template <class t_index>
void construct(t_index& idx, std::string file, uint8_t num_bytes = 0, bool move_input = false)
{
	tMSS		 file_map;
	cache_config config;
	if (is_ram_file(file)) {
		config.dir		   = "@";
		config.delete_data = move_input;
	}
	construct(idx, file, config, num_bytes);
}

template <class t_index, class t_data>
void construct_im(t_index& idx, t_data&& data, uint8_t num_bytes = 0)
{
	std::string tmp_file =
	ram_file_name(util::to_string(util::pid()) + "_" + util::to_string(util::id()));
	store_to_file(data, tmp_file);
	construct(idx, tmp_file, num_bytes, std::is_rvalue_reference<t_data&&>::value);
	ram_fs::remove(tmp_file);
}

//! Constructs an index object of type t_index for a text stored on disk.
/*!
 * \param idx       	t_index object.  Any sdsl suffix array of suffix tree.
 * \param file      	Name of the text file. The representation of the file
 *                  	is dependent on the next parameter.
 * \
 * \param num_bytes 	If `num_bytes` equals 0, the file format is a serialized
 *				    	int_vector<>. Otherwise the file is interpreted as sequence
 *                  	of `num_bytes`-byte integer stored in big endian order.
 */
template <class t_index>
void construct(t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes = 0)
{
	// delegate to CSA or CST construction
	typename t_index::index_category index_tag;
	construct(idx, file, config, num_bytes, index_tag);
}

// Specialization for WTs
template <class t_index>
void construct(
t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, wt_tag)
{
	auto event = memory_monitor::event("construct wavelet tree");
	if ((t_index::alphabet_category::WIDTH == 8 and num_bytes <= 1) or
		(t_index::alphabet_category::WIDTH == 0 and num_bytes != 'd')) {
		int_vector_buffer<t_index::alphabet_category::WIDTH> text_buf(
		file, std::ios::in, 1024 * 1024, num_bytes * 8, (bool)num_bytes);
		idx = t_index(text_buf.begin(), text_buf.end(), config.dir);
	} else {
		int_vector<t_index::alphabet_category::WIDTH> text;
		load_vector_from_file(text, file, num_bytes);
		std::string tmp_key = util::to_string(util::pid()) + "_" + util::to_string(util::id());
		std::string tmp_file_name = cache_file_name(tmp_key, config);
		store_to_file(text, tmp_file_name);
		util::clear(text);
		{
			int_vector_buffer<t_index::alphabet_category::WIDTH> text_buf(tmp_file_name);
			idx = t_index(text_buf.begin(), text_buf.end(), config.dir);
		}
		sdsl::remove(tmp_file_name);
	}
}

// Specialization for CSAs
template <class t_index>
void construct(
t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, csa_tag)
{
	auto			  event	= memory_monitor::event("construct CSA");
	constexpr uint8_t width	= t_index::alphabet_category::WIDTH;
	const char*		  KEY_TEXT = key_text_trait<width>::KEY_TEXT;
	const char*		  KEY_BWT  = key_bwt_trait<width>::KEY_BWT;
	typedef int_vector<t_index::alphabet_category::WIDTH> text_type;
	{
		auto event = memory_monitor::event("parse input text");
		// (1) check, if the text is cached
		if (!cache_file_exists(KEY_TEXT, config)) {
			text_type text;
			load_vector_from_file(text, file, num_bytes);
			if (contains_no_zero_symbol(text, file)) {
				if (!is_ram_file(file)) {
					append_zero_symbol(text);
					store_to_cache(text, KEY_TEXT, config);
				} else {
					auto text_mapper = write_out_mapper<width>::create(
					cache_file_name(KEY_TEXT, config), text.size() + 1, text.width());
					std::copy(text.begin(), text.end(), text_mapper.begin());
					text_mapper[text.size()] = 0;
				}
			}
		}
		register_cache_file(KEY_TEXT, config);
	}
	if (config.delete_data) {
		sdsl::remove(file);
	}
	{
		// (2) check, if the suffix array is cached
		auto event = memory_monitor::event("SA");
		if (!cache_file_exists(conf::KEY_SA, config)) {
			construct_sa<t_index::alphabet_category::WIDTH>(config);
		}
		register_cache_file(conf::KEY_SA, config);
	}
	{
		//  (3) construct BWT
		auto event = memory_monitor::event("BWT");
		if (!cache_file_exists(KEY_BWT, config)) {
			construct_bwt<t_index::alphabet_category::WIDTH>(config);
		}
		register_cache_file(KEY_BWT, config);
	}
	{
		//  (4) use BWT to construct the CSA
		auto event = memory_monitor::event("construct CSA");
		idx		   = t_index(config);
	}
	if (config.delete_files) {
		auto event = memory_monitor::event("delete temporary files");
		util::delete_all_files(config.file_map);
	}
}

// Specialization for standalone LCPs
template <class t_index, uint8_t t_width>
void construct(
t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, lcp_tag)
{
	auto						event	= memory_monitor::event("construct compressed LCP");
	const char*					KEY_TEXT = key_text_trait<t_width>::KEY_TEXT;
	typedef int_vector<t_width> text_type;
	{
		// (2) check, if the longest common prefix array is cached
		auto event = memory_monitor::event("LCP");
		if (!cache_file_exists(conf::KEY_LCP, config)) {
			{
				auto event = memory_monitor::event("parse input text");
				// (1) check, if the text is cached
				if (!cache_file_exists(KEY_TEXT, config)) {
					text_type text;
					load_vector_from_file(text, file, num_bytes);
					if (contains_no_zero_symbol(text, file)) {
						append_zero_symbol(text);
						store_to_cache(text, KEY_TEXT, config);
					}
				}
				register_cache_file(KEY_TEXT, config);
			}
			{
				// (2) check, if the suffix array is cached
				auto event = memory_monitor::event("SA");
				if (!cache_file_exists(conf::KEY_SA, config)) {
					construct_sa<t_width>(config);
				}
				register_cache_file(conf::KEY_SA, config);
			}
			if (t_width == 8) {
				construct_lcp_semi_extern_PHI(config);
			} else {
				construct_lcp_PHI<t_width>(config);
			}
		}
		register_cache_file(conf::KEY_LCP, config);
	}
	{
		auto event = memory_monitor::event("compressed LCP");
		idx		   = t_index(config);
	}
	if (config.delete_files) {
		auto event = memory_monitor::event("delete temporary files");
		util::delete_all_files(config.file_map);
	}
}

// Specialization for standalone LCPs
template <class t_index>
void construct(
t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, lcp_tag tag)
{
	if (1 == num_bytes) {
		construct<t_index, 8>(idx, file, config, num_bytes, tag);
	} else {
		construct<t_index, 0>(idx, file, config, num_bytes, tag);
	}
}


// Specialization for CSTs
template <class t_index>
void construct(
t_index& idx, const std::string& file, cache_config& config, uint8_t num_bytes, cst_tag)
{
	auto		event	= memory_monitor::event("construct CST");
	const char* KEY_TEXT = key_text_trait<t_index::alphabet_category::WIDTH>::KEY_TEXT;
	const char* KEY_BWT  = key_bwt_trait<t_index::alphabet_category::WIDTH>::KEY_BWT;
	csa_tag		csa_t;
	{
		// (1) check, if the compressed suffix array is cached
		typename t_index::csa_type csa;
		if (!cache_file_exists(std::string(conf::KEY_CSA) + "_" + util::class_to_hash(csa),
							   config)) {
			cache_config csa_config(false, config.dir, config.id, config.file_map);
			construct(csa, file, csa_config, num_bytes, csa_t);
			auto event		= memory_monitor::event("store CSA");
			config.file_map = csa_config.file_map;
			store_to_cache(
			csa, std::string(conf::KEY_CSA) + "_" + util::class_to_hash(csa), config);
		}
		register_cache_file(std::string(conf::KEY_CSA) + "_" + util::class_to_hash(csa), config);
	}
	{
		// (2) check, if the longest common prefix array is cached
		auto event = memory_monitor::event("LCP");
		register_cache_file(KEY_TEXT, config);
		register_cache_file(KEY_BWT, config);
		register_cache_file(conf::KEY_SA, config);
		if (!cache_file_exists(conf::KEY_LCP, config)) {
			if (t_index::alphabet_category::WIDTH == 8) {
				construct_lcp_semi_extern_PHI(config);
			} else {
				construct_lcp_PHI<t_index::alphabet_category::WIDTH>(config);
			}
		}
		register_cache_file(conf::KEY_LCP, config);
	}
	{
		auto event = memory_monitor::event("CST");
		idx		   = t_index(config);
	}
	if (config.delete_files) {
		auto event = memory_monitor::event("delete temporary files");
		util::delete_all_files(config.file_map);
	}
}

} // end namespace sdsl
#endif