1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
|
/* SPDX-License-Identifier: MIT */
/* Copyright © 2021 Max Bachmann */
#pragma once
#include <limits>
#include <rapidfuzz/details/common.hpp>
#include <rapidfuzz/distance/OSA_impl.hpp>
namespace rapidfuzz {
/**
* @brief Calculates the optimal string alignment (OSA) distance between two strings.
*
* @details
* Both strings require a similar length
*
*
* @tparam Sentence1 This is a string that can be converted to
* basic_string_view<char_type>
* @tparam Sentence2 This is a string that can be converted to
* basic_string_view<char_type>
*
* @param s1
* string to compare with s2 (for type info check Template parameters above)
* @param s2
* string to compare with s1 (for type info check Template parameters above)
* @param max
* Maximum OSA distance between s1 and s2, that is
* considered as a result. If the distance is bigger than max,
* max + 1 is returned instead. Default is std::numeric_limits<size_t>::max(),
* which deactivates this behaviour.
*
* @return OSA distance between s1 and s2
*/
template <typename InputIt1, typename InputIt2>
size_t osa_distance(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2,
size_t score_cutoff = std::numeric_limits<size_t>::max())
{
return detail::OSA::distance(first1, last1, first2, last2, score_cutoff, score_cutoff);
}
template <typename Sentence1, typename Sentence2>
size_t osa_distance(const Sentence1& s1, const Sentence2& s2,
size_t score_cutoff = std::numeric_limits<size_t>::max())
{
return detail::OSA::distance(s1, s2, score_cutoff, score_cutoff);
}
template <typename InputIt1, typename InputIt2>
size_t osa_similarity(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2,
size_t score_cutoff = 0)
{
return detail::OSA::similarity(first1, last1, first2, last2, score_cutoff, score_cutoff);
}
template <typename Sentence1, typename Sentence2>
size_t osa_similarity(const Sentence1& s1, const Sentence2& s2, size_t score_cutoff = 0)
{
return detail::OSA::similarity(s1, s2, score_cutoff, score_cutoff);
}
template <typename InputIt1, typename InputIt2>
double osa_normalized_distance(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2,
double score_cutoff = 1.0)
{
return detail::OSA::normalized_distance(first1, last1, first2, last2, score_cutoff, score_cutoff);
}
template <typename Sentence1, typename Sentence2>
double osa_normalized_distance(const Sentence1& s1, const Sentence2& s2, double score_cutoff = 1.0)
{
return detail::OSA::normalized_distance(s1, s2, score_cutoff, score_cutoff);
}
/**
* @brief Calculates a normalized hamming similarity
*
* @details
* Both string require a similar length
*
*
* @tparam Sentence1 This is a string that can be converted to
* basic_string_view<char_type>
* @tparam Sentence2 This is a string that can be converted to
* basic_string_view<char_type>
*
* @param s1
* string to compare with s2 (for type info check Template parameters above)
* @param s2
* string to compare with s1 (for type info check Template parameters above)
* @param score_cutoff
* Optional argument for a score threshold as a float between 0 and 1.0.
* For ratio < score_cutoff 0 is returned instead. Default is 0,
* which deactivates this behaviour.
*
* @return Normalized hamming distance between s1 and s2
* as a float between 0 and 1.0
*/
template <typename InputIt1, typename InputIt2>
double osa_normalized_similarity(InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2,
double score_cutoff = 0.0)
{
return detail::OSA::normalized_similarity(first1, last1, first2, last2, score_cutoff, score_cutoff);
}
template <typename Sentence1, typename Sentence2>
double osa_normalized_similarity(const Sentence1& s1, const Sentence2& s2, double score_cutoff = 0.0)
{
return detail::OSA::normalized_similarity(s1, s2, score_cutoff, score_cutoff);
}
#ifdef RAPIDFUZZ_SIMD
namespace experimental {
template <int MaxLen>
struct MultiOSA
: public detail::MultiDistanceBase<MultiOSA<MaxLen>, size_t, 0, std::numeric_limits<int64_t>::max()> {
private:
friend detail::MultiDistanceBase<MultiOSA<MaxLen>, size_t, 0, std::numeric_limits<int64_t>::max()>;
friend detail::MultiNormalizedMetricBase<MultiOSA<MaxLen>, size_t>;
RAPIDFUZZ_CONSTEXPR_CXX14 static size_t get_vec_size()
{
# ifdef RAPIDFUZZ_AVX2
using namespace detail::simd_avx2;
# else
using namespace detail::simd_sse2;
# endif
RAPIDFUZZ_IF_CONSTEXPR (MaxLen <= 8)
return native_simd<uint8_t>::size;
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen <= 16)
return native_simd<uint16_t>::size;
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen <= 32)
return native_simd<uint32_t>::size;
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen <= 64)
return native_simd<uint64_t>::size;
static_assert(MaxLen <= 64, "expected MaxLen <= 64");
}
static size_t find_block_count(size_t count)
{
size_t vec_size = get_vec_size();
size_t simd_vec_count = detail::ceil_div(count, vec_size);
return detail::ceil_div(simd_vec_count * vec_size * MaxLen, 64);
}
public:
MultiOSA(size_t count) : input_count(count), PM(find_block_count(count) * 64)
{
str_lens.resize(result_count());
}
/**
* @brief get minimum size required for result vectors passed into
* - distance
* - similarity
* - normalized_distance
* - normalized_similarity
*
* @return minimum vector size
*/
size_t result_count() const
{
size_t vec_size = get_vec_size();
size_t simd_vec_count = detail::ceil_div(input_count, vec_size);
return simd_vec_count * vec_size;
}
template <typename Sentence1>
void insert(const Sentence1& s1_)
{
insert(detail::to_begin(s1_), detail::to_end(s1_));
}
template <typename InputIt1>
void insert(InputIt1 first1, InputIt1 last1)
{
auto len = std::distance(first1, last1);
int block_pos = static_cast<int>((pos * MaxLen) % 64);
auto block = (pos * MaxLen) / 64;
assert(len <= MaxLen);
if (pos >= input_count) throw std::invalid_argument("out of bounds insert");
str_lens[pos] = static_cast<size_t>(len);
for (; first1 != last1; ++first1) {
PM.insert(block, *first1, block_pos);
block_pos++;
}
pos++;
}
private:
template <typename InputIt2>
void _distance(size_t* scores, size_t score_count, const detail::Range<InputIt2>& s2,
size_t score_cutoff = std::numeric_limits<size_t>::max()) const
{
if (score_count < result_count())
throw std::invalid_argument("scores has to have >= result_count() elements");
auto scores_ = detail::make_range(scores, scores + score_count);
RAPIDFUZZ_IF_CONSTEXPR (MaxLen == 8)
detail::osa_hyrroe2003_simd<uint8_t>(scores_, PM, str_lens, s2, score_cutoff);
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen == 16)
detail::osa_hyrroe2003_simd<uint16_t>(scores_, PM, str_lens, s2, score_cutoff);
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen == 32)
detail::osa_hyrroe2003_simd<uint32_t>(scores_, PM, str_lens, s2, score_cutoff);
else RAPIDFUZZ_IF_CONSTEXPR (MaxLen == 64)
detail::osa_hyrroe2003_simd<uint64_t>(scores_, PM, str_lens, s2, score_cutoff);
}
template <typename InputIt2>
size_t maximum(size_t s1_idx, const detail::Range<InputIt2>& s2) const
{
return std::max(str_lens[s1_idx], s2.size());
}
size_t get_input_count() const noexcept
{
return input_count;
}
size_t input_count;
size_t pos = 0;
detail::BlockPatternMatchVector PM;
std::vector<size_t> str_lens;
};
} /* namespace experimental */
#endif
template <typename CharT1>
struct CachedOSA
: public detail::CachedDistanceBase<CachedOSA<CharT1>, size_t, 0, std::numeric_limits<int64_t>::max()> {
template <typename Sentence1>
explicit CachedOSA(const Sentence1& s1_) : CachedOSA(detail::to_begin(s1_), detail::to_end(s1_))
{}
template <typename InputIt1>
CachedOSA(InputIt1 first1, InputIt1 last1) : s1(first1, last1), PM(detail::make_range(first1, last1))
{}
private:
friend detail::CachedDistanceBase<CachedOSA<CharT1>, size_t, 0, std::numeric_limits<int64_t>::max()>;
friend detail::CachedNormalizedMetricBase<CachedOSA<CharT1>>;
template <typename InputIt2>
size_t maximum(const detail::Range<InputIt2>& s2) const
{
return std::max(s1.size(), s2.size());
}
template <typename InputIt2>
size_t _distance(const detail::Range<InputIt2>& s2, size_t score_cutoff, size_t) const
{
size_t res;
if (s1.empty())
res = s2.size();
else if (s2.empty())
res = s1.size();
else if (s1.size() < 64)
res = detail::osa_hyrroe2003(PM, detail::make_range(s1), s2, score_cutoff);
else
res = detail::osa_hyrroe2003_block(PM, detail::make_range(s1), s2, score_cutoff);
return (res <= score_cutoff) ? res : score_cutoff + 1;
}
std::vector<CharT1> s1;
detail::BlockPatternMatchVector PM;
};
#ifdef RAPIDFUZZ_DEDUCTION_GUIDES
template <typename Sentence1>
CachedOSA(const Sentence1& s1_) -> CachedOSA<char_type<Sentence1>>;
template <typename InputIt1>
CachedOSA(InputIt1 first1, InputIt1 last1) -> CachedOSA<iter_value_t<InputIt1>>;
#endif
/**@}*/
} // namespace rapidfuzz
|