1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
# 2024 Aug 10
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer3
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
proc get_sod {args} { return "split_on_dot" }
proc get_lowercase {args} { return "lowercase" }
proc lowercase {flags txt} {
set n [string length $txt]
sqlite3_fts5_token [string tolower $txt] 0 $n
return 0
}
proc split_on_dot {flags txt} {
set iOff 0
foreach t [split $txt "."] {
set n [string length $txt]
sqlite3_fts5_token $t $iOff [expr $iOff+$n]
incr iOff [expr {$n+1}]
}
return ""
}
foreach {tn script} {
1 {
sqlite3_fts5_create_tokenizer db lowercase get_lowercase
sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
}
2 {
sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
}
3 {
sqlite3_fts5_create_tokenizer db lowercase get_lowercase
sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
}
4 {
sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
}
} {
reset_db
eval $script
do_execsql_test 1.$tn.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot);
CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance);
INSERT INTO t1 VALUES('ABC.Def.ghi');
}
do_execsql_test 1.$tn.1 {
SELECT term FROM t1vocab ORDER BY 1
} {abc def ghi}
}
finish_test
|
