1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
plugin_register language_model/knn
[[0,0.0,0.0],true]
table_create Data TABLE_NO_KEY
[[0,0.0,0.0],true]
column_create Data text COLUMN_SCALAR ShortText
[[0,0.0,0.0],true]
column_create Data rabitq_code COLUMN_SCALAR ShortBinary
[[0,0.0,0.0],true]
load --table Data
[
{"text": "I am a king."},
{"text": "I am a queen."}
]
[[0,0.0,0.0],2]
table_create RaBitQ TABLE_HASH_KEY ShortBinary --default_tokenizer 'TokenLanguageModelKNN("model", "hf:///groonga/multilingual-e5-base-Q4_K_M-GGUF", "code_column", "rabitq_code", "passage_prefix", "passage: ", "query_prefix", "query: ")'
[[0,0.0,0.0],true]
column_create RaBitQ data_text COLUMN_INDEX Data text
[[0,0.0,0.0],true]
#|w| load: model vocab missing newline token, using special_pad_id instead
#|w| load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
load --table Data
[
{"text": "I am a dog."}
]
[[0,0.0,0.0],1]
select Data --filter 'language_model_knn(text, "pet")' --output_columns text
[
[
0,
0.0,
0.0
],
[
[
[
3
],
[
[
"text",
"ShortText"
]
],
[
"I am a dog."
],
[
"I am a king."
],
[
"I am a queen."
]
]
]
]
|
