1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
|
<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
<meta charset="utf-8" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Cyrus IMAP Server: Internationalization — Cyrus IMAP 3.12.1 documentation</title>
<link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/graphviz.css" type="text/css" />
<link rel="stylesheet" href="../../../_static/cyrus.css" type="text/css" />
<script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
<script src="../../../_static/jquery.js"></script>
<script src="../../../_static/underscore.js"></script>
<script src="../../../_static/_sphinx_javascript_frameworks_compat.js"></script>
<script src="../../../_static/doctools.js"></script>
<script src="../../../_static/sphinx_highlight.js"></script>
<script src="../../../_static/js/theme.js"></script>
<link rel="index" title="Index" href="../../../genindex.html" />
<link rel="search" title="Search" href="../../../search.html" />
<link rel="next" title="Cyrus IMAP Server: Locking" href="locking.html" />
<link rel="prev" title="Cyrus IMAP Server: Hacking" href="hacking.html" />
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search" >
<a href="../../../index.html" class="icon icon-home">
Cyrus IMAP
</a>
<div class="version">
3.12.1
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
<p class="caption" role="heading"><span class="caption-text">Cyrus IMAP</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../../download.html">Download</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../quickstart.html">Quickstart Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../overview.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../setup.html">Setup</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../operations.html">Operations</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../../../developers.html">Developers</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../../../contribute.html">We need your help</a></li>
<li class="toctree-l2"><a class="reference internal" href="../documentation.html">Contribute docs</a></li>
<li class="toctree-l2"><a class="reference internal" href="../../developer.html">Contribute code and tests</a></li>
<li class="toctree-l2"><a class="reference internal" href="../cyrusworks.html">Cyrus.Works</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../../../developers.html#cyrus-internals">Cyrus Internals</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../API.html">Cyrus APIs</a></li>
<li class="toctree-l3"><a class="reference internal" href="../thoughts.html">Thoughts & Notes</a></li>
<li class="toctree-l3 current"><a class="reference internal" href="../guidance.html">Guidance for Developers</a><ul class="current">
<li class="toctree-l4"><a class="reference internal" href="hacking.html">Cyrus IMAP Server: Hacking</a></li>
<li class="toctree-l4 current"><a class="current reference internal" href="#">Cyrus IMAP Server: Internationalization</a></li>
<li class="toctree-l4"><a class="reference internal" href="locking.html">Cyrus IMAP Server: Locking</a></li>
<li class="toctree-l4"><a class="reference internal" href="mailbox-format.html">Cyrus IMAP Server: Mailbox File Formats</a></li>
<li class="toctree-l4"><a class="reference internal" href="namelocks.html">Cyrus IMAP Server: Namelocks</a></li>
<li class="toctree-l4"><a class="reference internal" href="prot.html">Cyrus IMAP Server: prot layer</a></li>
<li class="toctree-l4"><a class="reference internal" href="replication_examples.html">Cyrus IMAP Server: Replication Examples</a></li>
<li class="toctree-l4"><a class="reference internal" href="replication_protocol.html">Cyrus IMAP Server: Replication Protocol v2.4+</a></li>
<li class="toctree-l4"><a class="reference internal" href="special_chars.html">Cyrus IMAP Server: Special Characters</a></li>
<li class="toctree-l4"><a class="reference internal" href="var_directory_structure.html">Cyrus IMAP Server: var directory structure</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../../developers.html#unit-tests">Unit Tests</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../../support.html">Support/Community</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Cyrus SASL</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="http://www.cyrusimap.org/sasl">Cyrus SASL</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Cyrus IMAP</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="Page navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../../index.html" class="icon icon-home" aria-label="Home"></a></li>
<li class="breadcrumb-item"><a href="../../../developers.html">Developers</a></li>
<li class="breadcrumb-item"><a href="../guidance.html">Developer Guidance</a></li>
<li class="breadcrumb-item active">Cyrus IMAP Server: Internationalization</li>
<li class="wy-breadcrumbs-aside">
<a href="https://github.com/cyrusimap/cyrus-imapd/blob/master/docsrc/imap/developer/guidance/internationalization.rst" class="fa fa-github"> Edit on GitHub</a>
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<span class="target" id="imap-developer-guidance-internationalization"></span><section id="cyrus-imap-server-internationalization">
<h1>Cyrus IMAP Server: Internationalization<a class="headerlink" href="#cyrus-imap-server-internationalization" title="Permalink to this heading"></a></h1>
<section id="introduction">
<h2>introduction<a class="headerlink" href="#introduction" title="Permalink to this heading"></a></h2>
<p>Cyrus currently transcodes characters to a canonical UTF-8 form for
searching. The base spec of IMAP4 only requires understanding multiple
character sets to properly implement SEARCH. Since the base spec came
out, several extensions have been proposed that require further charset
support: SORT, THREAD, and the Sieve subsystem. As of this writing,
Cyrus doesn't correctly support these other commands.</p>
<p>Cyrus currently only believes in 16-bit characters. Technically, Unicode
has up to 21-bit characters (expressible in UTF-16 and 3-byte UTF-8) and
ISO 10646 allows up to 31-bit characters (though ISO's current policy is
to not allocate any characters outside of the 21-bit Unicode range). The
lower 16-bit characters make up the basic multilingual plane (BMP) where
the majority of languages live. This restriction is apparent in
<code class="docutils literal notranslate"><span class="pre">charset.c:writeutf8()</span></code>, the UTF-8 decoders, and the Unicode
canonicalization table used by Cyrus. Since Cyrus's known character sets
(except for UTF-8) don't contain any characters off of the BMP this
isn't seen to be a major problem.</p>
<p>Throughout this text, Unicode and ISO 10646 will be used interchangeable
to refer to the 16-bit character set of the BMP, regardless of encoding.
"Character", unless otherwise specified, refers to a single Unicode
character <code class="docutils literal notranslate"><span class="pre">ffff</span></code> or under.</p>
</section>
<section id="cyrus-canonical-form">
<h2>cyrus canonical form<a class="headerlink" href="#cyrus-canonical-form" title="Permalink to this heading"></a></h2>
<p>Since when users search e-mail messages it's much easier for them to
eliminate false positives than realize there are hits that aren't
displayed, the Cyrus searching algorithm errs on the side of more
matches. Before comparing any two strings, Cyrus puts them in a
canonical form. Logically, the process works as follows:</p>
<ul class="simple">
<li><p>the input string is translated into a sequence of Unicode characters.</p></li>
<li><p>each character is transformed into lowercase. (For some characters, a
single uppercase character may transform into multiple lowercase
characters.)</p></li>
<li><p>each character is fully decomposed.</p></li>
<li><p>all whitespace (Unicode general categories starting with <code class="docutils literal notranslate"><span class="pre">Z</span></code>) is
removed.</p></li>
<li><p>combining diacritical marks, such as the accent on é, are removed.
(These are Unicode characters <code class="docutils literal notranslate"><span class="pre">0300</span></code>-<code class="docutils literal notranslate"><span class="pre">03ff</span></code>.)</p></li>
<li><p>certain characters are expanded to alternative spellings using ASCII
characters, such as "æ" to "ae".</p></li>
<li><p>the output characters are then encoded in UTF-8.</p></li>
</ul>
<p>The actual transcoding does all of these steps at once with the aid of
tables, carefully built at compile-time.</p>
<p>The central part of Cyrus's internationalization support is it's
transcoding routines in <code class="docutils literal notranslate"><span class="pre">lib/charset.[ch]</span></code>, and
<code class="docutils literal notranslate"><span class="pre">lib/chartable.[ch]</span></code>. Cyrus's transcoding routines are very elegant
and very compact, thus somewhat intimidating. During compilation, Cyrus
builds up a large number of tables (see <a class="reference external" href="#mkchartable">mkchartable</a>)
and uses them so that it never has to consider more than a single octet
at a time while outputting the Cyrus canonical form for an input string.</p>
</section>
<section id="external-interface">
<h2>external interface<a class="headerlink" href="#external-interface" title="Permalink to this heading"></a></h2>
<p><code class="docutils literal notranslate"><span class="pre">lib/charset.h</span></code> is the public interface for Cyrus lib clients to get
character canonicalization and searching support. In contains the
following functions:</p>
<dl class="simple">
<dt><code class="docutils literal notranslate"><span class="pre">char</span> <span class="pre">*charset_convert(const</span> <span class="pre">char</span> <span class="pre">*s,</span> <span class="pre">int</span> <span class="pre">charset,</span> <span class="pre">char</span> <span class="pre">*buf,</span> <span class="pre">int</span> <span class="pre">bufsz)</span></code></dt><dd><p>Given a string <em>s</em> in charset <em>charset</em>, decode it into canonical
form in <em>buf</em>. <em>buf</em> must be reallocable and currently at least size
<em>bufsz</em>.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">char</span> <span class="pre">*charset_decode_mimeheader(const</span> <span class="pre">char</span> <span class="pre">*s,</span> <span class="pre">char</span> <span class="pre">*buf,</span> <span class="pre">int</span> <span class="pre">bufsz)</span></code></dt><dd><p>Given a string <em>s</em> containing possible MIME encoded substrings (per
RFC 2047), decode into canonical form in <em>buf</em>. <em>buf</em> must be
reallocable and currently at least size <em>bufsz</em>.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">charset_index</span> <span class="pre">charset_lookupname(const</span> <span class="pre">char</span> <span class="pre">*name)</span></code></dt><dd><p>Given <em>name</em> return the Cyrus charset index. 0 always represents
US-ASCII. The returned charset_index may be saved in a file; it is
stable and is an integer. If this version of Cyrus does not support
the charset, <code class="docutils literal notranslate"><span class="pre">CHARSET_UNKNOWN_CHARSET</span></code> is returned.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">comp_pat</span> <span class="pre">*charset_compilepat(const</span> <span class="pre">char</span> <span class="pre">*s)</span></code></dt><dd><p>Compiles a NUL-terminated canonicalized string <em>s</em> into a
Boyer-Moore table for fast searching. I'll describe these <a class="reference external" href="#comp_pat">compiled
patterns</a> later.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">void</span> <span class="pre">charset_freepat(comp_pat</span> <span class="pre">*pat)</span></code></dt><dd><p>Frees a pattern previously return by <code class="docutils literal notranslate"><span class="pre">charset_compilepat()</span></code>.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">int</span> <span class="pre">charset_searchstring(const</span> <span class="pre">char</span> <span class="pre">*substr,</span> <span class="pre">comp_pat</span> <span class="pre">*pat,</span>     <span class="pre">const</span> <span class="pre">char</span> <span class="pre">*s,</span> <span class="pre">int</span> <span class="pre">len)</span></code></dt><dd><p>Searches for a canonicalized string <em>substr</em> in the canonicalized
string <em>s</em>. <em>s</em> is of length <em>len</em>. <em>substr</em> must have been
previously compiled into <em>pat</em>. Returns non-zero for a hit, zero for
no match.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">int</span> <span class="pre">charset_searchfile(const</span> <span class="pre">char</span> <span class="pre">*substr,</span> <span class="pre">comp_pat</span> <span class="pre">*pat,</span>                               <span class="pre">const</span> <span class="pre">char</span> <span class="pre">*msg_base,</span> <span class="pre">int</span> <span class="pre">mapnl,</span> <span class="pre">int</span> <span class="pre">len,</span>                               <span class="pre">charset_index</span> <span class="pre">charset,</span> <span class="pre">int</span> <span class="pre">encoding)</span></code></dt><dd><p>Searches for the canonicalized string <em>substr</em> with compiled pattern
<em>pat</em> in a large buffer starting at <em>msg_base</em> of length <em>len</em>. The
large buffer is of charset <em>charset</em> with the encoding <em>encoding</em>.
<code class="docutils literal notranslate"><span class="pre">charset_searchfile()</span></code> will dynamically unencode and canonicalize
the search text looking for <em>substr</em>. (If <em>mapnl</em> is set, the buffer
has only <code class="docutils literal notranslate"><span class="pre">\n</span></code> instead of <code class="docutils literal notranslate"><span class="pre">\r\n</span></code>, but the length assumes that
each <code class="docutils literal notranslate"><span class="pre">\n</span></code> is dynamically converted to <code class="docutils literal notranslate"><span class="pre">\r\n</span></code>. This feature is
deprecated.)</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">char</span> <span class="pre">*charset_decode_mimebody(const</span> <span class="pre">char</span> <span class="pre">*msg_base,</span> <span class="pre">int</span> <span class="pre">len,</span>                                      <span class="pre">int</span> <span class="pre">encoding,</span> <span class="pre">char</span> <span class="pre">**buf,</span> <span class="pre">int</span> <span class="pre">*bufsz,</span>                                      <span class="pre">int</span> <span class="pre">*outlen)</span></code></dt><dd><p>Decode the MIME body part (per RFC 2045) located in the large buffer
starting at <em>msg_base</em> of length <em>len</em>. The large buffer is of
encoding <em>encoding</em>. <code class="docutils literal notranslate"><span class="pre">charset_decode_mimebody()</span></code> will decode into
<em>buf</em>. <em>buf</em> must be reallocable and currently at least size
<em>bufsz</em>. The number of decoded bytes is returned in <em>outlen</em>.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">charset_extractfile()</span></code></dt><dd><p>Used by <code class="docutils literal notranslate"><span class="pre">squatter</span></code> and possibly other text indexing engines, but
not described here.</p>
</dd>
</dl>
</section>
<section id="the-translate-macro-using-the-transcoding-tables">
<h2>the TRANSLATE macro: using the transcoding tables<a class="headerlink" href="#the-translate-macro-using-the-transcoding-tables" title="Permalink to this heading"></a></h2>
<p>The external interface is implemented with the help of the <code class="docutils literal notranslate"><span class="pre">START</span></code> and
<code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code> macros:</p>
<dl class="simple">
<dt><code class="docutils literal notranslate"><span class="pre">void</span> <span class="pre">START(struct</span> <span class="pre">decode_state</span> <span class="pre">*state,</span> <span class="pre">const</span> <span class="pre">unsigned</span> <span class="pre">char</span> <span class="pre">(*table)[256][4])</span></code></dt><dd><p><code class="docutils literal notranslate"><span class="pre">START</span></code> initializes <em>state</em> to be ready for transcoding of the
charset translation table given with <em>table</em>. The starting active
table is always the first one in the list passed in.</p>
</dd>
<dt><code class="docutils literal notranslate"><span class="pre">void</span> <span class="pre">TRANSLATE(struct</span> <span class="pre">decode_state</span> <span class="pre">*state,</span> <span class="pre">unsigned</span> <span class="pre">char</span> <span class="pre">input,</span> <span class="pre">unsigned</span> <span class="pre">char</span> <span class="pre">*outbuf,</span> <span class="pre">unsigned</span> <span class="pre">outindex)</span></code></dt><dd><p><code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code> takes four parameters: <em>state</em> is the current state of
the translation; it must have been initialized with <code class="docutils literal notranslate"><span class="pre">START</span></code> and is
modified by <code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code>; <em>input</em> is one octet of input from the
stream to be transcoded; <em>outbuf</em> is a pointer to the start of the
buffer to write output characters; <em>outindex</em> is the index where
this translation should be written. The size of <em>outbuf</em> must be at
least <em>outindex + charset_max_translation</em>.</p>
</dd>
</dl>
<p>Each charset consists of a set of one or more tables; the <em>table</em>
parameter passed into <code class="docutils literal notranslate"><span class="pre">START</span></code> is the first of these tables and the
others are adjacent in memory. Characters are transcoded by indexing
into the active table with <em>input</em> and examining the 4 octet
translation. The 4 octet translation may consist of 0–3 character
translations followed by a control code or a series of control codes. In
effect, the translation for a given octet is a mini-program that
consists either of UTF-8 octets or control codes. One of the control
codes RET, END, JSR, or JMP must occur in the 4 octet translation.</p>
<section id="control-codes">
<h3>control codes<a class="headerlink" href="#control-codes" title="Permalink to this heading"></a></h3>
<p>Control codes are represented by uppercase US-ASCII characters since no
uppercase characters can appear in the output translation (recall that
Cyrus canonical form downcases). Any uppercase US-ASCII character
(<code class="docutils literal notranslate"><span class="pre">[A</span> <span class="pre">..</span> <span class="pre">Z]</span></code>) is thus interpreted specially by the <code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code>
virtual machine. Any other octet encountered as an output translation is
presumed to be part of the UTF-8 output sequence and copied to the
output.</p>
<p>The names of control codes are actually C pre-processor defines to
uppercase US-ASCII characters. As the mnenomics are easier to
understand, I use them in discussing their semantics.</p>
</section>
<section id="control-code-reference">
<h3>control code reference<a class="headerlink" href="#control-code-reference" title="Permalink to this heading"></a></h3>
<p><code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code> recognizes the following "normal" control codes:</p>
<dl>
<dt>XLT</dt><dd><p>This is the first octet of the four octet sequence, indicating that
the desired translation is larger than 3 UTF-8 octets. The next two
octets represent an offset to look up in the special
chartables_long_translations[] table. After that translation is
copied to the outbuf, the final octet is interpreted (it must be
either a RET or an END).</p>
</dd>
<dt>JSR</dt><dd><p>The <code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code> virtual machine has a stack, fixed at size 1. A
JSR copies address of the current active table to the stack and
transitions to the active table given by the next two octets. (For
instance, table 1 would be the next table after the table given as a
parameter to <code class="docutils literal notranslate"><span class="pre">START</span></code>.) Translation of the current octet stops
after encountering a JSR.</p>
<p>JSRs are useful for converting a two octet input character: the
first octet in the character will make a JSR to some table; the
second octet will produce a translation and RET to the current
table.</p>
<p>Since the virtual machine has a fixed size stack, it would be highly
unusual for the virtual machine to encounter two different JSRs
without an intervening RET.</p>
</dd>
<dt>JMP</dt><dd><p>Similar to JSR, but does not change the stack. It is the equivalent
of a goto. JMPs are useful to deal with modal input character sets
(such as an escape in ISO-2022-JP, see <a class="reference external" href="#mkchartable">how the tables are
generated</a>).</p>
</dd>
<dt>RET</dt><dd><p>Indicates that we are done translating this input octet and we
should return to the previous active table. It might appear as the
first of the 4 translation octets, in which case this input
character translates into nothing (it might be whitespace, for
instance).</p>
</dd>
<dt>END</dt><dd><p>Indicates we are done translating this input octet. When
<code class="docutils literal notranslate"><span class="pre">TRANSLATE</span></code> is next called, that input octet will be interpreted
against the current active table; the stack does not change.</p>
</dd>
</dl>
<p>In addition, it recognizes the following "special" control codes for
charsets that aren't easily represented by a set of tables, UTF-8 and
UTF-7:</p>
<dl class="simple">
<dt>U7F</dt><dd><p>UTF-7 consists of US-ASCII characters and a special escape character
that indicates a transition to base-64 encoded UTF-16 characters.
The virtual machine has built in code to handle the base64 decoding.
In UTF-7's base64, 8 input octets result in 3 characters, so the
tables would be rather large.</p>
</dd>
<dt>U7N</dt><dd><p>This indicates that the current octet is the continuation of the
base-64 section.</p>
</dd>
<dt>U83</dt><dd><p>One and two character UTF-8 sequences are handled normally in the
charset code. To keep the table size down, 3 octet sequences are
handled specially. U83 indicates that the current input octet is the
start of a three character sequence. It is also an implicit jump to
the 2nd table in the UTF-8 sequence, ending this translation.</p>
</dd>
<dt>U83_2</dt><dd><p>This input octet 2nd of 3-octet UTF-8 input, with an implicit jump
to the 3rd table.</p>
</dd>
<dt>U83_3</dt><dd><p>3rd octet of a 3-octet UTF-8 input. This produces the output
characters and has an implicit jump to the 1st table of UTF-8.</p>
</dd>
</dl>
<p>Finally, it's useful to mention the special character <code class="docutils literal notranslate"><span class="pre">EMPTY</span></code> which is
guaranteed not to match any character. It is also represented by an
uppercase US-ASCII character.</p>
</section>
</section>
<section id="searching-and-compiled-patterns">
<h2>searching and compiled patterns<a class="headerlink" href="#searching-and-compiled-patterns" title="Permalink to this heading"></a></h2>
<section id="boyer-moore">
<h3>boyer-moore<a class="headerlink" href="#boyer-moore" title="Permalink to this heading"></a></h3>
<p>brief description of boyer-moore xxx</p>
</section>
<section id="cyrus-implementation">
<h3>cyrus implementation<a class="headerlink" href="#cyrus-implementation" title="Permalink to this heading"></a></h3>
<p>why two arrays? us-ascii optimization, really kinda useless now xxx</p>
<p>meta-data stored at the end xxx</p>
</section>
</section>
<section id="generating-the-tables-mkchartable">
<h2>generating the tables: <code class="docutils literal notranslate"><span class="pre">mkchartable</span></code><a class="headerlink" href="#generating-the-tables-mkchartable" title="Permalink to this heading"></a></h2>
<p>The program <code class="docutils literal notranslate"><span class="pre">mkchartable</span></code> is used to generate the charset transcoding
tables used by TRANSLATE. These tables are carefully constructed so no
more than a single octet need be examined at a time; this octet results
in either an output stream of UTF-8 characters being generated or some
sort of state change.</p>
<p><code class="docutils literal notranslate"><span class="pre">mkchartable</span></code> uses three different sorts of input files to generate
these tables. These files are located in the <code class="docutils literal notranslate"><span class="pre">lib/charset</span></code> directory.</p>
<section id="charset-tables">
<h3>charset tables<a class="headerlink" href="#charset-tables" title="Permalink to this heading"></a></h3>
<p>Each charset file maps a single charset to the corresponding Unicode
characters. For the US-ASCII and ISO-8859-x character sets this is
trivial: each input byte corresponds to a single Unicode character.
(Actually, some ISO-8859-x octets do not map to any Unicode character.
In that case, the file either does not contain that octet or map it to
"<code class="docutils literal notranslate"><span class="pre">????</span></code>".)</p>
<p>Other character sets are trickier. For instance, GB-2312 has both single
and double byte characters, but is still a simple map from input
character to output character. More complicated are modal character
encodings. For instance, ISO-2022-JP starts in US-ASCII mode and uses
<code class="docutils literal notranslate"><span class="pre">1B</span></code> as an escape character followed by another two characters to
select a new mode.</p>
<p>The input charset labels modes with "<code class="docutils literal notranslate"><span class="pre">:</span></code>" followed by the mode name.
The starting mode "<code class="docutils literal notranslate"><span class="pre">US-ASCII</span></code>" in ISO-2022-JP is preceded by
"<code class="docutils literal notranslate"><span class="pre">:US-ASCII</span></code>". Mode transitions are denoted by a Unicode conversion of
"<code class="docutils literal notranslate"><span class="pre">>newmode</span></code>" or "<code class="docutils literal notranslate"><span class="pre">:newmode</span></code>". To denote that the octet <code class="docutils literal notranslate"><span class="pre">42</span></code>
transitions into the "<code class="docutils literal notranslate"><span class="pre">US-ASCII</span></code>" mode, the charset file has
"<code class="docutils literal notranslate"><span class="pre">42</span> <span class="pre">>US-ASCII</span></code>". The mode names themselves are arbitrary labels and
have no effect on the output.</p>
<p>The input charset labels modes with ":" followed by the mode name. The
mode name is optionally followed by a space and the "<code class="docutils literal notranslate"><span class="pre"><</span></code>" character.
If the "<code class="docutils literal notranslate"><span class="pre"><</span></code>" character is present, then all translations will be
followed by a RET instruction instead of an END instruction.</p>
<p>The transition "<code class="docutils literal notranslate"><span class="pre">>newmode</span></code>" results in a JSR instruction being
generated. A JMP instruction is generated by a transition of
"<code class="docutils literal notranslate"><span class="pre">:newmode</span></code>".</p>
<p>The input byte can be specified as "<code class="docutils literal notranslate"><span class="pre">*</span></code>". This is used to define the
"default action" which is used for input bytes that are not otherwise
defined for the mode. If the default action is not explicitly stated, it
is a translation to EMPTY.</p>
</section>
<section id="unicode-data-table">
<h3>unicode data table<a class="headerlink" href="#unicode-data-table" title="Permalink to this heading"></a></h3>
<p>The <code class="docutils literal notranslate"><span class="pre">unidata2.txt</span></code> file is verbatim from the Unicode standard. More
recent versions should be available <a class="reference external" href="http://www.unicode.org/xxx">from their
website</a>. Each entry in the file
describers a Unicode character by the following properties, separated by
semicolons:</p>
<ul class="simple">
<li><p>code point (16-bit character value) in hex</p></li>
<li><p>character name (unused by Cyrus)</p></li>
<li><p>general category, such as whitespace or punctuation</p></li>
<li><p>the canonical combining class (unused)</p></li>
<li><p>bidirection category (unused)</p></li>
<li><p>character decomposition</p></li>
<li><p>decimal digit value (unused)</p></li>
<li><p>digit value (unused, and, no, I don't know the difference)</p></li>
<li><p>numeric value including fractions (unused)</p></li>
<li><p>mirrored character (unused)</p></li>
<li><p>Unicode 1.0 name (unused)</p></li>
<li><p>comment (unused)</p></li>
<li><p>upper case equivalent (unused)</p></li>
<li><p>lower case equivalent</p></li>
</ul>
<p>In general, Cyrus uses the lower case equivalent if there is one, and
the decomposed value otherwise.</p>
</section>
<section id="unicode-fixup-table">
<h3>unicode fixup table<a class="headerlink" href="#unicode-fixup-table" title="Permalink to this heading"></a></h3>
<p>The <code class="docutils literal notranslate"><span class="pre">unifix.txt</span></code> file contains Cyrus-specific mappings for characters.
It overrides the <code class="docutils literal notranslate"><span class="pre">unidata2.txt</span></code> table. Each rule in the file is
explained with a comment. It's helpful to remember that the Unicode
general categories starting with <code class="docutils literal notranslate"><span class="pre">Z</span></code> represent whitespace, and
whitespace is always removed.</p>
</section>
<section id="generating-chartable-c">
<h3>generating <code class="docutils literal notranslate"><span class="pre">chartable.c</span></code><a class="headerlink" href="#generating-chartable-c" title="Permalink to this heading"></a></h3>
<p>how <code class="docutils literal notranslate"><span class="pre">mkchartable</span></code> works: collapses the encoding modes, the unicode
translations, and other normalizations into the output tables described
above xxx</p>
</section>
</section>
<section id="for-the-future">
<h2>for the future<a class="headerlink" href="#for-the-future" title="Permalink to this heading"></a></h2>
<section id="sieve-acap-comparators">
<h3>Sieve/ACAP comparators<a class="headerlink" href="#sieve-acap-comparators" title="Permalink to this heading"></a></h3>
</section>
<section id="adjustable-normalization">
<h3>adjustable normalization?<a class="headerlink" href="#adjustable-normalization" title="Permalink to this heading"></a></h3>
<p>The use of uppercase US-ASCII characters is one of the annoyances in
trying to generalize the charset transcoding. If we continue to restrict
the characters under consideration to the BMP, switching to UTF-8
control codes that start 4 or 5 byte sequences is possible.</p>
<p>Another possibility is to use a NUL character as an escape sequence,
though this increases the size of each control code by 1 octet.</p>
</section>
<section id="handle-2-octet-input-characters">
<h3>handle >2 octet input characters<a class="headerlink" href="#handle-2-octet-input-characters" title="Permalink to this heading"></a></h3>
</section>
<section id="make-utf-8-more-regular">
<h3>make UTF-8 more regular<a class="headerlink" href="#make-utf-8-more-regular" title="Permalink to this heading"></a></h3>
<p>consider whether we really need U83, U83_2, U83_3. also consider
changing <code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">U83,</span> <span class="pre">0,</span> <span class="pre">0,</span> <span class="pre">0</span> <span class="pre">}</span></code> translations to <code class="docutils literal notranslate"><span class="pre">{</span> <span class="pre">U83,</span> <span class="pre">JMP,</span> <span class="pre">0,</span> <span class="pre">1</span> <span class="pre">}</span></code>
sequences to at least eliminate the implicit jump.</p>
</section>
<section id="require-minimal-utf-8-characters">
<h3>require minimal UTF-8 characters<a class="headerlink" href="#require-minimal-utf-8-characters" title="Permalink to this heading"></a></h3>
</section>
</section>
<section id="references">
<h2>references<a class="headerlink" href="#references" title="Permalink to this heading"></a></h2>
<p>xxx</p>
<ul class="simple">
<li><p>[UNICODE] Unicode / ISO 10646</p></li>
<li><p>[UTF-8] utf-8 RFC</p></li>
<li><p>[UTF-7] utf-7 RFC</p></li>
<li><p>[BM] boyer-moore</p></li>
<li><p>[ACAP] the comparators reference. see section XXX of RFC 2244.</p></li>
</ul>
</section>
</section>
</div>
</div>
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
<a href="hacking.html" class="btn btn-neutral float-left" title="Cyrus IMAP Server: Hacking" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
<a href="locking.html" class="btn btn-neutral float-right" title="Cyrus IMAP Server: Locking" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
</div>
<hr/>
<div role="contentinfo">
<p>© Copyright 1993–2025, The Cyrus Team.</p>
</div>
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script>
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>
|
