File: Netconversion.html

package info (click to toggle)
ocamlnet 2.2.9-8
links: PTS, VCS
area: main
in suites: squeeze
size: 17,724 kB
ctags: 10,053
sloc: ml: 63,928; ansic: 1,973; makefile: 800; sh: 651
file content (1334 lines) | stat: -rw-r--r-- 74,292 bytes
parent folder | download | duplicates (2)
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<link rel="stylesheet" href="style.css" type="text/css">
<meta content="text/html; charset=iso-8859-1" http-equiv="Content-Type">
<link rel="Start" href="index.html">
<link rel="previous" href="Netshm_intro.html">
<link rel="next" href="Netchannels.html">
<link rel="Up" href="index.html">
<link title="Index of types" rel=Appendix href="index_types.html">
<link title="Index of exceptions" rel=Appendix href="index_exceptions.html">
<link title="Index of values" rel=Appendix href="index_values.html">
<link title="Index of class attributes" rel=Appendix href="index_attributes.html">
<link title="Index of class methods" rel=Appendix href="index_methods.html">
<link title="Index of classes" rel=Appendix href="index_classes.html">
<link title="Index of class types" rel=Appendix href="index_class_types.html">
<link title="Index of modules" rel=Appendix href="index_modules.html">
<link title="Index of module types" rel=Appendix href="index_module_types.html">
<link title="Uq_gtk" rel="Chapter" href="Uq_gtk.html">
<link title="Equeue" rel="Chapter" href="Equeue.html">
<link title="Unixqueue" rel="Chapter" href="Unixqueue.html">
<link title="Uq_engines" rel="Chapter" href="Uq_engines.html">
<link title="Uq_socks5" rel="Chapter" href="Uq_socks5.html">
<link title="Unixqueue_mt" rel="Chapter" href="Unixqueue_mt.html">
<link title="Equeue_intro" rel="Chapter" href="Equeue_intro.html">
<link title="Uq_ssl" rel="Chapter" href="Uq_ssl.html">
<link title="Uq_tcl" rel="Chapter" href="Uq_tcl.html">
<link title="Netcgi_common" rel="Chapter" href="Netcgi_common.html">
<link title="Netcgi" rel="Chapter" href="Netcgi.html">
<link title="Netcgi_ajp" rel="Chapter" href="Netcgi_ajp.html">
<link title="Netcgi_scgi" rel="Chapter" href="Netcgi_scgi.html">
<link title="Netcgi_cgi" rel="Chapter" href="Netcgi_cgi.html">
<link title="Netcgi_fcgi" rel="Chapter" href="Netcgi_fcgi.html">
<link title="Netcgi_dbi" rel="Chapter" href="Netcgi_dbi.html">
<link title="Netcgi1_compat" rel="Chapter" href="Netcgi1_compat.html">
<link title="Netcgi_test" rel="Chapter" href="Netcgi_test.html">
<link title="Netcgi_porting" rel="Chapter" href="Netcgi_porting.html">
<link title="Netcgi_plex" rel="Chapter" href="Netcgi_plex.html">
<link title="Http_client" rel="Chapter" href="Http_client.html">
<link title="Telnet_client" rel="Chapter" href="Telnet_client.html">
<link title="Ftp_data_endpoint" rel="Chapter" href="Ftp_data_endpoint.html">
<link title="Ftp_client" rel="Chapter" href="Ftp_client.html">
<link title="Nethttpd_types" rel="Chapter" href="Nethttpd_types.html">
<link title="Nethttpd_kernel" rel="Chapter" href="Nethttpd_kernel.html">
<link title="Nethttpd_reactor" rel="Chapter" href="Nethttpd_reactor.html">
<link title="Nethttpd_engine" rel="Chapter" href="Nethttpd_engine.html">
<link title="Nethttpd_services" rel="Chapter" href="Nethttpd_services.html">
<link title="Nethttpd_plex" rel="Chapter" href="Nethttpd_plex.html">
<link title="Nethttpd_intro" rel="Chapter" href="Nethttpd_intro.html">
<link title="Netplex_types" rel="Chapter" href="Netplex_types.html">
<link title="Netplex_mp" rel="Chapter" href="Netplex_mp.html">
<link title="Netplex_mt" rel="Chapter" href="Netplex_mt.html">
<link title="Netplex_log" rel="Chapter" href="Netplex_log.html">
<link title="Netplex_controller" rel="Chapter" href="Netplex_controller.html">
<link title="Netplex_container" rel="Chapter" href="Netplex_container.html">
<link title="Netplex_sockserv" rel="Chapter" href="Netplex_sockserv.html">
<link title="Netplex_workload" rel="Chapter" href="Netplex_workload.html">
<link title="Netplex_main" rel="Chapter" href="Netplex_main.html">
<link title="Netplex_config" rel="Chapter" href="Netplex_config.html">
<link title="Netplex_kit" rel="Chapter" href="Netplex_kit.html">
<link title="Rpc_netplex" rel="Chapter" href="Rpc_netplex.html">
<link title="Netplex_cenv" rel="Chapter" href="Netplex_cenv.html">
<link title="Netplex_intro" rel="Chapter" href="Netplex_intro.html">
<link title="Netshm" rel="Chapter" href="Netshm.html">
<link title="Netshm_data" rel="Chapter" href="Netshm_data.html">
<link title="Netshm_hashtbl" rel="Chapter" href="Netshm_hashtbl.html">
<link title="Netshm_array" rel="Chapter" href="Netshm_array.html">
<link title="Netshm_intro" rel="Chapter" href="Netshm_intro.html">
<link title="Netconversion" rel="Chapter" href="Netconversion.html">
<link title="Netchannels" rel="Chapter" href="Netchannels.html">
<link title="Netstream" rel="Chapter" href="Netstream.html">
<link title="Mimestring" rel="Chapter" href="Mimestring.html">
<link title="Netmime" rel="Chapter" href="Netmime.html">
<link title="Netsendmail" rel="Chapter" href="Netsendmail.html">
<link title="Neturl" rel="Chapter" href="Neturl.html">
<link title="Netaddress" rel="Chapter" href="Netaddress.html">
<link title="Netbuffer" rel="Chapter" href="Netbuffer.html">
<link title="Netdate" rel="Chapter" href="Netdate.html">
<link title="Netencoding" rel="Chapter" href="Netencoding.html">
<link title="Netulex" rel="Chapter" href="Netulex.html">
<link title="Netaccel" rel="Chapter" href="Netaccel.html">
<link title="Netaccel_link" rel="Chapter" href="Netaccel_link.html">
<link title="Nethtml" rel="Chapter" href="Nethtml.html">
<link title="Netstring_str" rel="Chapter" href="Netstring_str.html">
<link title="Netstring_pcre" rel="Chapter" href="Netstring_pcre.html">
<link title="Netstring_mt" rel="Chapter" href="Netstring_mt.html">
<link title="Netmappings" rel="Chapter" href="Netmappings.html">
<link title="Netaux" rel="Chapter" href="Netaux.html">
<link title="Nethttp" rel="Chapter" href="Nethttp.html">
<link title="Netchannels_tut" rel="Chapter" href="Netchannels_tut.html">
<link title="Netmime_tut" rel="Chapter" href="Netmime_tut.html">
<link title="Netsendmail_tut" rel="Chapter" href="Netsendmail_tut.html">
<link title="Netulex_tut" rel="Chapter" href="Netulex_tut.html">
<link title="Neturl_tut" rel="Chapter" href="Neturl_tut.html">
<link title="Netsys" rel="Chapter" href="Netsys.html">
<link title="Netpop" rel="Chapter" href="Netpop.html">
<link title="Rpc_auth_dh" rel="Chapter" href="Rpc_auth_dh.html">
<link title="Rpc_key_service" rel="Chapter" href="Rpc_key_service.html">
<link title="Rpc_time" rel="Chapter" href="Rpc_time.html">
<link title="Rpc_auth_local" rel="Chapter" href="Rpc_auth_local.html">
<link title="Rtypes" rel="Chapter" href="Rtypes.html">
<link title="Xdr" rel="Chapter" href="Xdr.html">
<link title="Rpc" rel="Chapter" href="Rpc.html">
<link title="Rpc_program" rel="Chapter" href="Rpc_program.html">
<link title="Rpc_portmapper_aux" rel="Chapter" href="Rpc_portmapper_aux.html">
<link title="Rpc_packer" rel="Chapter" href="Rpc_packer.html">
<link title="Rpc_transport" rel="Chapter" href="Rpc_transport.html">
<link title="Rpc_client" rel="Chapter" href="Rpc_client.html">
<link title="Rpc_simple_client" rel="Chapter" href="Rpc_simple_client.html">
<link title="Rpc_portmapper_clnt" rel="Chapter" href="Rpc_portmapper_clnt.html">
<link title="Rpc_portmapper" rel="Chapter" href="Rpc_portmapper.html">
<link title="Rpc_server" rel="Chapter" href="Rpc_server.html">
<link title="Rpc_auth_sys" rel="Chapter" href="Rpc_auth_sys.html">
<link title="Rpc_intro" rel="Chapter" href="Rpc_intro.html">
<link title="Rpc_mapping_ref" rel="Chapter" href="Rpc_mapping_ref.html">
<link title="Rpc_ssl" rel="Chapter" href="Rpc_ssl.html">
<link title="Rpc_xti_client" rel="Chapter" href="Rpc_xti_client.html">
<link title="Shell_sys" rel="Chapter" href="Shell_sys.html">
<link title="Shell" rel="Chapter" href="Shell.html">
<link title="Shell_uq" rel="Chapter" href="Shell_uq.html">
<link title="Shell_mt" rel="Chapter" href="Shell_mt.html">
<link title="Shell_intro" rel="Chapter" href="Shell_intro.html">
<link title="Netsmtp" rel="Chapter" href="Netsmtp.html"><link title="Preliminaries" rel="Section" href="#preliminaries">
<link title="Interface" rel="Section" href="#interface">
<link title="Unicode" rel="Subsection" href="#unicode">
<link title="Subsets of Unicode" rel="Subsection" href="#subsets">
<link title="Linking this module" rel="Subsection" href="#linking">
<link title="Supported Encodings, Restrictions" rel="Subsection" href="#domain">
<link title="Known Problems" rel="Subsection" href="#problems">
<link title="Direct Conversion" rel="Subsection" href="#direct_conv">
<link title="Reading Text Using Cursors" rel="Subsection" href="#cursors">
<link title="Unicode String Functions" rel="Subsection" href="#unicode_functions">
<title>Ocamlnet 2 Reference Manual : Netconversion</title>
</head>
<body>
<div class="navbar"><a href="Netshm_intro.html">Previous</a>
&nbsp;<a href="index.html">Up</a>
&nbsp;<a href="Netchannels.html">Next</a>
</div>
<center><h1>Module <a href="type_Netconversion.html">Netconversion</a></h1></center>
<br>
<pre><span class="keyword">module</span> Netconversion: <code class="code">sig</code> <a href="Netconversion.html">..</a> <code class="code">end</code></pre>Conversion between character encodings 
<p>

 <b>Contents</b>
 <ul>
<li><a href="Netconversion.html#preliminaries"><i>Preliminaries</i></a>
     <ul>
<li><a href="Netconversion.html#unicode"><i>Unicode</i></a></li>
<li><a href="Netconversion.html#subsets"><i>Subsets of Unicode</i></a></li>
<li><a href="Netconversion.html#linking"><i>Linking this module</i></a></li>
<li><a href="Netconversion.html#domain"><i>Supported Encodings, Restrictions</i></a></li>
<li><a href="Netconversion.html#problems"><i>Known Problems</i></a></li>
</ul>
</li>
<li><a href="Netconversion.html#interface"><i>Interface</i></a>
     <ul>
<li><a href="Netconversion.html#direct_conv"><i>Direct Conversion</i></a></li>
<li><a href="Netconversion.html#cursors"><i>Reading Text Using Cursors</i></a>
           <ul>
<li><a href="Netconversion.html#bom"><i>Byte Order Marks</i></a></li>
</ul>
</li>
<li><a href="Netconversion.html#unicode_functions"><i>Unicode String Functions</i></a></li>
</ul>

   </li>
</ul>
<br>
<hr width="100%">
<br>
<a name="preliminaries"></a>
<h1>Preliminaries</h1>
<p>

 A <b>character set</b> is a set of characters where every character is
 identified by a <b>code point</b>. An <b>encoding</b> is a way of 
 representing characters from a set in byte strings. For example,
 the Unicode character set has more than 96000 characters, and
 the code points have values from 0 to 0x10ffff (not all code points
 are assigned yet). The UTF-8 encoding represents the code points
 by sequences of 1 to 4 bytes. There are also encodings that 
 represent code points from several sets, e.g EUC-JP covers four
 sets.
<p>

 Encodings are enumerated by the type <code class="code">encoding</code>, and names follow
 the convention <code class="code">`Enc_*</code>, e.g. <code class="code">`Enc_utf8</code>. 
 Character sets are enumerated by the type
 <code class="code">charset</code>, and names follow the convention <code class="code">`Set_*</code>, e.g.
 <code class="code">`Set_unicode</code>.
<p>

 This module deals mainly with encodings. It is important to know
 that the same character set may have several encodings. For example,
 the Unicode character set can be encoded as UTF-8 or UTF-16.
 For the 8 bit character sets, however, there is usually only one
 encoding, e.g <code class="code">`Set_iso88591</code> is always encoded as <code class="code">`Enc_iso88591</code>.
<p>

 In a <b>single-byte encoding</b> every code point is represented by
 one byte. This is what many programmers are accustomed at, and
 what the O'Caml language specially supports: A <code class="code">string</code> is
 a sequence of <code class="code">char</code>s, where <code class="code">char</code> means an 8 bit quantity
 interpreted as character. For example, the following piece of code allocates
 a <code class="code">string</code> of four <code class="code">char</code>s, and assigns them individually:
<p>

 <pre><code class="code"> let s = String.create 4 in
 s.[0] &lt;- 'G';
 s.[1] &lt;- 'e';
 s.[2] &lt;- 'r';
 s.[3] &lt;- 'd';
 </code></pre>
<p>

 In a <b>multi-byte encoding</b> there are code points that are represented
 by several bytes. As we still represent such text as <code class="code">string</code>, the
 problem arises that a single <code class="code">char</code>, actually a byte, often represents 
 only a fraction of a full multi-byte character. There are two solutions:<ul>
<li>Give up the principle that text is represented by <code class="code">string</code>.
   This is, for example, the approach chosen by <code class="code">Camomile</code>, another O'Caml
   library dealing with Unicode. Instead, text is represented as
   <code class="code">int array</code>. This way, the algorithms processing the text can
   remain the same.</li>
<li>Give up the principle that individual characters can be directly
   accessed in a text. This is the primary way chosen by Ocamlnet.
   This means that there is not any longer the possibility to read
   or write the <code class="code">n</code>th character of a text. One can, however, still 
   compose texts by just concatenating the strings representing
   individual characters. Furthermore, it is possible to define
   a cursor for a text that moves sequentially along the text.
   The consequence is that programmers are restricted to sequential
   algorithms. Note that the majority of text processing falls into
   this class.</li>
</ul>

 The corresponding piece of code for Ocamlnet's Unicode implementation
 is:
 <pre><code class="code"> let b = Buffer.create 80 in
 Buffer.add b (ustring_of_uchar `Enc_utf8 71);  (* 71 = code point of 'G' *)
 Buffer.add b (ustring_of_uchar `Enc_utf8 101); (* 101 = code point of 'e' *)
 Buffer.add b (ustring_of_uchar `Enc_utf8 114); (* 114 = code point of 'r' *)
 Buffer.add b (ustring_of_uchar `Enc_utf8 100); (* 100 = code point of 'd' *)
 let s = Buffer.contents b
 </code></pre>
<p>

 It is important to always remember that a <code class="code">char</code> is no longer 
 a character but simply a byte. In many of the following explanations,
 we strictly distinguish between <b>byte positions</b> or <b>byte counts</b>,
 and <b>character positions</b> or <b>character counts</b>.
<p>

 There a number of special effects that usually only occur in
 multi-byte encodings:
<p>
<ul>
<li>Bad encodings: Not every byte sequence is legal. When scanning
   such text, the functions will raise the exception <code class="code">Malformed_code</code>
   when they find illegal bytes.</li>
<li>Unassigned code points: It may happen that a byte sequence is
   a correct representation for a code point, but that the code point
   is unassigned in the character set. When scanning, this is also
   covered by the exception <code class="code">Malformed_code</code>. When converting from
   one encoding to another, it is also possible that the code point
   is only unassigned in the target character set. This case is
   usually handled by a substitution function <code class="code">subst</code>, and if no such
   function is defined, by the exception <code class="code">Cannot_represent</code>.</li>
<li>Incomplete characters: The trailing bytes of a string may be the
   correct beginning of a byte sequence for a character, but not a
   complete sequence. Of course, if that string is the end of a
   text, this is just illegal, and also a case for <code class="code">Malformed_code</code>.
   However, when text is processed chunk by chunk, this phenomenon
   may happen legally for all chunks but the last. For this reason,
   some of the functions below handle this case specially.</li>
<li>Byte order marks: Some encodings have both big and little endian
   variants. A byte order mark at the beginning of the text declares
   which variant is actually used. This byte order mark is a 
   declaration written like a character, but actually not a 
   character.</li>
</ul>

 There is a special class of encodings known as <b>ASCII-compatible</b>.
 They are important because there are lots of programs and protocols
 that only interpret bytes from 0 to 127, and treat the bytes from
 128 to 255 as data. These programs can process texts as long as
 the bytes from 0 to 127 are used as in ASCII. Fortunately, many
 encodings are ASCII-compatible, including UTF-8.
<p>

 <a name="unicode"></a>
<h2>Unicode</h2>
<p>

 <code class="code">Netconversion</code> is centred around Unicode.
 The conversion from one encoding to another works by finding the
 Unicode code point of the character
 to convert, and by representing the code point in the target encoding,
 even if neither encodings have to do with Unicode.
 Of course, this approach requires that all character sets handled
 by <code class="code">Netconversion</code> are subsets of Unicode.
<p>

 The supported range of Unicode code points: 0 to 0xd7ff, 0xe000 to 0xfffd,
 0x10000 to 0x10ffff. All these code points can be represented in 
 UTF-8 and UTF-16. <code class="code">Netconversion</code> does not know which of the code
 points are assigned and which not, and because of this, it simply
 allows all code points of the mentioned ranges (but for other character
 sets, the necessary lookup tables exist).
<p>

 <b>UTF-8:</b> The UTF-8 representation can have one to four bytes. Malformed 
   byte sequences are always rejected, even those that want to cheat the
   reader like "0xc0 0x80" for the code point 0. There is special support
   for the Java variant of UTF-8 (<code class="code">`Enc_java</code>). UTF-8 strings must not
   have a byte order mark (it would be interpreted as "zero-width space"
   character).
<p>

 <b>UTF-16:</b> When reading from a string encoded as <code class="code">`Enc_utf16</code>, a byte
   order mark is expected at the beginning. The detected variant 
   (<code class="code">`Enc_utf16_le</code> or <code class="code">`Enc_utf16_be</code>) is usually returned by the parsing
   function. The byte order mark is not included into the output string. - 
   Some functions of this
   module cannot cope with <code class="code">`Enc_utf16</code> (i.e. UTF-16 without endianess
   annotation), and will fail.
<p>

   Once the endianess is determined, the code point 0xfeff is no longer
   interpreted as byte order mark, but as "zero-width non-breakable space".
<p>

   Some code points are represented by pairs of 16 bit values, these
   are the so-called "surrogate pairs". They can only occur in UTF-16.
<p>

 <a name="subsets"></a>
<h2>Subsets of Unicode</h2>
<p>

 The non-Unicode character sets are subsets of Unicode. Here, it may
 happen that a Unicode code point does not have a corresponding 
 code point. In this case, certain rules are applied to handle
 this (see below). It is, however, ensured that every non-Unicode
 code point has a corresponding Unicode code point. (In other words,
 character sets cannot be supported for which this property does
 not hold.)
<p>

 It is even possible to create further subsets artificially. The
 encoding <code class="code">`Enc_subset(e,def)</code> means to derive a new encoding from
 the existing one <code class="code">e</code>, but to only accept the code points for which
 the definition function <code class="code">def</code> yields the value <code class="code">true</code>. For example,
 the encoding 
 <pre><code class="code"> `Enc_subset(`Enc_usascii, 
             fun i -&gt; i &lt;&gt; 34 &amp;&amp; i &lt;&gt; 38 &amp;&amp; i &lt;&gt; 60 &amp;&amp; i &lt;&gt; 62) </code></pre>
 is ASCII without the bracket angles, the quotation mark, and the
 ampersand character, i.e. the subset of ASCII that can be included
 in HTML text without escaping.
<p>

 If a code point is not defined by the encoding but found in a text, 
 the reader will raise the exception <code class="code">Malformed_code</code>. When text is
 output, however, the <code class="code">subst</code> function will be called for undefined code 
 points (which raises <code class="code">Cannot_represent</code> by default). The <code class="code">subst</code>
 function is an optional argument of many conversion functions that
 allows it to insert a substitution text for undefined code points.
 Note, however, that the substitution text is restricted to at most
 50 characters (because unlimited length would lead to difficult
 problems we would like to avoid).
<p>

 <a name="linking"></a>
<h2>Linking this module</h2>
<p>

 Many encodings require lookup tables. The following encodings
 are built-in and always supported:
<p>
<ul>
<li>Unicode: <code class="code">`Enc_utf8</code>, <code class="code">`Enc_java</code>, <code class="code">`Enc_utf16</code>, <code class="code">`Enc_utf16_le</code>, 
     <code class="code">`Enc_utf16_be</code></li>
<li>Other: <code class="code">`Enc_usascii</code>, <code class="code">`Enc_iso88591</code>, <code class="code">`Enc_empty</code></li>
</ul>

 The lookup tables for the other encodings are usually loaded at
 runtime, but it is also possible to embed them in the generated
 binary executable. See the file <code class="code">INSTALL</code> for details. The functions
 <code class="code">available_input_encodings</code> and <code class="code">available_output_encodings</code> can
 be invoked to find out which encodings can be loaded, or are available
 otherwise.
<p>

 <a name="domain"></a>
<h2>Supported Encodings, Restrictions</h2>
<p>

 I took the mappings from <code class="code">www.unicode.org</code>, and the standard names of
 the character sets from IANA. Obviously, many character sets are missing
 that can be supported; especially ISO646 character sets, and many EBCDIC 
 code pages. Stateful encodings like generic ISO-2022 have been omitted
 (stateless subsets of ISO-2022 like EUC can be supported, however;
 currently we support EUC-JP and EUC-KR).
<p>

 Because of the copyright statement from Unicode, I cannot put the
 source tables that describe the mappings into the distribution. They
 are publicly available from <code class="code">www.unicode.org</code>.
<p>

 <a name="problems"></a>
<h2>Known Problems</h2>
<p>
<ul>
<li>The following charsets do not have a bijective mapping to Unicode:
   adobe_standard_encoding, adobe_symbol_encoding, 
   adobe_zapf_dingbats_encoding, cp1002 (0xFEBE). The current implementation
   simply removes one of the conflicting code point pairs - this might
   not what you want.</li>
<li>Japanese encodings: 
   JIS X 0208: The character 1/32 is mapped to 0xFF3C, and not
   to 0x005C.</li>
</ul>
<br>
<br>
<a name="interface"></a>
<h1>Interface</h1>
<p>

 <b>Naming conventions:</b>
<p>

 As it is possible to refer to substrings by either giving a byte
 offset or by counting whole characters, these naming conventions
 are helpful:
<p>
<ul>
<li>Labels called <code class="code">range_pos</code> and <code class="code">range_len</code> refer to byte positions of
   characters, or substrings</li>
<li>Labels called <code class="code">count</code> refer to positions given as the number of characters
   relative to an origin</li>
</ul>

 Furthermore:
<p>
<ul>
<li>A <code class="code">uchar</code> is a single Unicode code point represented as int</li>
<li>A <code class="code">ustring</code> is a string of encoded characters</li>
<li>A <code class="code">uarray</code> is an <code class="code">array of int</code> representing a string</li>
</ul>
<br>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONMalformed_code"></a>Malformed_code</pre>
<div class="info">
Raised when an illegal byte sequence is found<br>
</div>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONCannot_represent"></a>Cannot_represent <span class="keyword">of</span> <code class="type">int</code></pre>
<div class="info">
Raised when a certain Unicode code point cannot be represented in
 the selected output encoding<br>
</div>
<pre><span class="keyword">type</span> <a name="TYPEencoding"></a><code class="type"></code>encoding = <code class="type">[ `Enc_adobe_standard_encoding<br>       | `Enc_adobe_symbol_encoding<br>       | `Enc_adobe_zapf_dingbats_encoding<br>       | `Enc_cp037<br>       | `Enc_cp1006<br>       | `Enc_cp1026<br>       | `Enc_cp1047<br>       | `Enc_cp424<br>       | `Enc_cp437<br>       | `Enc_cp500<br>       | `Enc_cp737<br>       | `Enc_cp775<br>       | `Enc_cp850<br>       | `Enc_cp852<br>       | `Enc_cp855<br>       | `Enc_cp856<br>       | `Enc_cp857<br>       | `Enc_cp860<br>       | `Enc_cp861<br>       | `Enc_cp862<br>       | `Enc_cp863<br>       | `Enc_cp864<br>       | `Enc_cp865<br>       | `Enc_cp866<br>       | `Enc_cp869<br>       | `Enc_cp874<br>       | `Enc_cp875<br>       | `Enc_empty<br>       | `Enc_eucjp<br>       | `Enc_euckr<br>       | `Enc_iso88591<br>       | `Enc_iso885910<br>       | `Enc_iso885911<br>       | `Enc_iso885913<br>       | `Enc_iso885914<br>       | `Enc_iso885915<br>       | `Enc_iso885916<br>       | `Enc_iso88592<br>       | `Enc_iso88593<br>       | `Enc_iso88594<br>       | `Enc_iso88595<br>       | `Enc_iso88596<br>       | `Enc_iso88597<br>       | `Enc_iso88598<br>       | `Enc_iso88599<br>       | `Enc_java<br>       | `Enc_jis0201<br>       | `Enc_koi8r<br>       | `Enc_macroman<br>       | `Enc_subset of <a href="Netconversion.html#TYPEencoding">encoding</a> * (int -> bool)<br>       | `Enc_usascii<br>       | `Enc_utf16<br>       | `Enc_utf16_be<br>       | `Enc_utf16_le<br>       | `Enc_utf8<br>       | `Enc_windows1250<br>       | `Enc_windows1251<br>       | `Enc_windows1252<br>       | `Enc_windows1253<br>       | `Enc_windows1254<br>       | `Enc_windows1255<br>       | `Enc_windows1256<br>       | `Enc_windows1257<br>       | `Enc_windows1258 ]</code> </pre>
<div class="info">
The polymorphic variant enumerating the supported encodings. We have:<ul>
<li><code class="code">`Enc_utf8</code>: UTF-8</li>
<li><code class="code">`Enc_java</code>: The UTF-8 variant used by Java (the only difference is
   the representation of NUL)</li>
<li><code class="code">`Enc_utf16</code>: UTF-16 with unspecified endianess (restricted)</li>
<li><code class="code">`Enc_utf16_le</code>: UTF-16 little endian</li>
<li><code class="code">`Enc_utf16_be</code>: UTF-16 big endian</li>
<li><code class="code">`Enc_usascii</code>: US-ASCII (7 bits)</li>
<li><code class="code">`Enc_iso8859</code><i>n</i>: ISO-8859-<i>n</i></li>
<li><code class="code">`Enc_koi8r</code>: KOI8-R</li>
<li><code class="code">`Enc_jis0201</code>: JIS-X-0201 (Roman and Katakana)</li>
<li><code class="code">`Enc_eucjp</code>: EUC-JP (code points from US-ASCII, JIS-X-0202, -0208, and
   -0212)</li>
<li><code class="code">`Enc_euckr</code>: EUC-KR (code points from US-ASCII, KS-X-1001)</li>
<li><code class="code">`Enc_windows</code><i>n</i>: WINDOWS-<i>n</i></li>
<li><code class="code">`Enc_cp</code><i>n</i>: IBM code page <i>n</i>. Note that there are both ASCII-
   and EBCDIC-based code pages</li>
<li><code class="code">`Enc_adobe_*</code>: Adobe-specific encodings, e.g. used in Adobe fonts</li>
<li><code class="code">`Enc_mac*</code>: Macintosh-specific encodings</li>
<li><code class="code">`Enc_subset(e,def)</code>: The subset of <code class="code">e</code> by applying the definition 
   function <code class="code">def</code></li>
<li><code class="code">`Enc_empty</code>: The empty encoding (does not represent any character)</li>
</ul>
<br>
</div>

<pre><span class="keyword">type</span> <a name="TYPEcharset"></a><code class="type"></code>charset = <code class="type">[ `Set_adobe_standard_encoding<br>       | `Set_adobe_symbol_encoding<br>       | `Set_adobe_zapf_dingbats_encoding<br>       | `Set_cp037<br>       | `Set_cp1006<br>       | `Set_cp1026<br>       | `Set_cp1047<br>       | `Set_cp424<br>       | `Set_cp437<br>       | `Set_cp500<br>       | `Set_cp737<br>       | `Set_cp775<br>       | `Set_cp850<br>       | `Set_cp852<br>       | `Set_cp855<br>       | `Set_cp856<br>       | `Set_cp857<br>       | `Set_cp860<br>       | `Set_cp861<br>       | `Set_cp862<br>       | `Set_cp863<br>       | `Set_cp864<br>       | `Set_cp865<br>       | `Set_cp866<br>       | `Set_cp869<br>       | `Set_cp874<br>       | `Set_cp875<br>       | `Set_iso88591<br>       | `Set_iso885910<br>       | `Set_iso885911<br>       | `Set_iso885913<br>       | `Set_iso885914<br>       | `Set_iso885915<br>       | `Set_iso885916<br>       | `Set_iso88592<br>       | `Set_iso88593<br>       | `Set_iso88594<br>       | `Set_iso88595<br>       | `Set_iso88596<br>       | `Set_iso88597<br>       | `Set_iso88598<br>       | `Set_iso88599<br>       | `Set_jis0201<br>       | `Set_jis0208<br>       | `Set_jis0212<br>       | `Set_koi8r<br>       | `Set_ks1001<br>       | `Set_macroman<br>       | `Set_unicode<br>       | `Set_usascii<br>       | `Set_windows1250<br>       | `Set_windows1251<br>       | `Set_windows1252<br>       | `Set_windows1253<br>       | `Set_windows1254<br>       | `Set_windows1255<br>       | `Set_windows1256<br>       | `Set_windows1257<br>       | `Set_windows1258 ]</code> </pre>
<div class="info">
A <code class="code">charset</code> is simply a set of code points. It does not say how
 the code points are encoded as bytes. Every encoding implies a certain
 charset (or several charsets) that can be encoded, but the reverse is 
 not true.<br>
</div>

<br>
<b>Pre-evaluation of the encoding argument:</b>
<p>

 A number of the following functions can be made run faster if they are
 called several times for the same encoding. In this case, it is recommended
 to apply the function once partially with the encoding argument, and to
 call the resulting closure instead. For example, <code class="code">ustring_of_uchar</code> supports
 this technique:
<p>

 <pre><code class="code">   let my_ustring_of_uchar = ustring_of_uchar my_enc in
   let s1 = my_ustring_of_uchar u1 ...
   let s2 = my_ustring_of_uchar u2 ... </code></pre>
<p>

 This is <b>much</b> faster than
<p>

 <pre><code class="code">   let s1 = ustring_of_uchar my_enc u1 ...
   let s2 = ustring_of_uchar my_enc u2 ... </code></pre>
<p>

 The availability of this optimization is indicated by the predicate
 PRE_EVAL(<i>arg</i>) where <i>arg</i> identifies the encoding argument.
<p>

 <b>Inlining</b>
<p>

 When a function can be inlined across module/library boundaries,
 this is indicated by the predicate INLINED. Of course, this works
 only for the ocamlopt compiler.<br>
<pre><span class="keyword">val</span> <a name="VALencoding_of_string"></a>encoding_of_string : <code class="type">string -> <a href="Netconversion.html#TYPEencoding">encoding</a></code></pre><div class="info">
Returns the encoding of the name of the encoding. Fails if the 
 encoding is unknown.
 E.g. <code class="code">encoding_of_string "iso-8859-1" = `Enc_iso88591</code> 
<p>

 Punctuation characters (e.g. "-") and year suffixes (e.g.
 ":1991") are ignored.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALstring_of_encoding"></a>string_of_encoding : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> string</code></pre><div class="info">
Returns the name of the encoding.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALis_ascii_compatible"></a>is_ascii_compatible : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> bool</code></pre><div class="info">
"ASCII compatible" means: The bytes 1 to 127 represent the ASCII
 codes 1 to 127, and no other representation of a character contains
 the bytes 1 to 127.
<p>

 For example, ISO-8859-1 is ASCII-compatible because the byte 1 to
 127 mean the same as in ASCII, and all other characters use bytes
 greater than 127. UTF-8 is ASCII-compatible for the same reasons,
 it does not matter that there are multi-byte characters.
 EBCDIC is not ASCII-compatible because the bytes 1 to 127 do not mean
 the same as in ASCII. UTF-16 is not ASCII-compatible because the bytes
 1 to 127 can occur in multi-byte representations of non-ASCII
 characters.
<p>

 The byte 0 has been excluded from this definition because the C
 language uses it with a special meaning that has nothing to do with
 characters, so it is questionable to interpret the byte 0 anyway.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALis_single_byte"></a>is_single_byte : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> bool</code></pre><div class="info">
Returns whether the encoding is a single-byte encoding<br>
</div>
<pre><span class="keyword">val</span> <a name="VALsame_encoding"></a>same_encoding : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> <a href="Netconversion.html#TYPEencoding">encoding</a> -> bool</code></pre><div class="info">
Whether both encodings are the same. <code class="code">`Enc_subset</code> encodings are only
 considered as equal when the definition functions are physically the same.
<p>

 Warning: Don't use ( = ) to compare encodings because this may
 fail.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALbyte_order_mark"></a>byte_order_mark : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> string</code></pre><div class="info">
Returns the byte order mark that must occur at the beginning of
 files to indicate whether "little endian" or "big endian" is used.
 If this does not apply to the encoding, an empty string is returned.
<p>

 See also the section about "<a href="Netconversion.html#bom"><i>Byte Order Marks</i></a>" below.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALmakechar"></a>makechar : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> int -> string</code></pre><div class="info">
<span class="warning">Deprecated.</span>This function is deprecated since ocamlnet-0.96. Use
   <code class="code">ustring_of_uchar</code> instead.<br>
<code class="code">makechar enc i:</code>
 Creates the string representing the Unicode code point <code class="code">i</code> in encoding
 <code class="code">enc</code>. Raises <code class="code">Not_found</code> if the character is legal but cannot be 
 represented in <code class="code">enc</code>.
<p>

 Possible encodings: everything but <code class="code">`Enc_utf16</code>.
<p>

 Evaluation hints:<ul>
<li>PRE_EVAL(encoding)</li>
</ul>

<br>
</div>
<pre><span class="keyword">val</span> <a name="VALustring_of_uchar"></a>ustring_of_uchar : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> int -> string</code></pre><div class="info">
<code class="code">ustring_of_uchar enc i</code>:
 Creates the string representing the Unicode code point <code class="code">i</code> in encoding
 <code class="code">enc</code>. Raises <code class="code">Cannot_represent i</code> if the character is legal but cannot be 
 represented in <code class="code">enc</code>.
<p>

 Possible encodings: everything but <code class="code">`Enc_utf16</code>.
<p>

 Evaluation hints:<ul>
<li>PRE_EVAL(encoding)</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALto_unicode"></a>to_unicode : <code class="type"><a href="Netconversion.html#TYPEcharset">charset</a> -> int -> int</code></pre><div class="info">
Maps the code point of the charset to the corresponding 
 Unicode code point, or raises <code class="code">Malformed_code</code>, when the
 input number does not correspond to a code point.
<p>

 Note <code class="code">`Set_jis0208</code> and <code class="code">`Set_jis0212</code>: Code points are usually
 given by a row and column number. The numeric code point returned by
 this function is computed by multiplying the row number (1..94) with 96,
 and by adding the column number (1..94), i.e. row*96+column.
<p>

 Evaluation hints:<ul>
<li>PRE_EVAL(charset)</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALfrom_unicode"></a>from_unicode : <code class="type"><a href="Netconversion.html#TYPEcharset">charset</a> -> int -> int</code></pre><div class="info">
Maps the Unicode code point to the corresponding code point of
 the charset, or raises <code class="code">Cannot_represent</code> when there is no such
 corresponding code point.
<p>

 Note <code class="code">`Set_jis0208</code> and <code class="code">`Set_jis0212</code>: Code points are usually
 given by a row and column number. The numeric code point returned by
 this function is computed by multiplying the row number (1..94) with 96,
 and by adding the column number (1..94), i.e. row*96+column.
<p>

 Evaluation hints:<ul>
<li>PRE_EVAL(charset)</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALavailable_input_encodings"></a>available_input_encodings : <code class="type">unit -> <a href="Netconversion.html#TYPEencoding">encoding</a> list</code></pre><div class="info">
Returns the list of all available encodings that can be used for
 input strings. The list reflects the set of loadable/linked <code class="code">Netmapping</code>
 modules.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALavailable_output_encodings"></a>available_output_encodings : <code class="type">unit -> <a href="Netconversion.html#TYPEencoding">encoding</a> list</code></pre><div class="info">
Returns the list of all available encodings that can be used for
 output strings. The list reflects the set of loadable/linked <code class="code">Netmapping</code>
 modules.<br>
</div>
<br>
<a name="direct_conv"></a>
<h2>Direct Conversion</h2><br>
<br>
In order to convert a string from one encoding to another, call
 <code class="code">convert</code> like in
<p>

 <pre><code class="code"> let s_utf8 = 
    convert ~in_enc:`Enc_iso88591 ~out_enc:`Enc_utf8 s_latin1 </code></pre>
<p>

 which converts the ISO-8859-1 string <code class="code">s_latin1</code> to the UTF-8 string
 <code class="code">s_utf8</code>.
<p>

 It is also possible to convert while reading from or writing to a file.
 This use case is effectively handled by the class 
 <code class="code">Netconversion.conversion_pipe</code>.
 See the explanations of this class for examples.<br>
<pre><span class="keyword">val</span> <a name="VALconvert"></a>convert : <code class="type">?subst:(int -> string) -><br>       in_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       out_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       ?range_pos:int -> ?range_len:int -> string -> string</code></pre><div class="info">
Converts the string from <code class="code">in_enc</code> to <code class="code">out_enc</code>, and returns it.
 The string must consist of a whole number of characters. If it
 ends with an incomplete multi-byte character, however, this is
 detected, and the exception <code class="code">Malformed_code</code> will be raised.
 This exception is also raised for other encoding errors in the
 input string.
<p>

<br>
</div>
<div class="param_info"><code class="code">subst</code> : This function is invoked for code points of <code class="code">in_enc</code> that
   cannot be represented in <code class="code">out_enc</code>, and the result of the function 
   invocation is substituted (directly, without any further conversion).
   Restriction: The string returned by <code class="code">subst</code> must not be longer than 50
   bytes.
   If <code class="code">subst</code> is missing, <code class="code">Cannot_represent</code> is raised in this case.</div>
<div class="param_info"><code class="code">range_pos</code> : Selects a substring for conversion. <code class="code">range_pos</code>
   is the byte position of the first character of the substring.
   (Default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : Selects a substring for conversion. <code class="code">range_len</code>
   is the length of the substring in bytes (Default: Length
   of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALrecode_string"></a>recode_string : <code class="type">in_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       out_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> ?subst:(int -> string) -> string -> string</code></pre><div class="info">
<span class="warning">Deprecated.</span>This function is obsolete since ocamlnet-0.96. Use
   <code class="code">convert</code> instead.<br>
Recodes a complete string from <code class="code">in_enc</code> to <code class="code">out_enc</code>, and returns it.
 The function <code class="code">subst</code> is invoked for code points of <code class="code">in_enc</code> that cannot
 be represented in <code class="code">out_enc</code>, and the result of the function invocation
 is substituted.
 Restriction: The string returned by <code class="code">subst</code> must not be longer than 50
 bytes.
 If <code class="code">subst</code> is missing, <code class="code">Not_found</code> is raised in this case.
<p>

<br>
</div>
<pre><span class="keyword">val</span> <a name="VALrecode"></a>recode : <code class="type">in_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       in_buf:string -><br>       in_pos:int -><br>       in_len:int -><br>       out_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       out_buf:string -><br>       out_pos:int -><br>       out_len:int -><br>       max_chars:int -> subst:(int -> string) -> int * int * <a href="Netconversion.html#TYPEencoding">encoding</a></code></pre><div class="info">
 Converts the character sequence contained in the at most <code class="code">in_len</code> bytes
 of <code class="code">in_buf</code> starting at byte position <code class="code">in_pos</code>, and writes the result 
 into at most <code class="code">out_len</code> bytes of <code class="code">out_buf</code> starting at byte position
 <code class="code">out_pos</code>. At most <code class="code">max_chars</code> characters are converted from 
 <code class="code">in_buf</code> to <code class="code">out_buf</code>.
<p>

 The characters in <code class="code">in_buf</code> are assumed to be encoded as <code class="code">in_enc</code>, and the 
 characters in <code class="code">out_buf</code> will be encoded as <code class="code">out_enc</code>. The case
 <code class="code">in_enc = out_enc</code> is not handled specially, and is carried out as
 fast as any other conversion.
<p>

 If there is a code point which cannot be represented in <code class="code">out_enc</code>,
 the function <code class="code">subst</code> is called with the code point as argument, and the
 resulting string (which must already be encoded as <code class="code">out_enc</code>) is
 inserted instead. 
 It is possible that <code class="code">subst</code> is called several times for the same
 character. Restriction: The string returned by subst must not be longer
 than 50 bytes.
<p>

 It is allowed that the input buffer ends with an incomplete
 multi-byte character. This character is not converted, i.e. the
 conversion ends just before this character. This special condition
 is not indicated to the caller.
<p>

<br>
<b>Returns</b> The triple <code class="code">(in_n, out_n, in_enc')</code> is returned:<ul>
<li><code class="code">in_n</code> is the actual number of bytes that have been converted from
   <code class="code">in_buf</code>; <code class="code">in_n</code> may be smaller than <code class="code">in_len</code> because of incomplete
   multi-byte characters, or because the output buffer has less space
   for characters than the input buffer, or because of a change
   of the encoding variant.</li>
<li><code class="code">out_n</code> is the actual number of bytes written into <code class="code">out_buf</code>.</li>
<li><code class="code">in_enc'</code> is normally identical to <code class="code">in_enc</code>. However, there are cases
   where the encoding can be refined when looking at the byte
   sequence; for example whether a little endian or big endian variant
   of the encoding is used. <code class="code">in_enc'</code> is the variant of <code class="code">in_enc</code> that was
   used for the last converted character.</li>
</ul>

 If there is at least one complete character in <code class="code">in_buf</code>, and at least
 space for one complete character in <code class="code">out_buf</code>, and <code class="code">max_chars &gt;= 1</code>, it is 
 guaranteed that <code class="code">in_n &gt; 0 &amp;&amp; out_n &gt; 0</code>.<br>
</div>
<pre><span class="keyword">class</span> <a name="TYPEconversion_pipe"></a><a href="Netconversion.conversion_pipe.html">conversion_pipe</a> : <code class="type">?subst:int -> string -> in_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> out_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> unit -> </code><code class="type"><a href="Netchannels.io_obj_channel.html">Netchannels.io_obj_channel</a></code></pre><div class="info">
This pipeline class (see <code class="code">Netchannels</code> for more information) can be used
 to recode a netchannel while reading or writing.
</div>
<pre><span class="keyword">class</span> <a name="TYPErecoding_pipe"></a><a href="Netconversion.recoding_pipe.html">recoding_pipe</a> : <code class="type">?subst:int -> string -> in_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> out_enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> unit -> </code><code class="type"><a href="Netchannels.io_obj_channel.html">Netchannels.io_obj_channel</a></code></pre><div class="info">
<font color="#CCCCCC">Recodes a channel like <code class="code">conversion_pipe</code>.
</font></div>
<br>
<a name="cursors"></a>
<h2>Reading Text Using Cursors</h2>
<p>

 A cursor is a reference to a character in an encoded string. The
 properties of the current character can be obtained, and the cursor
 can be moved relative to its current position.
<p>

 For example, the following loop outputs the Unicode code points
 of all characters of the UTF-8 input string <code class="code">s</code>:
<p>

 <pre><code class="code"> let cs = create_cursor `Enc_utf8 s in
 while not (cursor_at_end cs) do
   let n = cursor_char_count cs in
   let ch = uchar_at cs in
   printf "At position %d: %d\n" n ch;
   move cs;
 done
 </code></pre>
<p>

 For a more exact definition, cursors are modeled as follows: The reference
 to the encoded string is contained in the cursor. This
 can be a complete string, or an arbitrary substring (denoted by a
 range of valid byte positions). The cursor
 position can be initially set to an arbitrary byte position of the
 encoded string.
<p>

 Cursor positions can be denoted by<ul>
<li>byte positions <code class="code">p</code> in the encoded string, or by</li>
<li>character counts <code class="code">n</code> relative to the initial position.</li>
</ul>

 Valid cursor positions are:<ul>
<li><code class="code">n=0</code>: This is always the initial cursor position</li>
<li><code class="code">n&gt;0</code>: Positive char counts refer to characters right to the initial
   character. The rightmost position is the position <code class="code">n_max</code> past the
   rightmost character. The rightmost position does not have a
   code point.</li>
<li><code class="code">n&lt;0</code>: Negative char counts refer to characters left to the initial
   character. The leftmost position is the position <code class="code">n_min</code> of the
   leftmost character.</li>
</ul>

 For the empty string we have <code class="code">n_min = n_max = 0</code>, complementing the
 above definition.
<p>

 Cursors are moved to the left or right of their current position
 by a whole number of characters. When it is tried to move them
 past the leftmost or rightmost position, the cursor is placed to the
 leftmost or rightmost position, respectively, and the exception
 <code class="code">Cursor_out_of_range</code> is raised.
<p>

 There are two cases of illegal encodings:<ul>
<li>When the last byte sequence of the encoded string is an incomplete
   multi-byte character, this is detected, and the special exception
   <code class="code">Partial_character</code> is raised when the code point of this character
   is read. Note that this can only happen at position <code class="code">n_max-1</code>. It
   is allowed to move beyond this character to <code class="code">n_max</code>.</li>
<li>When an illegal byte sequence occurs in the encoded string (including
   an incomplete multi-byte character at the beginning of the string),
   it is not possible to move the cursor to this character, or across
   this character. When it is tried to do so, the cursor stops just
   before the bad sequence, and the exception <code class="code">Malformed_code</code> is
   raised.</li>
</ul>

 It is undefined what happens when the encoded string is modified
 while a cursor is in use referring to it.<br>
<pre><span class="keyword">type</span> <a name="TYPEcursor"></a><code class="type"></code>cursor </pre>
<div class="info">
A cursor denotes a character position in an encoded string<br>
</div>

<pre><span class="keyword">exception</span> <a name="EXCEPTIONEnd_of_string"></a>End_of_string</pre>
<div class="info">
Raised when it is tried to access the character after the end of the
 string (at position <code class="code">n_max</code>)<br>
</div>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONCursor_out_of_range"></a>Cursor_out_of_range</pre>
<div class="info">
Raised when it is tried to move the cursor beyond the beginning of the
 string or beyond the end of the string. In the latter case, it is
 legal to move the cursor to the position following the last character,
 but it is not possible to move it further.<br>
</div>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONPartial_character"></a>Partial_character</pre>
<div class="info">
Raised when the last character of the string is an incomplete
 multi-byte character, and it is tried to get the code point
 (using <code class="code">uchar_at</code>).<br>
</div>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONByte_order_mark"></a>Byte_order_mark</pre>
<div class="info">
Raised when it is tried to get the code point of the BOM at the
 beginning of the string<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcreate_cursor"></a>create_cursor : <code class="type">?range_pos:int -><br>       ?range_len:int -><br>       ?initial_rel_pos:int -><br>       <a href="Netconversion.html#TYPEencoding">encoding</a> -> string -> <a href="Netconversion.html#TYPEcursor">cursor</a></code></pre><div class="info">
Creates a new cursor for the passed string and the passed encoding.
 By default, the allowed range of the cursor is the whole string,
 and the cursor is intially positioned at the beginning of the string.
 The <b>range</b> is the part of the string the cursor can move within.
<p>

 <b>Special behaviour for <code class="code">`Enc_utf16</code>:</b> UTF-16 with unspecified
 endianess is handled specially. First, this encoding is only
 accepted when <code class="code">initial_rel_pos=0</code>. Second, the first two bytes
 must be a byte order mark (BOM) (if the string has a length of two
 bytes or more). The BOM counts as character without code point.
 The function <code class="code">uchar_at</code> raises the exception <code class="code">Byte_order_mark</code>
 when the BOM is accessed. Third, when the cursor is moved to the
 next character, the encoding as returned by <code class="code">cursor_encoding</code> is
 changed to either <code class="code">`Enc_utf16_le</code> or <code class="code">`Enc_utf16_be</code> according
 to the BOM. The encoding changes back to <code class="code">`Enc_utf16</code> when the
 cursor is moved back to the initial position.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : Restricts the range of the cursor to a substring.
   The argument <code class="code">range_pos</code> is the byte position of the beginning
   of the range. (Defaults to 0)</div>
<div class="param_info"><code class="code">range_len</code> : Restricts the range of the cursor to a substring.
   The argument <code class="code">range_len</code> is the length of the range.
   (Default: Length of the input string minus <code class="code">range_pos</code>)</div>
<div class="param_info"><code class="code">initial_rel_pos</code> : The initial position of the cursor, given
   as bytes relative to <code class="code">range_pos</code>. The character at this position
   is considered as the zeroth character of the string (as reported
   by <code class="code">cursor_char_count</code>)</div>
<pre><span class="keyword">val</span> <a name="VALreinit_cursor"></a>reinit_cursor : <code class="type">?range_pos:int -><br>       ?range_len:int -><br>       ?initial_rel_pos:int -><br>       ?enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> string -> <a href="Netconversion.html#TYPEcursor">cursor</a> -> unit</code></pre><div class="info">
Reuses an existing cursor for a new purpose. The arguments are
 as in <code class="code">create_cursor</code>.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcopy_cursor"></a>copy_cursor : <code class="type">?enc:<a href="Netconversion.html#TYPEencoding">encoding</a> -> <a href="Netconversion.html#TYPEcursor">cursor</a> -> <a href="Netconversion.html#TYPEcursor">cursor</a></code></pre><div class="info">
Copies the cursor. The copy can be moved independently of the original
 cursor, but is applied to the same string. The copy starts at the
 byte position of the string where the original cursor is currently
 positioned.
<p>

<br>
</div>
<div class="param_info"><code class="code">enc</code> : Optionally, the assumed
   encoding can be changed to a different one by passing <code class="code">enc</code>.</div>
<pre><span class="keyword">val</span> <a name="VALcursor_target"></a>cursor_target : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> string</code></pre><div class="info">
Returns the string of the cursor
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_range"></a>cursor_range : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int * int</code></pre><div class="info">
Returns the valid range of the cursor as pair <code class="code">(range_pos, range_len)</code> 
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_initial_rel_pos"></a>cursor_initial_rel_pos : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the initial relative byte position of the cursor 
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_char_count"></a>cursor_char_count : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the character count of the cursor. The initial position
 (when <code class="code">create_cursor</code> was called) has the number 0, positions to the
 right denote positive numbers, and positions to the left negative numbers.
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_pos"></a>cursor_pos : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the byte position of the cursor, i.e. the byte index of
 the string that corresponds to the cursor position. The function
 returns the absolute position (i.e. NOT relative to <code class="code">cursor_range</code>).
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALuchar_at"></a>uchar_at : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the Unicode code point of the character at the cursor.
 Raises <code class="code">End_of_string</code> if the cursor is positioned past the last
 character.
 Raises <code class="code">Partial_character</code> if the last character of the analysed 
 string range is an incomplete multi-byte character.
 Raises <code class="code">Byte_order_mark</code> if the first character of the string
 is a BOM (when the encoding has BOMs).
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_byte_length"></a>cursor_byte_length : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the byte length of the representation of the character at the
 cursor. This works also for incomplete multi-byte characters and
 BOMs.
 Raises <code class="code">End_of_string</code> if the cursor is positioned past the last
 character. 
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_at_end"></a>cursor_at_end : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> bool</code></pre><div class="info">
Returns whether the cursor is positioned past the last character.
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALmove"></a>move : <code class="type">?num:int -> <a href="Netconversion.html#TYPEcursor">cursor</a> -> unit</code></pre><div class="info">
Moves the cursor one character to the right, or if <code class="code">num</code> is passed,
 this number of characters to the right. <code class="code">num</code> can be negative in
 which case the cursor is moved to the left.
<p>

 If the cursor were placed outside the valid range, the cursor
 would go into an illegal state, and because of this, this is
 handled as follows: the cursor moves to the
 leftmost or rightmost position (depending on the direction),
 and the exception <code class="code">Cursor_out_of_range</code> is raised.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_encoding"></a>cursor_encoding : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> <a href="Netconversion.html#TYPEencoding">encoding</a></code></pre><div class="info">
Returns the encoding of the cursor. For some encodings, the
 returned encoding depends on the position of the cursor (see
 the note about UTF-8 in <code class="code">create_cursor</code>)
<p>

 Evaluation hints:<ul>
<li>INLINED</li>
</ul>
<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_blit"></a>cursor_blit : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int array -> int -> int -> int</code></pre><div class="info">
<code class="code">cursor_blit cs ua pos len</code>: Copies at most <code class="code">len</code> characters as code
 points from
 the cursor position and the following positions to the array <code class="code">ua</code>
 at index <code class="code">pos</code>. The number of copied characters is returned.
 If the cursor is already at the end of the string when this
 function is called, the exception <code class="code">End_of_string</code> will be raised instead,
 and no characters are copied. The cursor positions containing byte
 order marks and partial characters are never copied; this is ensured
 by stopping the copying procedure just before these positions. This
 may even make the function return the number 0.
<p>

 The function tries to copy as many characters as currently available
 in the already decoded part of the string the cursor is attached to.
 In the current implementation, this number is not higher than 250.
 You can call <code class="code">cursor_blit_maxlen</code> to get an upper limit.
<p>

 The function does not move the cursor.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_blit_maxlen"></a>cursor_blit_maxlen : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int</code></pre><div class="info">
Returns the maximum number of characters <code class="code">cursor_blit</code> can copy
 at the current cursor position. This is the number of characters
 <code class="code">cursor_blit</code> would copy if the <code class="code">len</code> argument were arbitrarily
 large.
<p>

 Note that the value depends on the cursor position and on the
 contents of the cursor string.
<p>

 This function raises <code class="code">End_of_string</code> if the cursor is positioned
 at the end of the string.<br>
</div>
<pre><span class="keyword">val</span> <a name="VALcursor_blit_positions"></a>cursor_blit_positions : <code class="type"><a href="Netconversion.html#TYPEcursor">cursor</a> -> int array -> int -> int -> int</code></pre><div class="info">
Works like <code class="code">cursor_blit</code>, but copies the byte positions of the
 characters into <code class="code">ua</code> instead of the code points.
<p>

 When called directly after <code class="code">cursor_blit</code> for the same cursor and
 with the same value of <code class="code">len</code>, this function copies as many characters
 and thus returns the same number:
<p>

 <pre><code class="code">let n1 = cursor_blit     cs ua ua_pos len in
 let n2 = cursor_blit_pos cs pa pa_pos len in
 assert (n1 = n2)</code></pre><br>
</div>
<br>
<a name="bom"></a>
<h3>Byte Order Marks</h3>
<p>

 Because UTF-16 allows both little and big endian, files and other
 permanent representations of UTF-16 text are usually prepended by
 a byte order mark (BOM). There is confusion about the BOM among
 Unicode users, so the following explanations may be helpful.
<p>

 Of course, the BOM is only used for external representations like
 files, as the endianess is always known for in-memory representations
 by the running program. This module has three encoding identifiers:<ul>
<li><code class="code">`Enc_utf16</code>: UTF-16 where the endianess is unknown</li>
<li><code class="code">`Enc_utf16_le</code>: UTF-16 little endian</li>
<li><code class="code">`Enc_utf16_be</code>: UTF-16 big endian</li>
</ul>

 When a file is read, the endianess is unknown at the beginning.
 This is expressed by <code class="code">`Enc_utf16</code>. When the BOM is read, the encoding
 is refined to either <code class="code">`Enc_utf16_le</code> or <code class="code">`Enc_utf16_be</code>, whatever
 the BOM says. This works as follows: The BOM is the representation
 of the code point 0xfeff as little or big endian, i.e. as byte sequences
 "0xfe 0xff" (big endian) or "0xff 0xfe" (little endian). As the "wrong"
 code point 0xfffe is intentionally unused, the reader can determine
 the endianess.
<p>

 There is one problem, though. Unfortunately, the code point 0xfeff
 is also used for the "zero width non-breakable space" character.
 When this code point occurs later in the text, it is interpreted as
 this character. Of course, this means that one must know whether
 there is a BOM at the beginning, and if not, one must know the
 endianess. One cannot program in the style "well, let's see what is
 coming and guess".
<p>

 Furthermore, the BOM is only used for encodings where one can specify
 the endianess. It must not be used for UTF-8, for example, as the
 byte order is fixed for this encoding. When a UTF-8 text begins with
 the code point 0xfeff, it is always the "zero width non-breakable space"
 character.
<p>

 The functions of this module can all deal with BOMs when reading
 encoded text. In most cases, the BOM is hidden from the caller,
 and just handled automatically. Cursors, however, treat BOMs as special
 characters outside of the code set (exception <code class="code">Byte_order_mark</code> is
 raised). The writing functions of this module do not generate BOMs,
 however, as there is no way to tell them that a BOM is needed. The
 function <code class="code">byte_order_mark</code> can be used to output the BOM manually.
<p>

 <a name="3_ExamplesforCursors"></a>
<h3>Examples for Cursors</h3>
<p>

 Create the cursor:
<p>

 <code class="code"> let cs = create_cursor `Enc_utf8 "B\195\164r";; </code>
<p>

 The cursor is now positioned at the 'B':
<p>

 <code class="code"> uchar_at cs </code> <i>returns</i> <code class="code">66</code> (i.e. B)
<p>

 Move the cursor one character to the right. In UTF-8, this is a
 two-byte character consisting of the bytes 195 and 164:
<p>

 <code class="code"> move cs ;; </code>
<p>

 <code class="code"> uchar_at cs </code> <i>returns</i> <code class="code">228</code> (i.e. a-Umlaut)
<p>

 One can easily move the cursor to the end of the string:
<p>

 <code class="code"> move ~num:max_int cs ;; </code>
<p>

 This raises <code class="code">Cursor_out_of_range</code>, but places the cursor at the end.
 This is the position past the last letter 'r':
<p>

 <code class="code"> uchar_at cs </code> <i>raises</i> <code class="code">End_of_string</code>
<p>

 Go one character to the left:
<p>

 <code class="code"> move ~num:(-1) cs ;; </code>
<p>

 <code class="code"> uchar_at cs </code> <i>returns</i> <code class="code">114</code> (i.e. r)
<p>

 Cursors can only move relative to their current position. Of course,
 one can easily write a function that moves to an absolute position,
 like
<p>

 <pre><code class="code"> let move_abs n cs = 
    let delta = n - cursor_pos cs in
    move ~num:delta cs </code></pre>
<p>

 However, this operation is expensive (O(string length)), and should
 be avoided for efficient algorithms. Cursors are not arrays, and an
 algorithm should only be based on cursors when it is possible to
 iterate over the characters of the string one after another.<br>
<br>
<a name="unicode_functions"></a>
<h2>Unicode String Functions</h2><br>
<pre><span class="keyword">val</span> <a name="VALustring_length"></a>ustring_length : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> ?range_pos:int -> ?range_len:int -> string -> int</code></pre><div class="info">
Returns the length of the string in characters. The function fails
 when illegal byte sequences or incomplete characters are found in the
 string with <code class="code">Malformed_code</code>.
<p>

 Evaluation hints:<ul>
<li>PRE_EVAL(encoding)</li>
</ul>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to measure
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to measure
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALustring_iter"></a>ustring_iter : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       (int -> unit) -> ?range_pos:int -> ?range_len:int -> string -> unit</code></pre><div class="info">
Iterates over the characters of a string, and calls the passed function
 for every code point. The function raises <code class="code">Malformed_code</code> when
 illegal byte sequences or incomplete characters are found.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to iterate over
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to iterate over
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALustring_map"></a>ustring_map : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       (int -> int list) -> ?range_pos:int -> ?range_len:int -> string -> string</code></pre><div class="info">
Maps every character of a string to a list of characters, and returns
 the concatenated string. 
 The <code class="code">encoding</code> argument determines the encoding of both the argument
 and the result string.
 The map function gets every character as its Unicode code point, and
 must return the list of code points to map to.
<p>

 The function raises <code class="code">Malformed_code</code> when
 illegal byte sequences or incomplete characters are found.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to map
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to map
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALustring_sub"></a>ustring_sub : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       int -> int -> ?range_pos:int -> ?range_len:int -> string -> string</code></pre><div class="info">
<code class="code">ustring_sub enc start length s</code>: Returns the substring of <code class="code">s</code> starting
 at character count <code class="code">start</code> and consisting of <code class="code">length</code> characters. Note
 that <code class="code">start</code> and <code class="code">length</code> select the substring by multiples of
 (usually multibyte) characters, not bytes.
<p>

 If the optional byte-based <code class="code">range_pos</code> and <code class="code">range_len</code> arguments are
 present, these arguments are taken to determine a first substring
 before <code class="code">start</code> and <code class="code">length</code> are applied to extract the final
 substring.
<p>

 The function raises <code class="code">Malformed_code</code> when
 illegal byte sequences or incomplete characters are found.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to extract
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to extract
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALustring_compare"></a>ustring_compare : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       (int -> int -> int) -><br>       ?range_pos:int -><br>       ?range_len:int -> string -> ?range_pos:int -> ?range_len:int -> string -> int</code></pre><div class="info">
Compares two strings lexicographically. The first argument is the
 encoding of both strings (which must be the same). The second argument
 is the function that compares two Unicode code points. It must return
 0 if both characters are the same, a negative value if the first
 character is the smaller one, and a positive value if the second
 character is the smaller one.
<p>

 The function raises <code class="code">Malformed_code</code> when
 illegal byte sequences or incomplete characters are found.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to compare
   (default: 0), referring to the following string argument</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to compare
   (default: byte length of the input string minus <code class="code">range_pos</code>),
   referring to the following string argument</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to compare
   (default: 0), referring to the following string argument</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to compare
   (default: byte length of the input string minus <code class="code">range_pos</code>),
   referring to the following string argument</div>
<pre><span class="keyword">val</span> <a name="VALuarray_of_ustring"></a>uarray_of_ustring : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -><br>       ?range_pos:int -> ?range_len:int -> string -> int array</code></pre><div class="info">
Returns the characters of the string as array of Unicode code points.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to extract
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to extract
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
<pre><span class="keyword">val</span> <a name="VALustring_of_uarray"></a>ustring_of_uarray : <code class="type">?subst:(int -> string) -><br>       <a href="Netconversion.html#TYPEencoding">encoding</a> -> ?pos:int -> ?len:int -> int array -> string</code></pre><div class="info">
Returns the array of Unicode code points as encoded string.
<p>

<br>
</div>
<div class="param_info"><code class="code">subst</code> : This function is called when a code point cannot be represented
   in the chosen character encoding. It must returns the (already encoded)
   string to substitute for this code point. By default (if ~subst is
   not passed), the exception <code class="code">Cannot_represent</code> will be raised in this
   case.</div>
<div class="param_info"><code class="code">pos</code> : Selects a subarray: <code class="code">pos</code> is the first array position
   to encode (default: 0)</div>
<div class="param_info"><code class="code">len</code> : Selects a subarray: <code class="code">len</code> is the length of the subarray
   to encode (default: array length minus <code class="code">pos</code>)</div>
<pre><span class="keyword">exception</span> <a name="EXCEPTIONMalformed_code_at"></a>Malformed_code_at <span class="keyword">of</span> <code class="type">int</code></pre>
<div class="info">
An illegal byte sequence is found at this byte position<br>
</div>
<pre><span class="keyword">val</span> <a name="VALverify"></a>verify : <code class="type"><a href="Netconversion.html#TYPEencoding">encoding</a> -> ?range_pos:int -> ?range_len:int -> string -> unit</code></pre><div class="info">
Checks whether the string is properly encoded. If so, () is returned.
 If not, the exception <code class="code">Malformed_code_at</code> will be raised indicating 
 the byte position where the problem occurs.
<p>

<br>
</div>
<div class="param_info"><code class="code">range_pos</code> : The byte position of the substring to verify
   (default: 0)</div>
<div class="param_info"><code class="code">range_len</code> : The byte length of the substring to verify
   (default: byte length of the input string minus <code class="code">range_pos</code>)</div>
</body></html>