% \iffalse
%% File: l3io.dtx Copyright (C) 1990-1998 LaTeX3 project
%
%<*dtx>
          \ProvidesFile{l3io.dtx}
%</dtx>
%<package>\NeedsTeXFormat{LaTeX2e}
%<package>\ProvidesPackage{l3io}
%<driver> \ProvidesFile{l3io.drv}
% \fi
%         \ProvidesFile{l3io.dtx}
          [1998/04/20 v1.0c L3 Experimental i/o module]
%
% \iffalse
%<*driver>
\documentclass{l3doc}

\begin{document}
\DocInput{l3io.dtx}
\end{document}
%</driver>
% \fi
%
% \section{Low-level file i/o}
%
% \TeX{} is capable of reading from and writing up to 16 individual
% streams. These i/o operations are accessable in \LaTeX3 with functions
% from the "\io.."  modules. In most cases it will be
% sufficiant for the programmer to use the functions provided by the
% auxiliary file module, but here are the necessary functions for
% manipulating private streams.
%
% Sometimes it is not know beforehand how much text is going to be
% written with a single call. As a result some internal \TeX{} buffer
% may overflow. To avoid this kind of problem, \LaTeX3 maintains beside
% direct write operations like "\iow_expanded:Nn" also so called ``long''
% writes where the output is broken into individual lines on every blank
% in the text to be written. The resulting files are difficult to read
% for humans but since they usually serve only as internal storage this
% poses no problem.
%
% Beside the functions that immediately act (e.g., "\iow_expanded:Nn",
% etc.) we also have deferred operations that are saved away until the
% next page is finished. This allow to expand the <tokens> at the right
% time to get correct page numbers etc.
%
% \subsection{Functions for output streams}
%
% \begin{function}{\iow_new:N |
%                  \iow_new:c
% }
% \begin{syntax}
%   "\iow_new:N" <stream>
% \end{syntax}
% Defines <stream> to be a new identifer denoting an output stream for
% use in subsequent functions.
% \begin{texnote}
% "\iow_new:N" corresponds to the plain \TeX{} \tn{newwrite}
% allocation routine.
% \end{texnote}
% \end{function}
%
% \begin{function}{\iow_open:Nn |
%                  \iow_open:cn
% }
% \begin{syntax}
%   "\iow_open:Nn" <stream> "{" <file name> "}"
% \end{syntax}
% Opens output stream <stream> to write to <file name>. The output
% stream is immediately available for use. If the <stream> was already
% used as an output stream to some other file, this file gets closed
% first.\footnote{This is a precaution since on some OS it is possible
% to open the same file for output more than once which then results in
% some internal errors at the end of the run.} Also, all output streams
% still open at the end of the \TeX{} run will be automatically closed.
% \end{function}
%
% \begin{function}{\iow_expanded:Nn |
%                  \iow_unexpanded:Nn
% }
% \begin{syntax}
%  "\iow_expanded:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% This function immediately writes the expansion of <tokens> to the
% output stream <stream>. If <stream> is not open output goes to the
% terminal. The variant "\iow_unexpanded:Nn" writes out <tokens> without any
% further expansion (verbatim).
% \end{function}
%
% \begin{function}{\iow_expanded_log:n |
%                  \iow_expanded_term:n |
%                  \iow_unexpanded_term:n
% }
% \begin{syntax}
%  "\iow_expanded_log:n" "{" <tokens> "}"
% \end{syntax}
% These functions write to the transcript or to the terminal
% respectively. So they are equivalent to "\iow_expanded:Nn" where <stream>
% is the transcript file ("\c_iow_log_stream") or the terminal
% ("\c_io_term_stream").
% \end{function}
%
% \begin{function}{\iow_long_expanded:Nx |
%                  \iow_long_unexpanded:Nn
% }
% \begin{syntax}
%  "\iow_long_expanded:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% Like "\iow_expanded:Nn" but splits <tokens> at every blank into separate
% lines.
% \end{function}
%
% \begin{function}{\iow_unexpanded_if_avail:Nn|
%                  \iow_unexpanded_if_avail:cn|}
% \begin{syntax}
%  "\iow_unexpanded_if_avail:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% This special function first checks if the <stream> is open of writing.
% If not it does nothing otherwise it behaves like "\iow_unexpanded:Nn".
% \end{function}
%
% \begin{function}{\iow_deferred_expanded:Nn |
%                  \iow_deferred_unexpanded:Nn}
% \begin{syntax}
%   "\iow_deferred_expanded:Nn"  <stream> "{" <tokens> "}"
% \end{syntax}
% These functions save away <tokens> until the next page is ready to be
% shipped out. Then, in case of "\iow_deferred_expanded:Nn" <tokens> get
% expanded and afterwards written to <stream>. "\iow_deferred_expanded:Nn"
% also always needs "{}" around the second argument. The use of
% "\iow_deferred_unexpanded:Nn" is probably seldom necessary.
% \begin{texnote}
% "\iow_deferred_expanded:Nn" was known as \tn{write}.
% \end{texnote}
% \end{function}
%
% \begin{function}{\iow_newline:}
% \begin{syntax}
%   "\iow_newline:"
% \end{syntax}
% Function that produces a new line when used within the <token list>
% that gets written some output stream in non-verbatim mode.
% \end{function}
%
% \subsection{Functions for input streams}
%
% \begin{function}{\ior_new:N
% }
% \begin{syntax}
%   "\ior_new:N" <stream>
% \end{syntax}
% This function defines <stream> to be a new input stream constant.
% \begin{texnote}
% This is the new name and new implementation for plain \TeX's
% \tn{newread}.
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_open:Nn}
% \begin{syntax}
%   "\ior_open:Nn" <stream> "{" <file name> "}"
% \end{syntax}
% This function opens <stream> as an input stream for the external file
% <file name>. If <file name> doesn't exist or is an empty file the
% stream is considered to be fully read, a condition which can be tested
% with "\ior_eof:NTF" etc. If <stream> was already used to read from some
% other file this file will be closed first. The input stream is ready
% for immediate use.
% \end{function}
%
% \begin{function}{\ior_close:N}
% \begin{syntax}
%   "\ior_close:N" <stream>
% \end{syntax}
% This function closes the read stream <stream>.
% \begin{texnote}
% This is a new name for \tn{closein} but it is considered bad practice
% to make use of this knowledge :-)
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_eof:NTF |
%                  \ior_eof:NF}
% \begin{syntax}
%   "\ior_eof:NTF" <stream> "{" <true code> "}{" <false code> "}"
% \end{syntax}
% Conditional that tests if some input stream is fully read. The
% condition is also true if the input stream is not open.
% \end{function}
%
% \begin{function}{\if_eof:w}
% \begin{syntax}
%   "\if_eof:w" <stream> <true code> "\else:" <false code> "\fi:"
% \end{syntax}
% \begin{texnote}
% This is the primitive \tn{ifeof} but we allow only a <stream> and not
% a plain number after it.
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_to:NN |
%                  \ior_gto:NN}
% \begin{syntax}
%   "\ior_to:NN" <stream> <tlp>
% \end{syntax}
% Functions that reads one or more lines (until an equal number of left
% and right braces are found) from the input stream <stream> and places
% the result locally or globally into <tlp>. If <stream> is not open
% input is requested from the terminal.
% \end{function}
%
% \subsection{Constants}
%
% \begin{variable}{\c_iow_comment_char |
%                  \c_iow_lbrace_char  |
%                  \c_iow_rbrace_char}
% Constants that can be used to represent comment character, left and
% right brace in token lists that should be written to a file.
% \end{variable}
%
% \begin{variable}{\c_io_term_stream}
% Input or output stream denoting the terminal. If used as an input
% stream the user is prompted with the name of the <tlp> (that is used
% in the call "\ior_to:NN" or "\ior_gto:NN") followed by an equal sign.
% If you don't want an automatic prompt of this sort ``misuse''
% "\c_iow_log_stream" as an input stream.
% \end{variable}
%
% \begin{variable}{\c_iow_log_stream}
% Output stream that writes only to the transcript file (e.g., the
% {\tt.log} file on most systems). You may ``misuse'' this stream as an
% input stream. In this case it acts as a terminal stream without user
% prompting.
% \end{variable}
%
% \begin{variable}{\g_iow_newline_code}
% Global variable holding the character number of the character that
% denotes a new line when something is written to an output stream.
% \begin{texnote}
% A.k.a \tn{newlinechar}.
% \end{texnote}
% \end{variable}
%
% \subsection{Internal functions}
%
% \begin{function}{\iow_long_expanded_aux:w}
%  Function used to implement immediate writing where a new line is
% started at every blank.
% \end{function}
%
% \begin{function}{%
%                  \tex_read:D |
%                  \tex_immediate:D |
%                  \tex_closeout:D |
%                  \tex_openin:D |
%                  \tex_openout:D |
% }
% These are the functions of the primitive interface to \TeX.
% \begin{texnote}
% The \TeX{} primitives \tn{read}, \tn{immediate}, \tn{closeout},
% \tn{openin}, and \tn{openout} are all renamed and should not be used
% by a programmer since the functionality is covered by the \LaTeX3
% functions above.
% \end{texnote}
% \end{function}
%
%
% \section{Lowlevel i/o-commands}
%
%    We start by ensuring that the required packages are loaded.
%    \begin{macrocode}
%<package&check>\RequirePackage{l3chk}\par
%<package>\RequirePackage{l3toks}\par
%<*package>
%    \end{macrocode}
%
% This section is primarily concerned with input and output streams.
% The naming conventions for i/o streams is |ior| (for read) and |iow|
% (for write) as module names. e.g.\ |\c_ior_test_stream| is an input stream
% variable called `test'.
%
% \subsection{Output streams}
%
% \begin{macro}{\iow_new:N}
% \begin{macro}{\iow_new:c}
%    Allocation of new output streams is done by these functions.
%    As we currently do not distribute a new allocation module we nick
%    the |\newwrite| function.
%    \begin{macrocode}
\def_new:Npn \iow_new:N {}
\let:NN \iow_new:N \newwrite
\def_new:Npn \iow_new:c {\exp_args:Nc \iow_new:N}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_open:Nn}
% \begin{macro}{\iow_open:cn}
%    To open streams for reading or writing the following two functions
%    are provided. The streams are opened immediately.
%
%    From some bad experiences on the mainframe, I learned that it is
%    better to force the close before opening a dataset for writing. 
%    We have to check whether this is also necessary in case of
%    |\tex_openin:D|.
%    \begin{macrocode}
\def_new:Npn \iow_open:Nn #1#2{\iow_close:N #1
     \tex_immediate:D\tex_openout:D#1#2\scan_stop:}
\def_new:Npn \iow_open:cn {\exp_args:Nc \iow_open:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_close:N}
%    Since we close output streams prior to opening, a separate closing
%    operation is probably not necessary. But here it is, just in
%    case\ldots. Actually you will need this if you intend to write
%    and then read in the same pass from some stream.
%    \begin{macrocode}
\def_new:Npn \iow_close:N {\tex_immediate:D\tex_closeout:D}
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\c_io_term_stream}
% \begin{macro}{\c_iow_log_stream}
%    Here we allocate two output streams for writing to the transcript
%    file only (|\c_iow_log_stream|) and to both the terminal and transcript
%    file (|\c_io_term_stream|). The latter can also be used to read
%    from therefore it is called |..io_..|.
%    \begin{macrocode}
\let_new:NN \c_io_term_stream \c_sixteen
\let_new:NN \c_iow_log_stream \c_minus_one
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \subsubsection{Immediate writing}
%
% \begin{macro}{\iow_expanded:Nn}
%    An abbreviation for an often used operation, which immediately
%    writes its second argument to the output stream.
%    \begin{macrocode}
\def_new:Npn \iow_expanded:Nn {\tex_immediate:D\iow_deferred_expanded:Nn}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\iow_unexpanded:Nn}
%    This routine writes the second argument verbatim onto the output
%    stream. If this stream isn't open, the output goes to the terminal.
%    If the first argument is no output stream at all, we get an
%    internal error.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded:Nn #1#2{\toks_gset:Nn \g_tmpa_toks {#2}
   \iow_expanded:Nn #1{\toks_use:N \g_tmpa_toks}}
%    \end{macrocode}
% \end{macro}
%
%
%  \begin{macro}{\iow_expanded_log:n}
%  \begin{macro}{\iow_expanded_term:n}
%    Now we redefine two functions for which we needed a definition
%    very early on. They both write their second argument fully
%    expanded to the output stream.
%    \begin{macrocode}
\def:Npn \iow_expanded_log:n {\iow_expanded:Nn \c_iow_log_stream}
\def:Npn \iow_expanded_term:n{\iow_expanded:Nn \c_io_term_stream}
%    \end{macrocode}
%    The second one isn't exactly equivalent to the old |\typeout| since
%    we need to control expansion in the function we provide for the user.
%  \end{macro}
%  \end{macro}

% \begin{macro}{\iow_unexpanded_term:n}
%    This function writes its argument verbatim to the the terminal.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_term:n {\iow_unexpanded:Nn \c_io_term_stream}
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\iow_unexpanded_if_avail:Nn}
% \begin{macro}{\iow_unexpanded_if_avail:cn}
%    |\iow_unexpanded_if_avail:Nn | \m{stream} \m{code}.  This routine writes
%    its second argument unexpanded to the stream given by the first
%    argument, provided that this stream was opened for writing. Note,
%    that |#| characters get doubled within \m{code}.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_if_avail:Nn #1{
%    \end{macrocode}
%    In this routine we have to check whether or not the output stream
%    that was requested is defined at all. 
%    So we check if the name is still free.
%    \begin{macrocode}
  \cs_free:NTF #1\use_none:n {\iow_unexpanded:Nn #1}}
%    \end{macrocode}
%    Note: the next function could be streamlined for speed if we use
%    the faster |\cs_free:cTF|. (space viz time).
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_if_avail:cn {
         \exp_args:Nc \iow_unexpanded_if_avail:Nn }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_long_expanded:Nn}
% \begin{macro}{\iow_long_unexpanded:Nn}
% \begin{macro}{\iow_long_expanded_aux:w}
%    Another type of writing onto an output stream is used for
%    potentially long token sequences. We break the output lines at
%    every blank in the second argument. This avoids the problem of
%    buffer overflow when reading back, or badly broken lines on
%    systems with limited file records.  The only thing we have to
%    take care of, is the danger of two blanks in succession since
%    these get converted into a |\par| when we read the stuff back.
%    But this can happen only if things like
%    two spaces find their way into the second argument.
%    Usually, multiple spaces are removed by \TeX's scanner.
%
%    \begin{macrocode}
\def_new:Npn \iow_long_expanded_aux:w #1#2#3{
   \group_begin:\g_iow_newline_code`\ #1#2{#3}\group_end:}
\def_new:Npn \iow_long_expanded:Nn {\iow_long_expanded_aux:w 
                                    \iow_expanded:Nn}
\def_new:Npn \iow_long_unexpanded:Nn {\iow_long_expanded_aux:w 
                                      \iow_unexpanded:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
%
% \subsubsection{Deferred writing}
%
% Deferred writing to output streams is a bit more complicated because
% there seems to be no nice hack for writing unexpanded. The only
% relatively sure bet is to use |\token_to_meaning:N| expansion of some token
% list.  That's the way the following functions are implemented.
%
% Another possibility would be to reserve a certain number of scratch
% token registers that could be used to hold the tokens until after the
% next |\tex_shipout:D|. But such an approach would probably fail because of
% the limited number of available token registers that would need to
% be reserved for this special application.
%
%
% \begin{macro}{\iow_deferred_expanded:Nn}
%    First the easy part, this is the primitive.
%    \begin{macrocode}
\let:NN \iow_deferred_expanded:Nn \tex_write:D
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\iow_deferred_unexpanded:Nn}
%    Now the harder part:
%    \begin{macrocode}
\def_new:Npn \iow_deferred_unexpanded:Nn #1#2{
  \tlp_set:Nn \l_tmpa_tlp {#2}
  \tlp_set:Nx \l_tmpb_tlp 
              {\iow_deferred_expanded:Nn #1{\tlp_to_str:N \l_tmpa_tlp}}
  \l_tmpb_tlp}
%    \end{macrocode}
% \end{macro}
%
% Long forms of these functions are not possible since the deferred
% writing will restore the value of |\g_iow_newline_code| before it will have
% a chance to act. But on the other hand it is nevertheless possible
% to make all deferred writes long by setting the |\g_iow_newline_code|
% inside the output routine just before the |\tex_shipout:D|. The only
% disadvantage of this method is the fact that messages to the
% terminal during this time will also then break at spaces. But we
% should consider this.
%
% \subsubsection{Special characters for writing}
%
%
% \begin{macro}{\c_iow_comment_char}
% \begin{macro}{\c_iow_lbrace_char}
% \begin{macro}{\c_iow_rbrace_char}
%   We also need to be able to write braces and the comment character.  We
%   achieve this by defining global constants to expand into a version of
%   these characters with |\tex_catcode:D|${}=12$.
%    \begin{macrocode}
\tex_catcode:D `\%= 12 \scan_stop:
\tlp_new:Nn \c_iow_comment_char {%}
\tex_catcode:D `\%= 14 \scan_stop:
%    \end{macrocode}
%    To avoid another allocation function which is probably only
%    necessary here we use the |\def:Npx| command directly.
%    \begin{macrocode}
\tlp_new:Nn \c_iow_lbrace_char{}
\tlp_new:Nn \c_iow_rbrace_char{}
\def:Npx\c_iow_lbrace_char {\token_to_string:N{}
\def:Npx\c_iow_rbrace_char {\token_to_string:N}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
%
% \subsection{Input streams}
%
% \begin{macro}{\ior_new:N}
%    Allocation of new input streams is done by this function.
%    As we currently do not distribute a new allocation module we nick
%    the |\newwread| function.
%    \begin{macrocode}
\def_new:Npn \ior_new:N {}
\let:NN \ior_new:N \newread
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ior_open:Nn}
%    Processing of input-streams (via |\tex_openin:D| and |closein|) is
%    always `immediate' as far as \TeX{} is concerned. An extra
%    |\tex_immediate:D| is silently ignored.
%    \begin{macrocode}
\def_new:Npn \ior_open:Nn #1#2{\ior_close:N #1\scan_stop:
                               \tex_openin:D#1#2\scan_stop:}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ior_eof:NTF}
%    |\ior_eof:NTF | \m{stream} \m{true case} \m{false case}.  To
%    test if some particular input stream is exhausted the following
%    conditional is provided:
%    \begin{macrocode}
\def_new:Npn \ior_eof:NTF #1{\if_eof:w#1
     \exp_after:NN\use_choice_i:nn \else:
     \exp_after:NN\use_choice_ii:nn \fi:}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ior_eof:NF}
%    |\ior_eof:NF | \m{stream}  \m{false case}.  Do something if
%    if there is still something to read from this file:
%    \begin{macrocode}
\def_new:Npn \ior_eof:NF #1{\if_eof:w#1
        \exp_after:NN \use_none:nn \fi: \use:n}
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\ior_to:NN}
% \begin{macro}{\ior_gto:NN}
%    And here we read from files.
%    \begin{macrocode}
%<*check>
\def_new:Npn \ior_to:NN #1#2{\tex_read:D#1to#2
   \chk_local_or_pref_global:N #2}
%</check>
%<-check> \def_new:Npn \ior_to:NN #1{\tex_read:D#1to}
\def_new:Npn \ior_gto:NN {
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \ior_to:NN}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
%    Show token usage:
%    \begin{macrocode}
%</package>
%<*showmemory>
\showMemUsage
%</showmemory>
%    \end{macrocode}
%
%