/** -*-C-*-ish
    HTMLDocument.k Copyright (C) 2005 Chris Morris

    This file is distributed under the terms of the GNU Lesser General
    Public Licence. See COPYING for licence.
*/
"<summary>HTML Document creation and editing</summary>
<prose>This module contains functions for creating and editing a HTML document using a tree-based approach. This ensures that documents are of a high code quality, makes maintenance easier, and provides protection against some common web security problems.</prose>"
module HTMLDocument;

import Prelude;
import System;
import Crypto;
import Strings;
import Regex;
import public ElementTreeData; // ideally import abstract, if such a thing existed
import ElementTree;
import HTMLnesting;
import XMLentities;
import WebCommon;
import Compress;
import Set;

/** Part 1: Data type definitions **/

globals {
  Int uniqueid = 0;
  Regex::Regex classregex = compile("^[A-Za-z][A-Za-z0-9-]*$");
  Regex::Regex idregex = compile("^[A-Za-z][A-Za-z0-9_:.-]*$");
}

/* Development notes:
 * We want a special data type for URIs so we can handle encoding properly
 * String will do for now, though.
 */

"<summary>A HTML document</summary>
<prose>This data type represents a HTML document. The doctype is set when an instance of this type is created. The document can be converted to a string, with each field forming a different part</prose>
<example>&lt;DOCTYPE ...> &lt;!-- from doctype field -->
&lt;html>
  &lt;head>
&lt;!-- links, stylesheets, scripts and metadata from the head field >
  &lt;/head>
  &lt;body>
&lt;!-- a HTML or XHTML element tree manipulated with the
     functions in this module -->
  &lt;/body>
&lt;/html></example>
<prose>The <code>httpheaders</code> field contains details of HTTP headers to be set if the document is served over HTTP (such as in a CGI program or Webapp)</prose>
<related><dataref>Doctype</dataref></related>
<related><dataref>ElementTreeData::ElementTree</dataref></related>
<related><dataref>MetaData</dataref></related>
<related><functionref>addHTTPHeader</functionref></related>
<related><functionref>new</functionref></related>
<related><functionref>setDocumentTitle</functionref></related>
<related><moduleref>CGI</moduleref></related>
<related><moduleref>Webapp</moduleref></related>"
public data HTMLDocument(MetaData head, ElementTree body, Doctype doctype, [Pair<String,String>] httpheaders);

// we don't support meta HTTP-equiv here because that's better handled via
// real HTTP headers, which we may add support for later in HTMLDocument
// abstract because it shouldn't be poked directly
"<summary>Meta data about a HTML document</summary>
<prose>This data type is used to create the <code>&lt;head></code> section of a HTML document. The following things can be added using this data type.</prose>
<list><item>Meta data <code>&lt;meta name='...' value='...'></code></item>
<item>External CSS Stylesheets</item>
<item>Document-level links (<code>&lt;link></code>)</item>
<item>External client-side Javascript</item></list>
<related><dataref>ClientScript</dataref></related>
<related><dataref>HeadLink</dataref></related>
<related><dataref>HTMLDocument</dataref></related>
<related><dataref>StyleSheet</dataref></related>
<related><functionref>addDocumentLink</functionref></related>
<related><functionref>addDocumentMetaData</functionref></related>
<related><functionref>addDocumentScripting</functionref></related>
<related><functionref>addDocumentStylesheet</functionref></related>
<related><functionref>setDocumentTitle</functionref></related>"
abstract data MetaData(String doctitle,
		     Dict<String,String> metakeys,
		     [StyleSheet] styles, // order is significant
		     [HeadLink] links,
		     [ClientScript] scripts);
// yes, I know, it's missing <script> and <style> support. Can't be
// bothered right now. <style> is better handled by stylesheets anyway

"<summary>The document type declaration.</summary>
<prose>The document type declaration for the current document. You can either use HTML 4.01 or XHTML 1.0 Strict doctypes (HTML is generally recommended due to the current lack of browser support for XHTML, but you may need XHTML output if you intend to further process the document with other XML tools). This changes the <code>&lt;!DOCTYPE ...></code> declaration at the top of the document, and has other minor effects on code output.</prose>
<prose>The Transitional doctypes are not supported in Kaya - these doctypes are intended only for transitioning legacy documents, and Kaya is sufficiently new not to have any such legacy documents existing.</prose>
<prose>The <code>TagSoup</code> 'Doctype' cannot be used for output, but may be useful for attempting to get some form of structure out of poor quality markup entered by users and processed by <functionref>readFromString</functionref>.</prose>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref>new</functionref></related>
<related><functionref>readFromString</functionref></related>
<related><link url=\"http://www.w3.org/QA/2002/04/valid-dtd-list.html\">W3C list of Doctypes</link></related>"
public data Doctype = HTML4Strict | XHTML1Strict | TagSoup;

"<summary>A CSS stylesheet</summary>
<prose>An external CSS stylesheet.</prose>
<related><dataref>MediaType</dataref></related>
<related><dataref>MetaData</dataref></related>
<related><functionref>addDocumentStylesheet</functionref></related>"
public data StyleSheet([MediaType] media, String uri);
// TODO: extend this to potentially support non-text/css stylesheets
// while keeping text/css as the default behaviour.

"<summary>An client-side script</summary>
<prose>An external client-side script (currently only Javascript is supported).</prose>
<related><dataref>MetaData</dataref></related>
<related><functionref>addDocumentScripting</functionref></related>"
public data ClientScript(String uri);
// TODO: extend to other than text/javascript

"<summary>A display media type</summary>
<prose>A display media type. This is used to select appropriate stylesheets for different browsing situations. <code>MTscreen</code> is the usual type for most CSS-supporting browsers. <code>MTprint</code> can be used to provide an alternative layout for printed documents (perhaps hiding navigation bars, etc). <code>MTall</code> applies the styles to all media.</prose>
<related><dataref>StyleSheet</dataref></related>
<related><functionref>addDocumentStylesheet</functionref></related>"
public data MediaType = MTscreen | MTtty | MTtv | MTprojection | MThandheld | MTprint | MTbraille | MTaural | MTall;

// lots more attributes that could be added here
"<summary>A document-level link</summary>
<prose>A document-level link. These links differ from hyperlinks in the body of the text in that they describe the relationship of this document as a whole to other documents.</prose>
<related><dataref>MetaData</dataref></related>
<related><dataref>Relationship</dataref></related>
<related><functionref>addDocumentLink</functionref></related>"
public data HeadLink(Relationship relate, String uri, String title, String ctype);

"<summary>A document relationship</summary>
<prose>A document relationship may either be 'forward' (<code>Rel</code>), which states that the related resource is the <variable>rel</variable> document of the current document, or 'reverse' (<code>Rev</code>), which is equivalent to that resource having a 'forward' relationship to this document. If the resource is not a HTML document, then reverse relationships may be the only way to express the relationship, but in practice forward relationships are far more useful.</prose>
<prose>There is no official list of link types, and <link url=\"http://webcoder.info/reference/LinkBars.html\">browser support</link> varies. Some common relationships such as 'next' or 'first' have consistent support, though.</prose>
<prose>A link type of <code>Rel(\"shortcut icon\")</code> is used to reference a 'favicon'.</prose>
<related><dataref>HeadLink</dataref></related>
<related><dataref>MetaData</dataref></related>
<related><functionref>addDocumentLink</functionref></related>"
public data Relationship = Rel(String rel) | Rev(String rev);

// TODO: Needs a lot more inline elements to be added.
"<summary>Inline elements</summary>
<prose>Non-empty inline elements that can be included within a HTML document. Example open tags of the HTML elements produced are below - see the <link url=\"http://www.w3.org/TR/HTML4/\">HTML specifications</link> for their semantics and uses.</prose>
<list>
<item><input>Emphasis</input> - <output>&lt;em></output></item>
<item><input>StrongEmphasis</input> - <output>&lt;strong></output></item>
<item><input>Abbreviation(\"Algebraic Data Type\")</input> - <output>&lt;abbr title=\"Algebraic Data Type\"></output></item>
<item><input>Citation</input> - <output>&lt;cite></output></item>
<item><input>Definition</input> - <output>&lt;dfn></output></item>
<item><input>ComputerCode</input> - <output>&lt;code></output></item>
<item><input>SampleInput</input> - <output>&lt;kbd></output></item>
<item><input>SampleOutput</input> - <output>&lt;samp></output></item>
<item><input>Variable</input> - <output>&lt;var></output></item>
<item><input>BiggerText</input> - <output>&lt;big></output></item>
<item><input>SmallerText</input> - <output>&lt;small></output></item>
<item><input>Bold</input> - <output>&lt;b></output></item>
<item><input>Italic</input> - <output>&lt;i></output></item>
<item><input>Subscript</input> - <output>&lt;sub></output></item>
<item><input>Superscript</input> - <output>&lt;sup></output></item>
<item><input>Hyperlink(\"http://www.example.com/\")</input> - <output>&lt;a href=\"http://www.example.com/\"></output></item>
<item><input>InlineQuote(\"http://www.example.com/\")</input> - <output>&lt;q cite=\"http://www.example.com/\"></output></item>
<item><input>Span(\"note\")</input> - <output>&lt;span class=\"note\"></output></item>
<item><input>FormLabel</input> - <output>&lt;label></output></item>
</list>
<related><functionref>addInlineElementAt</functionref></related>
<related><functionref>appendInlineElement</functionref></related>"
public data InlineElement = Emphasis | StrongEmphasis 
  | Abbreviation(String title) | Citation | Definition | ComputerCode 
  | SampleInput | SampleOutput | Variable // generic phrase elements
  | BiggerText | SmallerText | Bold | Italic | Subscript | Superscript // semi formatting elements
// special inline elements
  | Hyperlink(String uri)
  | InlineQuote(String citeuri)
  | Span(String class)
  | FormLabel;

"<summary>List type</summary>
<prose>Whether a list is <code>Unordered</code> (generally represented as a bulleted list) or <code>Ordered</code> (generally represented as a numbered list).</prose>
<related><functionref>addList</functionref></related>"
public data ListType = Ordered | Unordered;

"<summary>Initial table data</summary>
<prose>Initial table data. This allows quick initialisation of unformatted HTML tables. Required formatting can then be added using the usual methods by retrieving references to the individual table cells.</prose>
<prose>The table data consists of a header section (which may be empty), a footer section (which may be empty), and a list of body sections (of which at least one must exist). All sections have the same format - a list of rows each of which is a list of cells.</prose>
<prose>It is assumed that all rows have the same number of columns as the first row in the table, and cells overrunning this row length will be ignored.</prose>
<example>header = [[\"Version\",\"Date\"]];
footer = [];
body = [[[\"0.2.1\",\"2006-11-15\"],[\"0.2.2\",\"2006-11-26\"],[\"0.2.3\",\"2006-12-04\"]]];
itd = InitialTableData(header,footer,body);</example>
<prose>This will generate the following table</prose>
<example>&lt;table>
 &lt;thead>
  &lt;tr>&lt;th>Version&lt;/th>&lt;th>Date&lt;/th>&lt;/tr>
 &lt;/thead>
 &lt;tbody>
  &lt;tr>&lt;td>0.2.1&lt;/td>&lt;td>2006-11-15&lt;/td>&lt;/tr>
  &lt;tr>&lt;td>0.2.2&lt;/td>&lt;td>2006-11-26&lt;/td>&lt;/tr>
  &lt;tr>&lt;td>0.2.3&lt;/td>&lt;td>2006-12-04&lt;/td>&lt;/tr>
 &lt;/tbody>
&lt;/table></example>
<related><functionref>addTable</functionref></related>
<related><functionref>getTableCell</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public data InitialTableData([[String]] header, [[String]] footer, [[[String]]] sections);

"<summary>An inline image.</summary>
<prose>An inline image. The <code>alt</code> String should contain the same text that you would use if you were unable to add an image at this point.</prose>
<related><dataref>ImageDimensions</dataref></related>
<related><functionref>addImage</functionref></related>"
public data ImageData(String src, String alt, ImageDimensions dim);

"<summary>Image dimensions</summary>
<prose>Methods for giving image dimensions. <code>Specified(<variable>width</variable>,<variable>height</variable>)</code>
allows precise specification of image dimensions and for efficiency
this should always be used if those values are known. <code>Unspecified</code>
leaves the calculation to the client user-agent.</prose>
<prose>The width and height are given in pixels.</prose>
<related><dataref>ImageData</dataref></related>
<related><functionref>addImage</functionref></related>"
public data ImageDimensions = /*AutoFromSrc | AutoFromPath(String filepath) |*/ Specified(Int width, Int height) | Unspecified;

"<summary>The method used by a form</summary>
<prose>The method used by a form to submit data. <code>FormPost</code> uses the <code>application/x-www-form-urlencoded</code> content type, whereas <code>FormPostUpload</code> uses <code>multipart/form-data</code>.</prose>
<related><functionref>WebCommon::allowFileUploads</functionref></related>
<related><functionref>addRemoteForm</functionref></related>"
public data FormType = FormGet | FormPost | FormPostUpload;

"<summary>Types of text input</summary>
<prose>Types of text and selection input.</prose>
<related><functionref>addOption</functionref> is often a better way to add a checkbox</related>
<related><functionref>addOptionList</functionref> is often a better way to add a set of radio buttons</related>
<related><functionref>addLabelledInput</functionref></related>
<related><functionref>addSelectElement</functionref> handles &lt;select>s</related>
<related><functionref>addTextInput</functionref></related>"
public data TextInputType = InputText | InputPassword | InputHidden | InputRadio | InputCheck | InputFile;

"<summary>Types of control input</summary>
<prose>Types of form control inputs. Generally <code>InputSubmit</code> is the only useful one. <code>InputReset</code> has some specialised uses, but <link url=\"http://www.cs.tut.fi/~jkorpela/forms/imagereset.html#need\">in general it is harmful</link> for most forms.</prose>
<related><functionref>addLocalControlInput</functionref></related>
<related><functionref>addRemoteControlInput</functionref></related>"
public data ControlInputType = InputSubmit | InputReset;

"<summary>An option for selection</summary>
<prose>An option for selection. These can be used to make the &lt;option> elements for a &lt;select> or to describe a checkbox or radio button. The <code>value</code> field may be left blank, in which case the contents of the <code>label</code> field will be used in its place.</prose>
<prose>The <code>chosen</code> field determines whether this option is initially selected when the page loads. The behaviour when multiple options in a non-multiple select or multiple radio buttons with the same name are marked as initially chosen is undefined, and so you should avoid this in your code.</prose>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLazySelect</functionref></related>
<related><functionref>addOption</functionref></related>
<related><functionref>addOptionList</functionref></related>
<related><functionref>addSelectElement</functionref></related>"
public data SelectOption(String label, String value, Bool chosen);

"<summary>Elements to allow in String->HTML conversion</summary>
<prose>When converting from a String to HTML, rather than simply adding a String to an existing element where it will be escaped, the elements allowed in the conversion should depend on how trustworthy the String is. Generally, any unauthenticated user-supplied data should be treated extremely cautiously, and even authenticated user-supplied data should be treated with some caution in case the authentication is broken.</prose>
<prose>Use of String to HTML conversion allows potential for <link url=\"http://www.cert.org/archive/pdf/cross_site_scripting.pdf\">cross-site scripting attacks</link> against your application, especially if the allowed element list is generous.</prose>
<list>
<item><code>UltraSafe</code> - removes all tags and attributes. This differs from adding the string directly as text, which escapes them. This conversion method is immune to cross-site scripting.</item>
<item><code>InlineOnly</code> - allows only inline elements.</item>
<item><code>AllElements</code> - allows inline and block elements.</item>
<item><code>Unchecked</code> - allows all tags and attributes. Use this only on completely trusted data, as it allows trivial cross-site scripting attacks if an attacker can control the String being converted.</item>
<item><code>CustomWhitelist</code> - create your own whitelist of elements. The whitelist is a <moduleref>Dict</moduleref> with the allowed elements as the key and the list of allowed attributes for that element as the value. The string \"*\" will match any element as the key, or any attribute as an item in the value list, which is generally not a good idea for anything other than completely trusted data.</item>
</list>
<prose>For the <code>InlineOnly</code> and <code>AllElements</code> options, you also need to select a <dataref>ConversionSafety</dataref>.</prose>
<related><dataref>ConversionSafety</dataref></related>
<related><functionref>readFromString</functionref></related>"
public data WhiteList = UltraSafe | InlineOnly(ConversionSafety sa) | AllElements(ConversionSafety sb) | Unchecked | CustomWhitelist(Dict<String,[String]> whitelist);
// no tags | safe tags | no forms | no JS | all
"<summary>The conversion safety level for String->HTML conversion</summary>
<prose>If you are using the <code>InlineOnly</code> or <code>AllElements</code> option for <dataref>WhiteList</dataref> you can choose various sets of elements and attributes to allow.</prose>
<list>
<item><code>Safe</code> - a very restricted set of elements and attributes is allowed. Hyperlinks, images, forms, scripting, inline styles and so on are not allowed.</item>
<item><code>Unsafe</code> - As <code>Safe</code>, but hyperlinks, images and client-side scripting are allowed. Some cross-site scripting is possible as a result.</item>
<item><code>VeryUnsafe</code> - As <code>Unsafe</code>, but form controls are also allowed. This allows some potentially very nasty cross-site scripting attacks to be carried out with ease if an attacker is able to influence the String being converted, so use this with extreme caution.</item>
</list>
<prose>None of these allow the direct addition of &lt;script> elements or the <code>on<variable>X</variable></code> event handlers.</prose>
<related><dataref>WhiteList</dataref></related>
<related><functionref>readFromString</functionref></related>"
public data ConversionSafety = Safe | Unsafe | VeryUnsafe;

"<argument>The String contains details of the error</argument>
<summary>A required element is missing</summary>
<prose>This Exception is thrown by various parts of the module if a required element is missing, or an operation that can only be applied to certain elements is applied to a different element.</prose>"
Exception RequiresElement(String err);

"<argument>The String contains details of the error</argument>
<summary>A required attribute is missing</summary>
<prose>This Exception is thrown by various parts of the module, usually when an attribute has the wrong format. For example, the <code>name</code> attribute of an <code>input</code> element may not be empty.</prose>"
Exception RequiresAttribute(String err);
"<argument>The String contains details of the error</argument>
<summary>The specified range is invalid</summary>
<prose>This Exception is thrown by <functionref>addInlineElementAt</functionref> if the start or end positions are outside the bounds of the parent element.</prose>"
Exception InvalidRange(String err);
"<argument>The String contains details of the error</argument>
<summary>The specified range causes invalid nesting</summary>
<prose>This Exception is thrown by <functionref>addInlineElementAt</functionref> if the start or end positions would cause invalid nesting.</prose>
<example>p = addParagraph(parent,\"This paragraph\");
em = addInlineElementAt(p,Emphasis,3,7);
code = addInlineElementAt(p,ComputerCode,6,9);
// this attempts to make &lt;p>Thi&lt;em>s p&lt;code>a&lt;/em>ra&lt;/code>graph&lt;/p>
// which is invalid, and so an Exception is thrown.</example>
<prose>This Exception is also thrown by several other functions if you try to add an element inside another element which cannot contain it.</prose>
<example>p = addParagraph(parent,\"\");
list = addList(p,Unordered,0);
// lists cannot be contained inside paragraphs, so an exception is thrown</example>"
Exception InvalidNesting(String err);
"<summary>TagSoup is not an output format</summary>
<prose>This Exception is thrown if the TagSoup Doctype is used for output - it may only be used with <functionref>readFromString</functionref>.</prose>"
Exception TagSoupNotForOutput();

/** Part 2: General document construction **/

/* ElementTree is entirely generic. It guarantees well-formedness and
   nothing else. So in the interests of producing valid as well as
   well-formed code, modules that use HTMLDocument to generate
   documents shouldn't let people directly touch ElementTree. */

"<argument name='doctype'>The document type declaration to use. HTML4Strict is recommended for now due to browser support issues.</argument>
<argument name='title'>The title for the entire document.</argument>
<summary>A new empty HTML document</summary>
<prose>Generates a skeleton HTML document with the specified Doctype and document title. The document title must not be empty. Operations on a HTML document are generally either performed on the document itself, or on its <code>body</code> field.</prose>
<example>doc = HTMLDocument::new(HTML4Strict,\"Hello World!\");</example>
<related><dataref>Doctype</dataref></related>
<related><dataref>HTMLDocument</dataref></related>"
public HTMLDocument new(Doctype doctype, String title) {
  if (title == "") {
    throw(RequiresElement("The document title may not be blank."));
  }
  case doctype of {
    HTML4Strict -> ctype = ("Content-type","text/html; charset=UTF-8");
    | XHTML1Strict -> ctype = ("Content-type","application/xhtml+xml; charset=UTF-8");
    | TagSoup -> throw(TagSoupNotForOutput);
  }
  return HTMLDocument(
		      MetaData(title,
			       Dict::new(17,strHash),
			       createArray(5), // styles
			       createArray(20), // links
			       createArray(5)), // scripts
		      mkElement("body"),
		      doctype,
		      // change content type if doctype is XHTML
		      [ctype]);
}

"<argument name='doc'>The HTML document</argument>
<argument name='uform'>The mode for handling multibyte UTF-8 characters. <code>LiteralUTF8</code> will give a smaller String, and one that is more readable in other Unicode-aware applications. <code>NumericReference</code> will give a larger String, especially if multibyte characters are very common in the text, but will work better if the String will later be edited in non-Unicode applications. This argument is optional, and defaults to the recommended <code>LiteralUTF8</code></argument>
<summary>Convert an entire HTML document to a String</summary>
<prose>Converts the entire document to String form for output. If the String will then be immediately printed, the <functionref>write</functionref> function is considerably more efficient.</prose>
<example>str = string(doc);
str = string(doc,NumericReference);</example>
<related><dataref>ElementTreeData::UnicodeFormat</dataref></related>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref>substring</functionref></related>
<related><functionref>write</functionref></related>"
public String string(HTMLDocument doc, UnicodeFormat uform = LiteralUTF8) {
  output = headString(doc,uform);
  case doc.doctype of {
    HTML4Strict -> output += string(doc.body,ImpliedSingleton,1,uform,tagFormats,getAlwaysEmpty);
    | XHTML1Strict -> output += string(doc.body,Singleton,1,uform,tagFormats,getAlwaysEmpty);
  }
  output += "\n</html>\n";
  return output;
}

"<argument name='tree'>An element from the document body (it may be the document body itself)</argument>
<argument name='uform'>The method for displaying multibyte Unicode characters. This argument is optional, with a default of <code>LiteralUTF8</code></argument>
<argument name='doctype'>The document type declaration to assume for conversion. This need not be the same as the doctype used for the HTMLDocument, if this element tree is part of a HTMLDocument. This argument is optional, with a default of <code>HTML4Strict</code>.</argument>
<summary>Converts a part of the document to string form</summary>
<prose>Convert a part of the document (or a fragment of HTML unconnected to the document) into String form. This can be used as an easy way of giving HTML source examples:</prose>
<example>p = addParagraph(parent,\"An image: \");
img = addImage(p,ImageData(\"example.png\",\"an example\",Unspecified));
p2 = addParagraph(parent,\"HTML code for the image: \"+substring(img));</example>
<related><dataref>Doctype</dataref></related>
<related><dataref>ElementTreeData::UnicodeFormat</dataref></related>
<related><functionref>string</functionref></related>"
public String substring(ElementTree tree, UnicodeFormat uform = LiteralUTF8, Doctype doctype=HTML4Strict) {
  case doctype of {
    HTML4Strict -> return string(tree,ImpliedSingleton,0,uform,tagFormats,getAlwaysEmpty);
    | XHTML1Strict -> return string(tree,Singleton,0,uform,tagFormats,getAlwaysEmpty);
  }
}

String headString(HTMLDocument doc, UnicodeFormat uform) {
  case doc.doctype of {
    HTML4Strict -> output = "<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 4.01//EN\"\n\t\"http://www.w3.org/TR/html4/strict.dtd\">\n";
    output += "<html>\n";
    singletoncloser = createString(0);
    | XHTML1Strict -> output = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
    output += "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\"\n\t\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n";
    output += "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n";
    singletoncloser = " /";
  }
  output += " <head>\n";
  if (doc.head.doctitle == "") {
    throw(RequiresElement("The document title may not be blank."));
  }
  output += "  <title>" + simpleEncode(doc.head.doctitle,false,uform) + "</title>\n";
  // add code to handle meta/styles/links here 
  for metadata in entries(doc.head.metakeys) {
    output += "  <meta name=\"" + simpleEncode(metadata.fst,false,uform) + "\" value=\"" + simpleEncode(metadata.snd,false,uform) + "\"" + singletoncloser + ">\n";
  }
  for link in doc.head.links {
    output += "  <link ";
    case link.relate of {
      Rel(rel) -> output += "rel=\"" + simpleEncode(rel,false,uform) + "\" ";
      | Rev(rev) -> output += "rev=\"" + simpleEncode(rev,false,uform) + "\" ";
    }
    output += "href=\""+simpleEncode(link.uri,false,uform)+"\"";
    if (link.title != "") {
      output += " title=\""+simpleEncode(link.title,false,uform)+"\"";
    }
    if (link.ctype != "") {
      output += " type=\""+simpleEncode(link.ctype,false,uform)+"\"";
    }
    output += singletoncloser + ">\n";
  }
  for stylesheet in doc.head.styles {
    output += "  <link rel=\"stylesheet\" href=\"" + simpleEncode(stylesheet.uri,false,uform) + "\" type=\"text/css\" media=\"";
    output += mediaToString(stylesheet.media);
    output += "\"" + singletoncloser + ">\n";
  }
  for script in doc.head.scripts {
    output += "  <script type=\"text/javascript\" src=\"" + simpleEncode(script.uri,false,uform) + "\"></script>\n";
  }
  output += " </head>";
  return output;
}

"<argument name='doc'>The HTML document</argument>
<argument name='uform'>The mode for handling multibyte UTF-8 characters. <code>LiteralUTF8</code> will give a smaller String, readable in Unicode-aware applications. <code>NumericReference</code> will give a larger String, especially if multibyte characters are very common in the text, but will work better if the String will later be edited in non-Unicode applications. This argument is optional, and defaults to the recommended <code>LiteralUTF8</code>, but you may wish to use <code>NumericReference</code> if standard output is connected to a non-Unicode terminal, for example.</argument>
<summary>Print a HTML document on standard output</summary>
<prose>Print the HTML document on standard output. This is considerably more efficient than <code>putStr(string(doc));</code></prose>
<related><dataref>ElementTreeData::UnicodeFormat</dataref></related>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref>string</functionref></related>
<related><functionref>writeTo</functionref></related>"
public Void write(HTMLDocument doc, UnicodeFormat uform = LiteralUTF8) {
  putStr(headString(doc,uform));
  case doc.doctype of {
    HTML4Strict -> lazyPrint(doc.body,ImpliedSingleton,1,uform,tagFormats,getAlwaysEmpty);
    | XHTML1Strict -> lazyPrint(doc.body,Singleton,1,uform,tagFormats,getAlwaysEmpty);
  }
  putStr("\n</html>\n");
}


"<argument name='doc'>The HTML document</argument>
<argument name='output'>A function that outputs the strings given to it (using <code>Prelude::putStr@()</code> is equivalent to <functionref>write</functionref>)</argument>
<argument name='uform'>The mode for handling multibyte UTF-8 characters. <code>LiteralUTF8</code> will give a smaller String, readable in Unicode-aware applications. <code>NumericReference</code> will give a larger String, especially if multibyte characters are very common in the text, but will work better if the String will later be edited in non-Unicode applications. This argument is optional, and defaults to the recommended <code>LiteralUTF8</code>, but you may wish to use <code>NumericReference</code> if the output is not Unicode-aware, for example.</argument>
<summary>Print a HTML document with a specified output function</summary>
<prose>Print a HTML document with a specified output function. This is considerably more efficient than <code><variable>output</variable>(string(doc));</code></prose>
<related><dataref>ElementTreeData::UnicodeFormat</dataref></related>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref>string</functionref></related>
<related><functionref>write</functionref></related>"
public Void writeTo(HTMLDocument doc, Void(String) output, UnicodeFormat uform = LiteralUTF8) {
  output(headString(doc,uform));
  case doc.doctype of {
    HTML4Strict -> lazyOutput(doc.body,ImpliedSingleton,1,uform,tagFormats,getAlwaysEmpty,output);
    | XHTML1Strict -> lazyOutput(doc.body,Singleton,1,uform,tagFormats,getAlwaysEmpty,output);
  }
  output("\n</html>\n");
}

private String mediaToString([MediaType] media) {
  nub(media);
  if (elem(MTall,media) || size(media) == 0) {
    return "all"; // don't need to look in this case
  } else {
    mstr = createArray(size(media));
    for mt in media {
      case mt of {
	MTscreen -> mtstr = "screen";
	| MTtty -> mtstr = "tty";
	| MTtv -> mtstr = "tv";
	| MTprojection -> mtstr = "projection";
	| MThandheld -> mtstr = "handheld";
	| MTprint -> mtstr = "print";
	| MTbraille -> mtstr = "braille";
	| MTaural -> mtstr = "aural";
      }
      push(mstr,mtstr);
    }
    return Strings::join(mstr,',');
  }
}


/** Part 3: Header editing **/

"<argument name='doc'>The HTML document</argument>
<argument name='name'>The name of the header</argument>
<argument name='value'>The contents of the header</argument>
<summary>Adds a HTTP header to the document</summary>
<prose>Adds a HTTP header to the document. If the document is then printed using <functionref>Webapp::displayPage</functionref>, these headers will be sent to the client.</prose>
<prose>The effects of multiple HTTP headers with the same name vary. Some headers may be rewritten by the web server if they appear multiple times.</prose>
<prose>You can change the HTTP status code using the special <code>Status</code> header. The default is 200, of course - due to a bug in older versions of Apache, explicitly setting a 200 status is not recommended. Headers with names beginning \"X-\" are non-standard and could mean anything.</prose>
<example>addHTTPHeader(doc,\"Last-Modified\",rfc2822Time(lmstamp));
addHTTPHeader(doc,\"Status\",\"404 File not found\");
addHTTPHeader(doc,\"X-Generator\",\"Kaya\");</example>
<prose>The characters allowed in HTTP headers are relatively restricted, especially in the <code>name</code> field, the most obvious restriction being that they may not contain new lines. <functionref>Webapp::displayPage</functionref> will throw an Exception if illegal characters are found - be sure to check this if you write your own header output function.</prose>
<prose><link url=\"http://www.w3.org/Protocols/rfc2616/\">RFC 2616</link> describes the HTTP protocol including HTTP headers in detail</prose>
<prose>You may set multiple headers with the same name, but this may not be sensible for some headers (for example, multiple Status headers make no sense, whereas multiple Set-Cookie headers are commonly used).</prose>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref>addHTTPHeader</functionref></related>
<related><functionref>Webapp::displayPage</functionref></related>"
public Void addHTTPHeader(HTMLDocument doc, String name, String value) {
  // TODO: checks for headers such as Content-type that can only appear once
  Pair<String,String> val = (name,value);
  push(doc.httpheaders,val);
}

"<argument name='doc'>The HTML document</argument>
<argument name='header'>A pair of Strings describing the header</argument>
<summary>Adds a HTTP header to the document</summary>
<prose>Adds a HTTP header to the document. If the document is then printed using <functionref>Webapp::displayPage</functionref>, these headers will be sent to the client.</prose>
<prose>The effects of multiple HTTP headers with the same name vary. Some headers may be rewritten by the web server if they appear multiple times.</prose>
<prose>You can change the HTTP status code using the special <code>Status</code> header. The default is 200, of course - due to a bug in older versions of Apache, explicitly setting a 200 status is not recommended. Headers with names beginning \"X-\" are non-standard and could mean anything.</prose>
<example>addHTTPHeader(doc,setCookie(\"session\",getSessionId()));</example>
<prose>The characters allowed in HTTP headers are relatively restricted, especially in the <code>name</code> field, the most obvious restriction being that they may not contain new lines. <functionref>Webapp::displayPage</functionref> will throw an Exception if illegal characters are found - be sure to check this if you write your own header output function.</prose>
<prose><link url=\"http://www.w3.org/Protocols/rfc2616/\">RFC 2616</link> describes the HTTP protocol including HTTP headers in detail</prose>
<prose>You may set multiple headers with the same name, but this may not be sensible for some headers (for example, multiple Status headers make no sense, whereas multiple Set-Cookie headers are commonly used).</prose>
<related><dataref>HTMLDocument</dataref></related>
<related><functionref index='1'>addHTTPHeader</functionref></related>
<related><functionref>Webapp::displayPage</functionref></related>
<related><functionref>WebCommon::setCookie</functionref></related>"
public Void addHTTPHeader(HTMLDocument doc, (String,String) header) {
  // TODO: checks for headers such as Content-type that can only appear once
  push(doc.httpheaders,copy(header));
}

"<argument name='doc'>The HTML document</argument>
<argument name='key'>The key name for the meta-data value</argument>
<argument name='value'>The meta-data value</argument>
<summary>Adds a meta-data key/value pair to the document</summary>
<prose>Adds a meta-data key/value pair to the document. This can be used to add additional information to the document useful to automated tools. The use of this data in the ranking algorithms of internet search engines has been widely debated but appears to be marginal at best. It may, however, be useful for local search engines or cataloguing software.</prose>
<example>addDocumentMetaData(doc,\"Author\",\"John Smith\");
addDocumentMetaData(doc,\"Description\",\"An example document\");</example>
<prose>\"HTTP-equiv\" meta-data is not supported by this module - just add a real HTTP header using <functionref>addHTTPHeader</functionref> instead.</prose>
<related><dataref>MetaData</dataref></related>
<related><functionref>addHTTPHeader</functionref></related>"
public Void addDocumentMetaData(HTMLDocument doc, String key, String value) {
  add(doc.head.metakeys,key,value);
}

"<argument name='doc'>The HTML document</argument>
<argument name='media'>A list of media types that this stylesheet applies to</argument>
<argument name='uri'>The location of the stylesheet.</argument>
<summary>Adds a linked external stylesheet to the document</summary>
<prose>Adds a linked external stylesheet to the document. See the <dataref>MediaType</dataref> documentation for an explanation of the various media types. Currently only CSS stylesheets are supported by this function.</prose>
<example>addDocumentStylesheet(doc,[MTall],
                      \"/styles/base.css\");
addDocumentStylesheet(doc,[MTprint],
                      \"/styles/print.css\");
addDocumentStylesheet(doc,[MTscreen,MTprojection,MThandheld],
                      \"/styles/visual.css\");</example>
<prose>If the media array is empty, <code>[MTall]</code> will be used.</prose>
<related><dataref>MediaType</dataref></related>"
public Void addDocumentStylesheet(HTMLDocument doc, [MediaType] media, String uri) {
  push(doc.head.styles,StyleSheet(media,uri));
}

"<argument name='doc'>The HTML document</argument>
<argument name='rel'>The relationship this link describes</argument>
<argument name='uri'>The resource linked to.</argument>
<argument name='type'>The content-type of the linked document (optional)</argument>
<summary>Adds a document-level link</summary>
<prose>Adds a document-level link to another resource.</prose>
<example>addDocumentLink(doc,Rel(\"next\"),
                webappName()+\"slide=\"+(slideno+1));
addDocumentLink(doc,Rev(\"made\"),\"author@example.invalid\");</example>
<related>The <dataref>Relationship</dataref> documentation explains the semantics of forward and reverse document links.</related>"
public Void addDocumentLink(HTMLDocument doc, Relationship relate, String uri,  String title=createString(0), String ctype=createString(0)) {
  push(doc.head.links,HeadLink(relate,uri,title,ctype));
}

"<argument name='doc'>The HTML document</argument>
<argument name='scripturi'>The location of the Javascript file.</argument>
<summary>Adds an external script to the document</summary>
<prose>Adds an external script to the document. This script will define functions which may then be called using the <code>onX</code> event handler attributes for various elements in the document.</prose>"
public Void addDocumentScripting(HTMLDocument doc, String scripturi) {
  push(doc.head.scripts,ClientScript(scripturi));
}

"<argument name='doc'>The HTML document</argument>
<argument name='newtitle'>The document title, which may not be blank (an Exception is thrown if it is)</argument>
<summary>Changes the document title</summary>
<prose>Changes the document title, which is commonly used by browsers when bookmarking the page and in their user interface, and by search engines for indexing purposes. Ideally, all pages should have a unique title.</prose>
<related><functionref>addDocumentMetaData</functionref></related>"
public Void setDocumentTitle(HTMLDocument doc, String newtitle) {
  if (newtitle == "") {
    throw(RequiresElement("The document title may not be blank."));
  }
  doc.head.doctitle = newtitle;
}


/** Part 4: Body editing - common attributes **/


"<argument name='element'>The HTML element</argument>
<argument name='classname'>The CSS class to set</argument>
<summary>Set the CSS class</summary>
<prose>Set the CSS class(es) of a HTML element. Multiple classes can be specified by separating them with a space. If a class is given which does not match the allowed characters for class names, a <exceptref>RequiresAttribute</exceptref> Exception will be thrown.</prose>
<example>setClass(paragraph,\"important\");
setClass(link,\"external recommended\");</example>
<related><functionref>setAttribute</functionref></related>
<related><functionref>setID</functionref></related>
<related><functionref>setTitle</functionref></related>"
public Void setClass(ElementTree element, String classname) {
  // doesn't allow for unicode or escaped character classnames
  // but they're very rarely used. We can add support later.
  classnames = words(classname);
  for cn in classnames {
    if (!quickMatch(classregex,classname)) {
      throw(RequiresAttribute("Class format incorrect"));
    }
  }
  module::setAttribute(element,"class",classname);
}

"<argument name='element'>The HTML element</argument>
<argument name='classname'>The ID to set</argument>
<summary>Sets the ID of a HTML element</summary>
<prose>Sets the ID of a HTML element. The ID is a unique identifier within the document for the element. Setting IDs is useful for styling, scripting, and creating anchors within a document.</prose>
<example>h = addHeading(body,2,\"Compiler options\");
setID(h,\"opts\");
// appending #opts to the URL will now jump straight to this heading</example>
<related><functionref>setAttribute</functionref></related>
<related><functionref>setClass</functionref></related>
<related><functionref>setTitle</functionref></related>"
public Void setID(ElementTree element, String idname) {
  if (!quickMatch(idregex,idname)) {
    throw(RequiresAttribute("ID format incorrect"));
  }
  // Ideally we'd check for duplicate ids at this stage, but since we
  // don't have a way to find the root element, we can't.
  module::setAttribute(element,"id",idname);
}

private String nextUniqueID() {
  if (uniqueid < 1) {
    uniqueid = 1;
  } else {
    uniqueid++;
  }
  return "Kay"+String(uniqueid);
}

"<argument name='element'>The HTML element</argument>
<argument name='title'>The title to set</argument>
<summary>Sets the title of a HTML element</summary>
<prose>Sets the title of a HTML element. The title can be used to provide optional additional information about an element. Not all browsers will display title information, and the method of display will vary - common graphical browsers tend to display it as a tooltip when someone hovers over the element.</prose>
<prose>A common use is providing optional additional information about a link.</prose>
<example>url = Hyperlink(\"http://kayalang.org/library/latest/Prelude/putStr\");
link = appendInlineElement(paragraph,url,\"putStr\");
setTitle(link,\"API documentation for Prelude::putStr\");</example>
<related><functionref>setAttribute</functionref></related>
<related><functionref>setClass</functionref></related>
<related><functionref>setID</functionref></related>"
public Void setTitle(ElementTree element, String title) {
  module::setAttribute(element,"title",title);
}

"<argument name='element'>The HTML element</argument>
<argument name='name'>The attribute to set</argument>
<argument name='value'>The value to use</argument>
<summary>Sets an attribute on a HTML element</summary>
<prose>Sets an attribute on a HTML element. Required attributes will be set when you add the element by the addition function, but you may wish to add optional attributes here, or change the value of an existing attribute based on later calculations.</prose>
<example>setAttribute(quotation,\"lang\",\"fr\");</example>
<prose>The <functionref>setClass</functionref> and <functionref>setID</functionref> functions should always be used in preference to this function for those purposes, as they do some input validation (and are shorter to type!)</prose>
<related><functionref>getAttribute</functionref></related>
<related><functionref>setClass</functionref></related>
<related><functionref>setID</functionref></related>
<related><functionref>setTitle</functionref></related>"
public Void setAttribute(ElementTree element, String name, String value) {
  ElementTree::setAttribute(element,name,value);
}

"<argument name='element'>The HTML element</argument>
<argument name='name'>The attribute name</argument>
<summary>Gets an attribute value from a HTML element</summary>
<prose>Gets an attribute value from a HTML element, or <code>nothing</code> if the attribute is not set.</prose>
<example>case getAttribute(el,\"class\") of {
    nothing -> setClass(el,\"recentchange\");
    | just(c) -> setClass(el,c+\" recentchange\");
}</example>
<related><functionref>setAttribute</functionref></related>"
public Maybe<String> getAttribute(ElementTree element, String name) {
  return ElementTree::getAttribute(element,name);
}

// TODO: language, directionality, possibly scripting events


/** Part 5: Body editing - adding blocks **/


// TODO: need a decent table editing interface here, as well
// as other block-level elements..

// this should really check for proper header nesting somewhere
"<argument name='parent'>The parent element</argument>
<argument name='headinglevel'>The level of heading (1-6)</argument>
<argument name='initialtext'>Optionally, any initial text the element should contain</argument>
<summary>Add a heading</summary>
<prose>Add a heading to the end of the specified element. Heading levels go from 1 to 6, and form a document outline - there should generally be one level 1 heading in the document, with level 2 headings below that, level 3 headings below the level 2 headings, and so on. In practice, headings less important than level 4 are very rarely used.</prose>"
public ElementTree addHeading(ElementTree parent, Int headinglevel, String initialtext=createString(0)) {
  if (headinglevel < 1 || headinglevel > 6) {
    throw(RequiresElement(headinglevel+" is not a valid heading depth"));
  }
  return addGenericBlock(parent,"h"+String(headinglevel),initialtext);
}

"<argument name='parent'>The parent element</argument>
<argument name='initialtext'>Optionally, initial text content for the element.</argument>
<summary>Add a paragraph</summary>
<prose>Add a paragraph to the end of the specified element. Unlike most block-level elements, paragraphs may only contain inline elements.</prose>"
public ElementTree addParagraph(ElementTree parent, String initialtext=createString(0)) {
  return addGenericBlock(parent,"p",initialtext);
}

"<argument name='parent'>The parent element</argument>
<argument name='initialtext'>Optionally, initial text content for the element.</argument>
<summary>Add an address</summary>
<prose>Add an address to the end of the specified element. The <code>address</code> element is intended to contain contact details for the document author (or document section author)</prose>"
public ElementTree addAddress(ElementTree parent, String initialtext=createString(0)) {
  return addGenericBlock(parent,"address",initialtext);
}


// No initial text option because blockquotes can't directly contain text.
"<argument name='parent'>The parent element</argument>
<summary>Add a quotation block</summary>
<prose>Add a quotation block to the end of the specified element. Quotation blocks may not contain text directly - you must add further block-level elements to the quotation block.</prose>"
public ElementTree addBlockQuote(ElementTree parent) {
  if (!canContainBlock(parent.name)) {
    throw(InvalidNesting("Can't add block quotation inside "+parent.name));
  }
  // need to check that parent can contain paragraphs (e.g. not ol)
  bquote = mkElement("blockquote");
  pushElement(parent,bquote);
  return bquote;
}

"<argument name='parent'>The parent element</argument>
<argument name='initialtext'>Optionally, initial text content for the element.</argument>
<argument name='cssclass'>Optionally, the CSS class to apply to the division</argument>
<summary>Add a generic block</summary>
<prose>Add a generic block to the end of the specified element. Unlike paragraphs, lists, etc. this block has no defined semantics, and so is useful for grouping and styling purposes.</prose>"
public ElementTree addDivision(ElementTree parent, String initialtext=createString(0), String cssclass=createString(0)) {
  div = addGenericBlock(parent,"div",initialtext);
  if (cssclass != "") {
    setClass(div,cssclass);
  }
  return div;
}

"<argument name='parent'>The parent element</argument>
<argument name='initialtext'>Optionally, initial text content for the element.</argument>
<summary>Add a preformatted block</summary>
<prose>Add a preformatted block to the end of the specified element. This is useful for displaying program code and other text where indentation may be significant or useful to preserve.</prose>
<example>pre = addPreformatted(parent,\"\");
code = appendInlineElement(pre,\"j = 0;
for i in [1..10] {
  j += i;
}\");</example>"
public ElementTree addPreformatted(ElementTree parent, String initialtext=createString(0)) {
  return addGenericBlock(parent,"pre",initialtext);
}

private Bool canContainBlock(String ename) = containsBlock(ename);

private Bool canContainException(String parent, String child) = nestingExceptions((parent,child));

// might make this public later, if people insist...
private ElementTree addGenericBlock(ElementTree parent, String element, String initialtext=createString(0)) {
  // tweak to allow ins and del to be added in an inline context too
  if (canContainBlock(parent.name) || ((element == "del" || element == "ins") && canContainInline(parent.name,false))) {
    // need to check that parent can contain paragraphs (e.g. not ol)
    newelement = mkElement(element);
    pushElement(parent,newelement);
    if (initialtext != "") {
      // because it's only called with initial text if this is possible
      doAddString(newelement,initialtext);
    }
    return newelement;
  } else {
    throw(InvalidNesting("Can't add "+element+" inside "+parent.name));
  }
}

"<argument name='parent'>The parent element</argument>
<argument name='ltype'>Is the list <code>Unordered</code> or <code>Ordered</code></argument>
<argument name='initialsize'>The initial number of list elements to have</argument>
<argument name='initialcontents'>Optionally, the initial text content for some or all of the list elements. If this list is shorter than <code>initialsize</code>, then only the first list items will be populated. If it is longer, then the extra list items will be ignored.</argument>
<summary>Add a list</summary>
<prose>Add a list to the parent element. Lists may only directly contain list items.</prose>
<example>steps = [\"Catch flamingo\",\"Place flamingo in pan\",\"Boil for 3 hours\"];
list = addList(parent,Ordered,3,steps);</example>
<related><dataref>ListType</dataref></related>
<related><functionref>addListItem</functionref></related>
<related><functionref>getListItem</functionref></related>
<related><functionref>pushListItem</functionref></related>"
public ElementTree addList(ElementTree parent, ListType ltype, Int initialsize, [String] initialcontents=createArray(1)) {
  if (!canContainBlock(parent.name)) {
    throw(InvalidNesting("Can't add lists inside "+parent.name));
  }
  case ltype of {
    Ordered -> etype = "ol";
    | Unordered -> etype = "ul";
  }
  newlist = mkElement(etype);
  for i in [1..initialsize] {
    listitem = mkElement("li");
    pushElement(newlist,listitem);
    if (initialcontents[i-1] != "") {
      doAddString(listitem,initialcontents[i-1]);
    }
  }
  pushElement(parent,newlist);
  return newlist;
}

"<argument name='list'>The list to search</argument>
<argument name='index'>The index to retrieve (starting at zero)</argument>
<summary>Retrieve a list item</summary>
<prose>Gets the list item within a list at the specified index. If the index given is not in the list, an <exceptref>Array::OutOfBounds</exceptref> Exception will be thrown. If the list given is not actually a list, a <exceptref>RequiresElement</exceptref> Exception is thrown.</prose>
<example>list = addList(parent,Unordered,10);
for i in [0..9] {
   item = getListItem(list,i);
   addString(item,\"This is item \"+(i+1));
}</example>
<related><functionref>addList</functionref></related>
<related><functionref>addListItem</functionref></related>
<related><functionref>pushListItem</functionref></related>"
public ElementTree getListItem(ElementTree list, Int index) {
  if (list.name == "ul" || list.name == "ol") {
    if (index < 0 || size(list.elements) <= index) {
      throw(OutOfBounds);
    }
    // ol and ul should only contain li
    // this will throw a mildly unhelpful exception if CData has crept in
    return list.elements[index].nested;
    
  } else {
    throw(RequiresElement("Parent element is not a list"));
  }
}


"<argument name='list'>The list to add to</argument>
<argument name='index'>The index to insert at (starting at zero)</argument>
<argument name='initialcontents'>Optionally, the initial contents of the list item</argument>
<summary>Add a list item</summary>
<prose>Add a list item to a list at the specified index. If the list given is not actually a list, an <exceptref>InvalidNesting</exceptref> Exception is thrown.</prose>
<example>if (extradetails) {
    addListItem(instructions,5,\"Check that all leads are attached.\");    
    addListItem(instructions,12,\"Check that the batteries are charged.\");    
}</example>
<related><functionref>addList</functionref></related>
<related><functionref>getListItem</functionref></related>
<related><functionref>pushListItem</functionref></related>"
public ElementTree addListItem(ElementTree list, Int index, String initialcontents=createString(0)) {
  if (list.name == "ul" || list.name == "ol") {
    listitem = mkElement("li");
    if (initialcontents != "") {
      doAddString(listitem,initialcontents);
    }
    addElementAt(list,listitem,index);
    return listitem;
  } else {
    throw(InvalidNesting("Can't add a list item here - improper nesting"));
  }
}

"<argument name='list'>The list to append to</argument>
<argument name='initialcontents'>Optionally, the initial contents of the list item</argument>
<summary>Append a list item</summary>
<prose>Append a list item to a list. If the list given is not actually a list, an <exceptref>InvalidNesting</exceptref> Exception is thrown. This is sometimes a more convenient way to build up lists than using an initial content array on <code>addList</code>, especially if the final size of the list is variable.</prose>
<example>list = addList(parent,Ordered,0);
while (text = getNextText(source)) {
    pushListItem(list,text);
}</example>
<related><functionref>addList</functionref></related>
<related><functionref>addListItem</functionref></related>
<related><functionref>getListItem</functionref></related>"
public ElementTree pushListItem(ElementTree list, String initialcontents=createString(0)) {
  if (list.name == "ul" || list.name == "ol") {
    listitem = mkElement("li");
    if (initialcontents != "") {
      doAddString(listitem,initialcontents);
    }
    pushElement(list,listitem);
    return listitem;
  } else {
    throw(InvalidNesting("Can't add a list item here - improper nesting"));
  }
}

/** Part 6: Body editing - adding inline content **/


private Bool canContainInline(String elemname, Bool implyCdata=true) {
  return (containsCData(elemname)
	  || !(isBlock(elemname)
	       || alwaysEmpty(elemname)
	       || pretendEmpty(elemname))
	  || (implyCdata && containsCDataOnly(elemname)));
}

// everything should call this rather than using pushData directly
"<argument name='block'>The element to add text to</argument>
<argument name='text'>The text to add</argument>
<summary>Append text to an element</summary>
<prose>Append text directly to an element.</prose>
<example>p = addParagraph(p,\"This is a \");
addString(p,\"paragraph.\");
// &lt;p>This is a paragraph.&lt;/p></example>
<related><functionref>appendInlineElement</functionref> is used to add text within a containing element.</related>"
public Void addString(ElementTree block, String text) {
  if (canContainInline(block.name,true)) {
    doAddString(block,text);
  } else {
    throw(InvalidNesting(block.name+" can't (directly?) contain text"));
  }
}

// except that where we know strings are allowed we can use this
private Void doAddString(ElementTree block, String text) {
  text = entityToLiteral(text);
  pushData(block,text);
}

private ElementTree makeInlineElement(InlineElement inline) {
  case inline of {
    // phrase elements
    Emphasis -> elem = mkElement("em");
    | StrongEmphasis -> elem = mkElement("strong");
    | Abbreviation(title) -> elem = mkElement("abbr");
    if (title != "") { setTitle(elem,title); }
    | Citation -> elem = mkElement("cite");
    | Definition  -> elem = mkElement("dfn");
    | ComputerCode  -> elem = mkElement("code");
    | SampleInput -> elem = mkElement("kbd");
    | SampleOutput -> elem = mkElement("samp");
    | Variable -> elem = mkElement("var");
    | BiggerText -> elem = mkElement("big");
    | SmallerText -> elem = mkElement("small");
    | Bold -> elem = mkElement("b");
    | Italic -> elem = mkElement("i");
    | Subscript -> elem = mkElement("sub");
    | Superscript -> elem = mkElement("sup");
    // special elements
    | Hyperlink(uri) -> elem = mkElement("a");
    module::setAttribute(elem,"href",uri);
    | InlineQuote(citeuri) -> elem = mkElement("q");
    if (citeuri != "") {
      module::setAttribute(elem,"cite",citeuri);
    }
    | Span(class) -> elem = mkElement("span");
    if (class != "") {
      setClass(elem,class);
    }
    | FormLabel -> elem = mkElement("label");
  }
  return elem;
}

"<argument name='block'>The element to add to</argument>
<argument name='inline'>The inline element to add</argument>
<argument name='initialtext'>The initial text content of the inline element</argument>
<summary>Append an inline element</summary>
<prose>Adds a new inline element containing some text to the end of a block.</prose>
<example>p = addParagraph(parent,\"This is \");
em = appendInlineElement(p,Emphasis,\"important\");
// &lt;p>This is &lt;em>important&lt;/em>&lt;/p></example>
<related><dataref>InlineElement</dataref></related>
<related><functionref>addInlineElementAt</functionref></related>
<related><functionref>addString</functionref></related>"
public ElementTree appendInlineElement(ElementTree block, InlineElement inline, String initialtext) {
  if (canContainInline(block.name,false)) {  
    inl = makeInlineElement(inline);
    doAddString(inl,initialtext);
    pushElement(block,inl);
    return inl;
  } else {
    throw(InvalidNesting("Can't add an inline element here"));
  }
}

"<argument name='block'>The element to add to</argument>
<argument name='inline'>The inline element to add</argument>
<argument name='startpos'>The start position for the inline element</argument>
<argument name='endpos'>The end position for the inline element</argument>
<summary>Wrap existing text in an inline element</summary>
<prose>Encloses existing text within an element within an inline element. The characters from <code>startpos</code> to <code>endpos</code> inclusive (with a starting index of zero) will be included in the inline element.</prose>
<example>p = addParagraph(parent,\"abcdefghijklmnopqrstuvwxyz\");
void(addInlineElementAt(p,Emphasis,5,10));
void(addInlineElementAt(p,StrongEmphasis,6,8));
// &lt;p>abcde&lt;em>f&lt;strong>ghi&lt;/strong>jk&lt;/em>lmnopqrstuvwxyz&lt;/p></example>
<prose>An <exceptref>InvalidNesting</exceptref> Exception will be thrown if the element being added to cannot contain inline elements, or an impossible nesting situation would be created. An <exceptref>InvalidRange</exceptref> Exception will be thrown if the <code>startpos</code> or <code>endpos</code> are outside the text contents of the block.</prose>
<example>p = addParagraph(parent,\"abcdefghijklmnopqrstuvwxyz\");
void(addInlineElementAt(p,Emphasis,5,10));
void(addInlineElementAt(p,StrongEmphasis,6,15));
// exception thrown </example>
<prose>The <code>firstOccurs</code> function is useful for highlighting a particular word within a string.</prose>
<example>haystack = getAllText(element);
npos = firstOccurs(needle,haystack);
if (npos < length(haystack)) {
    try {
        added = addInlineElementAt(element,Emphasis,npos,npos+length(needle)+1);
    } catch(InvalidNesting(details)) {
        // probably overlapping elements
    }
}</example>
<related><dataref>InlineElement</dataref></related>
<related><functionref>appendInlineElement</functionref></related>
<related><functionref index='1'>Strings::firstOccurs</functionref></related>"
public ElementTree addInlineElementAt(ElementTree block, InlineElement inline, Int startpos, Int endpos) {
  // iterates through existing structure, places start tag at startpos
  // (in char) and end tag at endpos (unless this would lead to
  // improper nesting)
  if (!canContainInline(block.name,false)) {  
    throw(InvalidNesting("Can't add an inline element here"));
  }
  if (startpos < 0) {
    throw(InvalidRange("Can't start before beginning of element"));
  }
  if (startpos > endpos) {
    throw(InvalidRange("Can't start after end"));
  }
  currentpos = 0;
  // if we've always been using pushData and unshiftData then there
  // should never be two adjacent data blocks

  // if we can nest properly, then the nesting level below block at
  // startpos and endpos must be equal, and between startpos and
  // endpos it must never fall below the level at startpos.
  totalsize = 0;
  for i in [0..size(block.elements)-1] {
    blocksizes[i] = totalsize + textSizeOfBlock(block.elements[i]);
    totalsize = blocksizes[i];
  }
  if (totalsize < endpos) {
    throw(InvalidRange("Can't end after end of element"));
  }
  startblock = 0;
  while (blocksizes[startblock] <= startpos) {
    startblock++;
  }
  endblock = startblock;
  while (blocksizes[endblock] <= endpos) {
    endblock++;
  }

  if (startblock == 0) {
    substart = startpos;
  } else {
    substart = startpos - blocksizes[startblock-1];
  }
  if (endblock == 0) {
    subend = endpos;
  } else {
    subend = endpos - blocksizes[endblock-1];
  }


  if (endblock != startblock) {
    case block.elements[endblock] of {
      SubElement(el) -> throw(InvalidNesting("Improper nesting"));
      | CData(c) -> case block.elements[startblock] of {
	SubElement(el) -> throw(InvalidNesting("Improper nesting"));
	| CData(c) -> elem = insertInlineAroundBlocks(block,inline,startblock,substart,endblock,subend);
	    // start in start block, end in end block,
	    // everything in between moves down a layer
      }
    }
  } else {
    case block.elements[endblock] of {
      // call it again on the sub element
      SubElement(el) -> elem = addInlineElementAt(el,inline,substart,subend);
      | CData(c) -> elem = insertInlineToCData(block,inline,substart,subend,endblock); // split into three parts a, <>b</>, c
    }
  }

  return elem;
}
// TODO: specify start and end by word position or by providing a
// substring to delimit.

// this function shouldn't need error-checking inside it.
private ElementTree insertInlineToCData(ElementTree block, InlineElement inline, Int startpos, Int endpos, Int blocknumber) {
  initial = block.elements[blocknumber].cdata;
  if (startpos > 0) {
    prefix = substr(initial,0,startpos);
  } else {
    prefix = createString(0);
  }
  if (endpos < length(initial)-1) {
    suffix = substr(initial,endpos+1,(length(initial)-endpos));
  } else {
    suffix = createString(0);
  }
  len = (endpos-startpos)+1;
  if (len < 1) {
    middle = createString(0);
  } else {
    middle = substr(initial,startpos,len);
  }
  
  toshove = createArray(3);
  if (prefix != "") {
    push(toshove,CData(prefix));
  }
  // this code needs making into a function elsewhere  
  elem = makeInlineElement(inline);
  doAddString(elem,middle);
  push(toshove,SubElement(elem));
  if (suffix != "") {
    push(toshove,CData(suffix));
  }
  
  subarrayReplace(block.elements,blocknumber,toshove);


  return elem;
}

// rewrite this to use addidx
private Void subarrayReplace([a] main, Int index, [a] shove) {
  main[index] = shove[0];
  if (size(shove) > 1) {
    difference = size(shove)-1;
    // move everything in main up a few places
    for (i=size(main)-1;i>index;i--) {
      main[i+difference] = main[i];
    }
    // and shove the replacement into the gap
    for (j=1;j<size(shove);j++) {
      main[index+j] = shove[j];
    }
  }
}

// both startblock and endblock are CData blocks
private ElementTree insertInlineAroundBlocks(ElementTree block, InlineElement inline, Int startblock, Int startpos, Int endblock, Int endpos) {
  // start block
  leftstr = block.elements[startblock].cdata;
  if (startpos == 0) {
    leftprefix = createString(0);
    leftsuffix = leftstr;
  } else {
    leftprefix = substr(leftstr,0,startpos);
    leftsuffix = substr(leftstr,startpos,length(leftstr)-startpos);
  }
  rightstr = block.elements[endblock].cdata;
  if (endpos == 0) {
    rightprefix = createString(0);
    rightsuffix = rightstr;
  } else {
    rightprefix = substr(rightstr,0,endpos);
    rightsuffix = substr(rightstr,endpos,length(rightstr)-endpos);
  }
  if (leftprefix != "") {
    block.elements[startblock] = CData(leftprefix);
  } else {
    startblock--; // so we fake having dealt with it.
  }
  if (rightsuffix != "") {
    block.elements[endblock] = CData(rightsuffix);
  } else {
    endblock++; //similarly
  }
  // startblock-endblock >= 2 always at this point.
  elem = makeInlineElement(inline);
  for i in [startblock+1..endblock-1] {
    case block.elements[i] of {
      SubElement(el) -> pushElement(elem,el);
      | CData(cd) -> doAddString(elem,cd);
    }
  }
  for i in [startblock+2..endblock-1] {
    removeAt(block.elements,startblock+2);
  }
  doAddString(elem,rightprefix);
  unshiftData(elem,leftsuffix);
  block.elements[startblock+1] = SubElement(elem);
  return elem;
}

"<argument name='block'>The block to add to</argument>
<summary>Add a line break</summary>
<prose>Append a line break to the block. Consecutive line breaks are not allowed - there are better ways to do this.</prose>"
public ElementTree addLineBreak(ElementTree block) {
  if (!canContainInline(block.name,false)) {  
    throw(InvalidNesting("Can't add an line break here"));
  }  
  if (size(block.elements) != 0) {
    lastelem = block.elements[size(block.elements)-1];
    case lastelem of {
      SubElement(el) -> if (el.name == "br") {
	throw(InvalidNesting("Doesn't make sense to have two line breaks in a row."));
      }
      | _ -> ;
    }
  }
  br = mkElement("br");
  pushElement(block,br);
  return br;
}

"<argument name='block'>The block to add to</argument>
<summary>Add a separator</summary>
<prose>Append a horizontal rule to the block as a separator. Consecutive separators are not allowed.</prose>"
public ElementTree addHorizontalRule(ElementTree block) {
  if (size(block.elements) != 0) {
    lastelem = block.elements[size(block.elements)-1];
    case lastelem of {
      SubElement(el) -> if (el.name == "hr") {
	throw(InvalidNesting("Doesn't make sense to have two horizontal rules in a row."));
      }
      | _ -> ;
    }
  }
  hr = mkElement("hr");
  pushElement(block,hr);
  return hr;
}


"<argument name='block'>The block to add to</argument>
<argument name='image'>The image to add</argument>
<summary>Add an image</summary>
<prose>Append the image to the block. See the <dataref>ImageData</dataref> documentation for more information about image definitions. Images require both a source and alternative text, with the best way to select the alternative text being to consider the text you would use in this context if no image was available. (Many decorative images will therefore have \"\" as their alternative text)</prose>
<example>p = addParagraph(parent,\"Sponsored by \");
logo = addImage(p,ImageData(\"logo.jpg\",\"MegaCorp\",Specified(100,18)));</example>
<related><dataref>ImageData</dataref></related>"
public ElementTree addImage(ElementTree block, ImageData image) {
  if (!canContainInline(block.name,false)) {  
    throw(InvalidNesting("Can't add an image here"));
  }
  img = mkElement("img");
  module::setAttribute(img,"src",image.src);
  module::setAttribute(img,"alt",image.alt);
  case image.dim of {
    Specified(width,height) -> module::setAttribute(img,"width",String(width));
    module::setAttribute(img,"height",String(height));
    // TODO: add support for AutoFromSrc and AutoFromPath
    | _ -> ;
  }

  pushElement(block,img);
  return img;
}


/** Part 7: Table editing - adding and editing tables **/


"<argument name='parent'>The parent element</argument>
<argument name='captiontext'>Optionally, text for a table caption.</argument>
<summary>Add a new empty table</summary>
<prose>Add a new empty table to the document. You can then set up optional header and footer sections, and at least one body section. The rows and columns of the table are then contained within the header, footer and body sections.</prose>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>getTableHeader</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>
<related><functionref>setTableCaption</functionref></related>"
public ElementTree addTable(ElementTree parent, String captiontext=createString(0)) {
  if (!canContainBlock(parent.name)) {
     throw(InvalidNesting("Can't add table inside "+parent.name));
  }
  table = mkElement("table");
  pushElement(parent,table);
  if (captiontext != "") {
    caption = mkElement("caption");
    doAddString(caption,captiontext);
    pushElement(table,caption);
  }
  return table;
}

"<argument name='table'>The table</argument>
<argument name='captiontext'>The new caption</argument>
<summary>Sets the table caption</summary>
<prose>Sets the caption of the table to the new value, replacing any existing caption. This is the only table manipulation function that will work on a <functionref>lazyTable</functionref>.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public Void setTableCaption(ElementTree table, String captiontext) {
  if (table.name != "table") {
    throw(RequiresElement("setTableCaption can only be used on table elements!"));
  }

  case table.elements[0] of {
    SubElement(el) -> if (el.name == "caption") {
      removeAt(table.elements,0);
    }
    | _ -> ;
  }

  caption = mkElement("caption");
  doAddString(caption,captiontext);
  unshiftElement(table,caption);
}

"<argument name='table'>The table</argument>
<summary>Get the table header section</summary>
<prose>Gets the table header section. If the table header section does not yet exist, it will be created. Rows can then be added to the section.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public ElementTree getTableHeader(ElementTree table) {
  if (table.name != "table") {
    throw(RequiresElement("Can't get a header from table elements!"));
  }
  for elem in table.elements {
    // should never be raw CData inside a table, so hit them with an
    // exception if they've managed to do it...
    if (elem.nested.name == "thead") {
      // return the existing thead if it's there.
      return elem.nested;
    }
  }
  // so, position it before the tfoot, the first tbody, at the end, in
  // that order.
  thead = mkElement("thead");
  index = -1;
  case findElement(table,"tfoot") of {
    just(x) -> index = x;
    | nothing -> case findElement(table,"tbody") of {
      just(x) -> index = x;
      | nothing -> ;
    }
  }
  if (index == -1) {
    pushElement(table,thead); // on the end
  } else {
    addElementAt(table,thead,index); // in the right place
  }
  return thead;
}

"<argument name='table'>The table</argument>
<summary>Get the table footer section</summary>
<prose>Gets the table footer section. If the table footer section does not yet exist, it will be created. Rows can then be added to the section.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>getTableHeader</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public ElementTree getTableFooter(ElementTree table) {
  if (table.name != "table") {
    throw(RequiresElement("Can't get a footer from non-table elements!"));
  }
  for elem in table.elements {
    // should never be raw CData inside a table, so hit them with an
    // exception if they've managed to do it...
    if (elem.nested.name == "tfoot") {
      // return the existing thead if it's there.
      return elem.nested;
    }
  }
  // so, position it before the tfoot, the first tbody, at the end, in
  // that order.
  tfoot = mkElement("tfoot");
  case findElement(table,"tbody") of {
    just(x) -> addElementAt(table,tfoot,x);
    | nothing -> pushElement(table,tfoot);
  }
  return tfoot;
}

"<argument name='table'>The table</argument>
<summary>Add a new table body section</summary>
<prose>Adds a new table body section to the end of the table. Rows can then be added to the section. Tables should have at least one body section, but may have an unlimited number.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>getTableHeader</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public ElementTree addTableBodySection(ElementTree table) {
  if (table.name != "table") {
    throw(RequiresElement("Can't get a tbody from non-table elements!"));
  }
  tbody = mkElement("tbody");
  pushElement(table,tbody);
  return tbody;
}

"<argument name='table'>The table</argument>
<summary>Get the table body sections</summary>
<prose>Get the table body sections. If no table body sections exist, one will be created. Rows can then be added to the section.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>getTableHeader</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>lazyTable</functionref></related>"
public [ElementTree] getTableBodySections(ElementTree table) {
  if (table.name != "table") {
    throw(RequiresElement("Can't get a tbody from non-table elements!"));
  }
  case findElement(table,"tbody") of {
    nothing -> return [addTableBodySection(table)];
    | just(x) -> sz = size(table.elements);
    arr = subarray(table.elements,x,sz-x);
    return map(extractSubElement,arr);
  }
}

// this doesn't work with lambda, oddly
private ElementTree extractSubElement(Element val) {
  return val.nested;
}

private Bool isTsect(String name) {
  return (name=="tbody" || name=="thead" || name=="tfoot");
}

"<argument name='table'>The table</argument>
<summary>Get the number of columns in a table</summary>
<prose>Get the number of columns in a table</prose>
<related><functionref>addTableColumns</functionref></related>"
public Int numTableColumns(ElementTree table) {
  if (table.name != "table") {
    throw(RequiresElement("Can't get number of columns if it's not a table!"));
  }
  // whatever the first row of the table is can't be miscounted
  // by rowspans.
  for i in table.elements {
    // no Cdata should be here
    if (isTsect(i.nested.name)) {
      // ignore empty tsects.
      // any non-empty tsect will have the same number of columns
      if (size(i.nested.elements) != 0) {
	return numTsectColumns(i.nested);
      }
    }
  }
  return 0;
}

private Int numTsectColumns(ElementTree tsect) {
  cols = 0;
  // use the first one to avoid needing rowspan calculations
  for td in tsect.elements[0].nested.elements {
    case lookup(td.nested.attributes,"colspan") of {
      nothing -> cols++;
      | just(c) -> cols += Int(c);
    }
  }
  return cols;
}

"<argument name='tsect'>The table header, footer or body section</argument>
<argument name='class'>Optionally, the CSS class to apply to the row</argument>
<summary>Adds a table row</summary>
<prose>Add a new row to the end of the selected header, body or footer section of a table. Once the table as a whole has at least one row, columns may be added.</prose>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>addTableColumns</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>getTableCell</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>getTableHeader</functionref></related>"
public ElementTree addTableRow(ElementTree tsect, String class=createString(0)) {
  if (!isTsect(tsect.name)) {
    throw(RequiresElement("Can't add a table row here!"));
  }
  tr = mkElement("tr");
  pushElement(tsect,tr);
  if (class != "") {
    setClass(tr,class);
  }
  columns = numTsectColumns(tsect);
  if (columns > 0) {
    for i in [1..columns] {
      // can do this because we don't let rowspans extend outside the
      // existing tsect rows
      pushElement(tr,mkElement("td"));
    }
  }
  return tr;
}

"<argument name='table'>The table</argument>
<argument name='num'>Optionally, the number of columns to add. If omitted, a single column is added.</argument>
<summary>Add a new table column</summary>
<prose>Add a new column to the table. The table must have at least one section with at least one row for this to work - a <exceptref>RequiresElement</exceptref> Exception will be thrown otherwise. Unlike most of the table functions, this function does not return the new elements created since they will be in different rows.</prose>
<related><functionref>addTable</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableCell</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>numTableColumns</functionref></related>"
public Void addTableColumns(ElementTree table, Int num=1) {
  if (table.name != "table") {
    throw(RequiresElement("Can't add columns if it's not a table!"));
  }
  for j in [1..num] {
    valid = false;
    for i in table.elements {
      // no Cdata should be here
      if (isTsect(i.nested.name)) {
	for tr in i.nested.elements {
	  pushElement(tr.nested,mkElement("td"));
	  valid = true;
	}
      }
    }
    if (!valid) {
      throw(RequiresElement("Can't add columns until the table has at least one row"));
    }
  }
}

"<argument name='tsect'>The table header, footer or body section</argument>
<argument name='row'>The row to retrieve from (starting at 0)</argument>
<argument name='col'>The column to retrieve from (starting at 0)</argument>
<summary>Make a table cell a header cell</summary>
<prose>Makes a table cell a header cell and returns the cell. Apart from converting the cell to a header cell if it wasn't already, this is identical to <functionref>getTableCell</functionref>.</prose>
<related><functionref>getTableCell</functionref></related>
<related><functionref>makeDataCell</functionref></related>"
public ElementTree makeHeaderCell(ElementTree tsect, Int row, Int col) {
  cell = getTableCell(tsect,row,col);
  cell.name = "th";
  return cell;
}

"<argument name='tsect'>The table header, footer or body section</argument>
<argument name='row'>The row to retrieve from (starting at 0)</argument>
<argument name='col'>The column to retrieve from (starting at 0)</argument>
<summary>Make a table cell a data cell</summary>
<prose>Makes a table cell a data cell and returns the cell. Apart from converting the cell to a data cell if it wasn't already, this is identical to <functionref>getTableCell</functionref>. All cells are data cells by default, unless created with <functionref>initialiseTable</functionref> (or <functionref>lazyTable</functionref>, though of course those cells are not retrievable with this function) in the header section.</prose>
<related><functionref>getTableCell</functionref></related>
<related><functionref>makeHeaderCell</functionref></related>"
public ElementTree makeDataCell(ElementTree tsect, Int row, Int col) {
  cell = getTableCell(tsect,row,col);
  cell.name = "td";
  return cell;
}

"<argument name='tsect'>The table header, footer or body section</argument>
<argument name='row'>The row to retrieve from (starting at 0)</argument>
<argument name='col'>The column to retrieve from (starting at 0)</argument>
<summary>Retrieve a table cell</summary>
<prose>Retrieve a table cell so that content may be added to it. An <exceptref>Array::OutOfBounds</exceptref> Exception will be thrown if the row and column are not within the table.</prose>
<example>table = addTable(parent,\"Example table\");
tbody = addTableBodySection(table);
for i in [0..2] {
    void(addTableRow(tbody));
}
addTableColumns(table,3);
for i in [0..2] {
    for j in [0..2] {
        td = getTableCell(tbody,i,j);
        addString(td,String(i+j));
    }
}
/*
Example table
+---+---+---+
| 0 | 1 | 2 |
+---+---+---+
| 1 | 2 | 3 |
+---+---+---+
| 2 | 3 | 4 |
+---+---+---+
*/</example>
<prose>Naturally, the table in this simple example would be easier to generate using <functionref>initialiseTable</functionref>.</prose>
<related><functionref>addTableBodySection</functionref></related>
<related><functionref>addTableColumns</functionref></related>
<related><functionref>addTableRow</functionref></related>
<related><functionref>getTableBodySections</functionref></related>
<related><functionref>makeDataCell</functionref></related>
<related><functionref>makeHeaderCell</functionref></related>
<related><functionref>getTableFooter</functionref></related>
<related><functionref>getTableHeader</functionref></related>"
public ElementTree getTableCell(ElementTree tsect, Int row, Int col) {
  // this does some error checking for mHC and mDC
  if (!isTsect(tsect.name)) {
    throw(RequiresElement("Can only get cells from tbody, thead or tfoot!"));
  }
  if (size(tsect.elements) <= row) {
    throw(OutOfBounds);
  }
  if (size(tsect.elements[row].nested.elements) <= col) {
    throw(OutOfBounds);
  }
  return tsect.elements[row].nested.elements[col].nested;
}

"<argument name='parent'>The parent element</argument>
<argument name='itd'>The <dataref>InitialTableData</dataref></argument>
<argument name='caption'>Optionally, a caption for the table</argument>
<summary>Create a table and initialise contents</summary>
<prose>Creates a table with the sections, rows, columns and cells described in the <code>itd</code> parameter - see the <dataref>InitialTableData</dataref> documentation for more details. Cells will be created as data cells by default, except for cells in the header section, which will be created as header cells.</prose>
<related><dataref>InitialTableData</dataref></related>
<related><functionref>addTable</functionref></related>
<related><functionref>lazyTable</functionref></related>
<related><functionref>setTableCaption</functionref></related>"
public ElementTree initialiseTable(ElementTree parent, InitialTableData itd, String caption=createString(0)) {
  table = addTable(parent,caption);
  // create the rows
  if (size(itd.header) > 0 && size(itd.header[0]) > 0) {
    thead = getTableHeader(table);
    for i in [1..size(itd.header)] {
      tr = addTableRow(thead);
    }
    fillheader = true;
    cols = size(itd.header[0]);
  } else {
    fillheader = false;
  }
  if (size(itd.footer) > 0 && size(itd.footer[0]) > 0) {
    tfoot = getTableFooter(table);
    for i in [1..size(itd.footer)] {
      tr = addTableRow(tfoot);
    }
    cols = size(itd.footer[0]);
    fillfooter = true;
  } else {
    fillfooter = false;
  }
  if (size(itd.sections) > 0 && size(itd.sections[0]) > 0 && size(itd.sections[0][0]) > 0) {
    tbc = 0;
    for tbodies in itd.sections {
      tbody[tbc] = addTableBodySection(table);
      for i in [1..size(tbodies)] {
	tr = addTableRow(tbody[tbc]);
      } 
      tbc++;
    }
    cols = size(itd.sections[0][0]);
    fillbody = true;
  } else {
    fillbody = false;
  }
  // create the columns
  addTableColumns(table,cols);

  // add the data
  if (fillheader) {
    fillBlockWithData(thead,itd.header,true);
  }
  if (fillfooter) {
    fillBlockWithData(tfoot,itd.footer);
  }
  if (fillbody) {
    for (k=0;k<size(itd.sections);k++) {
      fillBlockWithData(tbody[k],itd.sections[k]);
    }
  }
  return table;
}

// utility function for above
private Void fillBlockWithData(ElementTree block, [[String]] cdata, Bool forceheader=false) {
  for (i=0;i<size(cdata);i++) {
    for (j=0;j<size(cdata[i]);j++) {
      if (forceheader) {
	tc = makeHeaderCell(block,i,j);
      } else {
	tc = getTableCell(block,i,j);
      }
      doAddString(tc,cdata[i][j]);
    }
  }
}

"<argument name='parent'>The parent element</argument>
<argument name='itd'>The <dataref>InitialTableData</dataref></argument>
<argument name='caption'>Optionally, a caption for the table</argument>
<summary>Create a lazy table</summary>
<prose>Creates a lazy table with the sections, rows, columns and cells described in the <code>itd</code> parameter - see the <dataref>InitialTableData</dataref> documentation for more details. Cells will be created as data cells by default, except for cells in the header section, which will be created as header cells.</prose>
<prose>This function differs from <code>initialiseTable</code> in that the contents of the table are only partially applied. They are generated only when the table is converted to a string, which saves a considerable amount of memory for large tables, but has the disadvantage that the contents may not be edited, and so may only consist of plain text.</prose>
<related><dataref>InitialTableData</dataref></related>
<related><functionref>addTable</functionref></related>
<related><functionref>initialiseTable</functionref></related>
<related><functionref>setTableCaption</functionref></related>"
public ElementTree lazyTable(ElementTree parent, InitialTableData itd, String caption=createString(0)) {
  table = addTable(parent,caption);
  // create the rows
  if (size(itd.header) > 0 && size(itd.header[0]) > 0) {
    fillheader = true;
    cols = size(itd.header[0]);
  } else {
    fillheader = false;
  }
  if (size(itd.footer) > 0 && size(itd.footer[0]) > 0) {
    cols = size(itd.footer[0]);
    fillfooter = true;
  } else {
    fillfooter = false;
  }
  if (size(itd.sections) > 0 && size(itd.sections[0]) > 0 && size(itd.sections[0][0]) > 0) {
    tbc = 0;
    cols = size(itd.sections[0][0]);
    fillbody = true;
  } else {
    fillbody = false;
  }
  
  if (fillheader) {
    appendGenerator(table,lazyTHead@(cols,itd.header));
  }
  if (fillfooter) {
    appendGenerator(table,lazyTFoot@(cols,itd.footer));
  }
  if (fillbody) {
    for tbody in itd.sections {
      appendGenerator(table,lazyTBody@(cols,tbody));
    }
  }
  return table;
}

/* LAZY */
ElementTree lazyTHead(Int cols, [[String]] dat) = lazyTBlock("th","thead",cols,dat);
ElementTree lazyTFoot(Int cols, [[String]] dat) = lazyTBlock("td","tfoot",cols,dat);
ElementTree lazyTBody(Int cols, [[String]] dat) = lazyTBlock("td","tbody",cols,dat);

ElementTree lazyTBlock(String cell, String block, Int cols, [[String]] dat) {
  tblock = mkElement(block,size(dat));
  for row in dat {
    appendGenerator(tblock,lazyTRow@(cell,cols,row));
  }
  return tblock;
}

// probably a row at a time is lazy enough.
ElementTree lazyTRow(String cell, Int cols, [String] row) {
  tr = mkElement("tr",cols);
  for i in [0..cols-1] {
    tcell = mkElement(cell);
    doAddString(tcell,row[i]);
    pushElement(tr,tcell);
  }
  return tr;
}



/** Part 8: Form editing - including Kaya state maintenance magic **/


"<argument name='parent'>The parent element</argument>
<argument name='fileupload'>Does this particular form allow file uploads. This argument can be omitted, for a default of disallowing them. Note that <functionref>WebCommon::allowFileUploads</functionref> must also be called for this to be successfully enabled.</argument>
<summary>Add a form for the current application</summary>
<prose>This adds a form that points to the current web application to call a named function. You then need to use <functionref>addLocalControlInput</functionref> to choose the function to call.</prose>
<prose>If you are using <code>mod_rewrite</code> in Apache, or other methods of rewriting URLs at the webserver level, then you may need to call <code>setAttribute(form,\"action\",rewritten)</code>, as Kaya cannot know about this URL rewriting layer.</prose>
<prose>Note that in HTML form input controls may not be placed directly inside a form - you must add block-level elements to the form and add controls to those elements. The <code>fieldset</code> element is especially suited to this and the <functionref>addFieldset</functionref> function handles this.</prose>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLocalControlInput</functionref></related>
<related><functionref>addRemoteForm</functionref></related>"
public ElementTree addLocalForm(ElementTree parent, Bool fileupload=false) {
  form = addForm(parent);
  module::setAttribute(form,"action",webappName());
  module::setAttribute(form,"method","post");
  if (fileupload) {
    module::setAttribute(form,"enctype","multipart/form-data");
  }
  return form;
}


"<argument name='parent'>The parent element</argument>
<argument name='action'>The URL of the other application</argument>
<argument name='method'>The method to use to send the form data</argument>
<summary>Add a form to call another application</summary>
<prose>This adds a form that points to a specified URL. Unless the other URL is a (byte for byte) identical copy of the current application, you will not be able to use Kaya's special state handling, and must construct another way of passing the information. If the security of the information is not critical, and the other application is also a Kaya application, then the <moduleref>Pickle</moduleref> module may be of use.</prose>
<example>form = addRemoteForm(doc.body,
                     \"http://www.example.com/email.cgi\",
                     FormPost);</example>
<prose>Note that in HTML form input controls may not be placed directly inside a form - you must add block-level elements to the form and add controls to those elements. The <code>fieldset</code> element is especially suited to this and the <functionref>addFieldset</functionref> function handles this.</prose>
<related><dataref>FormType</dataref></related>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLocalForm</functionref></related>"
public ElementTree addRemoteForm(ElementTree parent, String action, FormType method) {
  form = addForm(parent);
  module::setAttribute(form,"action",action);
  case method of {
    FormGet -> module::setAttribute(form,"method","get");
    | FormPost -> module::setAttribute(form,"method","post");
    | FormPostUpload -> module::setAttribute(form,"method","post");
    module::setAttribute(form,"enctype","multipart/form-data");
  }
  return form;
}

// called by local and remote form
private ElementTree addForm(ElementTree parent) {
  if (!canContainBlock(parent.name)) {
     throw(InvalidNesting("Can't add a form inside "+parent.name));
  }
  form = mkElement("form");
  pushElement(parent,form);
  return form;
}

"<argument name='parent'>The parent element</argument>
<argument name='itype'>The type of text input to add</argument>
<argument name='iname'>The name of the input. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='ivalue'>The initial value of the input. For text and password fields, this may be changed by the user in their web browser (someone attacking your application may of course change any value, regardless of the input type). This value is generally ignored by web browsers for file upload inputs. This value is optional, and defaults to the empty String.</argument>
<argument name='isize'>Optionally, the display size of the input element. Browser rules for determining this size vary - it is an approximate number of characters, but only approximate. Obviously browsers ignore this field for hidden fields and checkbox and radio inputs, and usually for file upload inputs too.</argument>
<summary>Adds a text input</summary>
<prose>Adds a text input to a form. In HTML, controls may not be added directly to a form, so you must add them to a block-level element such as <code>fieldset</code> within the form.</prose>
<related><dataref>TextInputType</dataref></related>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLabelledInput</functionref></related>
<related><functionref>addOption</functionref></related>
<related><functionref>addSelectElement</functionref></related>
<related><functionref>addTextarea</functionref></related>"
public ElementTree addTextInput(ElementTree parent, TextInputType itype, String iname, String ivalue=createString(0), Int isize=0) {
  if (!canContainInline(parent.name)) {
     throw(InvalidNesting("Can't add a form input inside "+parent.name));
  }
  if (iname == "") {
     throw(RequiresAttribute("Input name can't be empty."));
  }
  input = mkElement("input");
  pushElement(parent,input);
  module::setAttribute(input,"name",iname);
  module::setAttribute(input,"value",ivalue);
  case itype of {
    InputHidden -> module::setAttribute(input,"type","hidden");
    | InputPassword -> module::setAttribute(input,"type","password");    
    module::setAttribute(input,"size",String(isize));
    | InputText -> module::setAttribute(input,"type","text");
    module::setAttribute(input,"size",String(isize));
    | InputCheck -> module::setAttribute(input,"type","checkbox");
    | InputRadio -> module::setAttribute(input,"type","radio");
    | InputFile -> module::setAttribute(input,"type","file");
  }

  return input;
}

"<argument name='parent'>The parent element</argument>
<argument name='fn'>The function to call</argument>
<argument name='state'>The state to pass to the function</argument>
<summary>Call a CGI function</summary>
<prose>Adds the state and processing function variables to a block-level element within a form created with <functionref>addLocalForm</functionref>. This function only works with the <moduleref>CGI</moduleref> state handling model, and is equivalent to using <functionref>CGI::formHandler</functionref> for non-HTMLDocument CGI programs.</prose>
<related><moduleref>CGI</moduleref></related>"
public Void addStateFields(ElementTree parent, Void(a) fn, a state) {
  discard = addTextInput(parent,InputHidden,"kaya_function",encode(String(fnid(fn))));
  discard = addTextInput(parent,InputHidden,"kaya_arg",encode(marshal(state,fnid(fn))));
}

"<argument name='parent'>The parent element (often the form, but in many cases nested fieldsets make sense)</argument>
<argument name='legendtext'>The 'legend' of the fieldset. This text is generally displayed at the top of the fieldset as a descriptive heading.</argument>
<summary>Add a new fieldset</summary>
<prose>Add a new fieldset to a form. Fieldsets are very useful for containing form controls which may not be directly contained within a form.</prose>"
public ElementTree addFieldset(ElementTree parent, String legendtext) {
  if (!canContainBlock(parent.name)) {
     throw(InvalidNesting("Can't add a fieldset inside "+parent.name));
  }
  fset = mkElement("fieldset");
  pushElement(parent,fset);
  legend = mkElement("legend");
  doAddString(legend,legendtext);
  pushElement(fset,legend);
  return fset;
}

"<argument name='block'>The element containing the text</argument>
<argument name='startpos'>The character position of the first character</argument>
<argument name='endpos'>The character position of the last character</argument>
<argument name='control'>The form control</argument>
<summary>Associates some text with a form control</summary>
<prose>Associates text with a form control. In some common graphical browsers, clicking on the text will then focus or activate the form control, which is especially useful for checkboxes and radio buttons. The related functions below all add form controls which are already associated with their label texts, and so this function is not needed for these.</prose>
<related><functionref>addLabelledInput</functionref></related> 
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLabelledTextarea</functionref></related>
<related><functionref>addOptionList</functionref></related>"
public Void associateTextAndInput(ElementTree block, Int startpos, Int endpos, ElementTree control) {
  label = addInlineElementAt(block,FormLabel,startpos,endpos);
  doAssociateTandI(label,control);
}

private Void doAssociateTandI(ElementTree label, ElementTree control) {
  // private function so don't need to do this checking
  /*  if (label.name != "label") {
    throw(RequiresElement("The first parameter must be a label element"));
  }
  if (control.name == "input" || control.name == "select" || control.name == "textarea") {
  */
  uid = nextUniqueID();
  module::setAttribute(label,"for",uid);
  setID(control,uid);
  /*  } else {
    throw(RequiresElement("The second parameter must be a form field element"));
    }*/
}

Void normaliseOpt([SelectOption] opts, Bool ismult) {
  for opt in opts {
    if (opt.chosen) {
      if (ismult) {
	opt.chosen = false;
      } else {
	ismult = true;
      }
    } 
  }
}

Int normaliseOptgroups([(String,[SelectOption])] optgroups, Bool allowmult) {
  isize = 0;
  ismult = false;
  for optgroup in optgroups {
    if (optgroup.fst != "") {
      isize++;
    } else {
      isize += size(optgroup.snd);
    }
    if (allowmult==false) {
      normaliseOpt(optgroup.snd,ismult);
    }
  }
  return isize;
}

"<argument name='parent'>The parent element</argument>
<argument name='name'>The name of the input. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='ssize'>The size of the select element. If this is zero, the select element will only allow one option to be selected at any one time. If this is one or more, the select element will allow multiple options to be selected, and suggest to the browser that this many options be displayed simultaneously.</argument>
<argument name='optgroups'>The options to select from</argument>
<summary>Adds a selection box</summary>
<prose>Adds a selection box to a form. The <code>optgroups</code> parameter is a list of pairs. The first element of the pair is the 'heading' for the option group, and the second element is a list of options in that group. For most simple selectors, a single option group with no heading (the empty string) is sufficient.</prose>
<example>options = [
    SelectOption(\"Express delivery\",\"1\",true),
    SelectOption(\"Standard delivery\",\"2\",false),
    SelectOption(\"Slow delivery\",\"3\",false)
];
sel = addSelectElement(fieldset,\"choice\",0,[(\"\",options)]);
/* // produces 
&lt;select>
  &lt;option value='1' selected='selected'>Express delivery&lt;/option>
  &lt;option value='2'>Standard delivery&lt;/option>
  &lt;option value='3'>Slow delivery&lt;/option>
&lt;/select>
*/</example>
<prose>Using multiple groups of options is useful for larger select elements, where it can make the form clearer.</prose>
<example>singles = [\"A1\",\"A2\",\"B5\"];
twins = [\"A4\",\"C2\"];
doubles = [\"A7\",\"C1\",\"C3\"];
sopts = [];
topts = [];
dopts = [];
for s in singles {
    push(sopts,SelectOption(s,s,false);
}
for t in twins {
    push(topts,SelectOption(s,s,false);
}
for d in doubles {
    push(dopts,SelectOption(s,s,false);
}
options = [
    (\"Single rooms\",sopts),
    (\"Twin rooms\",topts),
    (\"Double rooms\",dopts)
];
sel = addSelectElement(roombooker,\"room\",0,options);
</example>
<prose>Select elements allowing multiple options to be selected have very bad usability in most browsers - it is often better to use <functionref>addOptionList</functionref> to generate a set of checkboxes instead. Whichever method you use for multiple selection, remember that you need to use <functionref>WebCommon::incomingData</functionref> to correctly retrieve the selections from the user's form submission.</prose>
<related><dataref>SelectOption</dataref></related>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLazySelect</functionref></related>
<related><functionref>addOptionList</functionref></related>
<related><functionref>addTextarea</functionref></related>
<related><functionref>addTextInput</functionref></related>"
public ElementTree addSelectElement(ElementTree block, String name, Int ssize, [(String,[SelectOption])] optgroups) {
  if (!canContainInline(block.name)) {
    throw(InvalidNesting("Can't add a form input inside "+block.name));
  }
  isize = normaliseOptgroups(optgroups,(ssize>0));
  select = mkElement("select",isize);
  module::setAttribute(select,"name",name);
  if (ssize > 0) {
    module::setAttribute(select,"size",String(ssize));
    module::setAttribute(select,"multiple","multiple");
  }
  for optgroup in optgroups {
    if (optgroup.fst != "") {
      og = mkElement("optgroup");
      module::setAttribute(og,"label",optgroup.fst);
      for option in optgroup.snd {
	if (option.value == "") {
	  option.value = option.label;
	}
	opt = mkElement("option");
	module::setAttribute(opt,"value",option.value);
	module::setAttribute(opt,"label",option.label);
	if (option.chosen) {
	  module::setAttribute(opt,"selected","selected");
	}
	doAddString(opt,optgroup.fst+": "+option.label);
	pushElement(og,opt);
      }
      pushElement(select,og);
    } else {
      for option in optgroup.snd {
	if (option.value == "") {
	  option.value = option.label;
	}
	opt = mkElement("option");
	module::setAttribute(opt,"value",option.value);
	if (option.chosen) {
	  module::setAttribute(opt,"selected","selected");
	}
	doAddString(opt,option.label);
	pushElement(select,opt);
      }
    }
  }
  pushElement(block,select);
  return select;
}

"<argument name='parent'>The parent element</argument>
<argument name='name'>The name of the input. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='ssize'>The size of the select element. If this is zero, the select element will only allow one option to be selected at any one time. If this is one or more, the select element will allow multiple options to be selected, and suggest to the browser that this many options be displayed simultaneously.</argument>
<argument name='optgroups'>The options to select from</argument>
<summary>Lazily adds a selection box</summary>
<prose>Adds a lazy selection box to a form. The end result after conversion to a <code>String</code> is identical to that produced by <functionref>addSelectElement</functionref>, but the memory usage is lower. The disadvantage, however, is that <functionref>Webapp::autoFill</functionref> cannot be used to automatically complete forms containing lazy elements such as this one.</prose>
<related><dataref>SelectOption</dataref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addSelectElement</functionref></related>"
public ElementTree addLazySelect(ElementTree block, String name, Int ssize, [(String,[SelectOption])] optgroups) {
  if (!canContainInline(block.name)) {
    throw(InvalidNesting("Can't add a form input inside "+block.name));
  }
  isize = normaliseOptgroups(optgroups,(ssize>0));
  select = mkElement("select",isize);
  module::setAttribute(select,"name",name);
  if (ssize > 0) {
    module::setAttribute(select,"size",String(ssize));
    module::setAttribute(select,"multiple","multiple");
  }
  for optgroup in optgroups {
    if (optgroup.fst != "") {
      appendGenerator(select,lazyOptgroup@(optgroup));
    } else {
      for option in optgroup.snd {
	appendGenerator(select,lazyOption@(option,createString(0)));
      }
    }
  }
  pushElement(block,select);
  return select;
}

ElementTree lazyOptgroup((String,[SelectOption]) optgroup) {
  og = mkElement("optgroup");
  module::setAttribute(og,"label",optgroup.fst);
  for option in optgroup.snd {
    appendGenerator(og,lazyOption@(option,optgroup.fst));
  }
  return og;
}

ElementTree lazyOption(SelectOption option, String group) {
  opt = mkElement("option");
  if (option.value == "") {
    option.value = option.label;
  } else {
    module::setAttribute(opt,"value",option.value);
  }
  module::setAttribute(opt,"label",option.label);
  if (option.chosen) {
    module::setAttribute(opt,"selected","selected");
  }
  if (group != "") {
    doAddString(opt,group+": "+option.label);
  } else {
    doAddString(opt,option.label);
  }
  return opt;
}

"<argument name='parent'>The parent element</argument>
<argument name='labeltext'>The label for the selector</argument>
<argument name='name'>The name of the input. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='ssize'>The size of the select element. If this is zero, the select element will only allow one option to be selected at any one time. If this is one or more, the select element will allow multiple options to be selected, and suggest to the browser that this many options be displayed simultaneously.</argument>
<argument name='optgroups'>The options to select from</argument>
<argument name='lazy'>This parameter is optional and defaults to <code>false</code>. If it is explicitly set to <code>true</code> then the selector will be generated lazily.</argument>
<summary>Adds a labelled selection box</summary>
<prose>Adds a labelled selection box to a form. The selector itself is identical to that produced by <functionref>addSelectElement</functionref> or <functionref>addLazySelect</functionref>, depending on the value of the <code>lazy</code> parameter. Generally, it is more convenient to use this function to label selectors, than to create them individally.</prose>
<related><dataref>SelectOption</dataref></related>
<related><functionref>addLazySelect</functionref></related>
<related><functionref>addOptionList</functionref></related>
<related><functionref>addSelectElement</functionref></related>"
public ElementTree addLabelledSelect(ElementTree block, String labeltext, String name, Int ssize, [(String,[SelectOption])] optgroups, Bool lazy=false) {
  div = addDivision(block);
  label = appendInlineElement(div,FormLabel,labeltext);
  if (!lazy) {
    control = addSelectElement(div,name,ssize,optgroups);
  } else {
    control = addLazySelect(div,name,ssize,optgroups);
  }
  doAssociateTandI(label,control);
  return div;

}

"<argument name='fn'>The function to call</argument>
<argument name='state'>The state to pass</argument>
<argument name='prefix'>An optional prefix to the control variables. Different components may use different prefixes, allowing multiple <functionref>Webapp::runHandler</functionref> functions.</argument>
<summary>Create a URL that calls a local function</summary>
<prose>Create a URL that calls a local function for use in hyperlinks. You may pass additional parameters by appending them to the URL, of course.</prose>
<example>url = localControlURL(browseData,logindata);
url += \";datasource=12\";</example>
<related><functionref>CGI::imageHandler</functionref></related>
<related><functionref>CGI::linkURL</functionref></related>
<related><functionref>encodeControlState</functionref></related>
<related><functionref>addLocalControlInput</functionref></related>"
public String localControlURL(b(a) fn, a state, String prefix="") {
  name = prefix+"kaya_control";
  val = urlEncode(encodeControlState(@fn,state,prefix));
  return webappName() + "?" + name + "=" + val;
}

"<argument name='fn'>The function to call</argument>
<argument name='state'>The state to pass</argument>
<argument name='prefix'>An optional prefix to the control variables. Different components may use different prefixes, allowing multiple <functionref>Webapp::runHandler</functionref> functions. In this context it performs the same role as the marshalling ID - if you are calling this function directly you almost certainly don't need it.</argument>
<summary>Encode application state for later retrieval</summary>
<prose>Encode an application state into a string. This is useful with <functionref>Webapp::storeFunction</functionref> for constructing shorter URLs, or allowing someone to resume a process at a much later date.</prose>
<related><functionref>Webapp::storeFunction</functionref></related>"
public String encodeControlState(b(a) fn, a state, String prefix="") {
  return encode(compressString(marshal((@fn,state,prefix),228)));
}

"<argument name='parent'>The element to add the button to</argument>
<argument name='label'>The text for the button</argument>
<argument name='fn'>The function to call</argument>
<argument name='state'>The state to pass</argument>
<argument name='prefix'>An optional prefix to the control variables. Different components may use different prefixes, allowing multiple <functionref>Webapp::runHandler</functionref> functions.</argument>
<summary>Add a form submit button</summary>
<prose>Create a submit button that calls a local function. This only works if added to a form created with <functionref>addLocalForm</functionref>.</prose>
<related><functionref>addLocalForm</functionref></related>
<related><functionref>addRemoteControlInput</functionref></related>"
public ElementTree addLocalControlInput(ElementTree parent, String label, b(a) fn, a state, String prefix="") {
  if (!canContainInline(parent.name)) {
     throw(InvalidNesting("Can't add a form input inside "+parent.name));
  }
  input = mkElement("input");
  pushElement(parent,input);
  module::setAttribute(input,"type","submit");
  module::setAttribute(input,"value",label);

  submitid = nextUniqueID();
  module::setAttribute(input,"name",prefix+"kaya_submit_"+submitid);

  stateinfo = encode(compressString(marshal((fn,state,prefix),Int(substr(submitid,3,length(submitid)-3)))));
  hinput = addTextInput(parent,InputHidden,prefix+"kaya_control_"+submitid,stateinfo);
  
  return input;
}

"<argument name='parent'>The element to add the button to</argument>
<argument name='itype'>The type of the button</argument>
<argument name='iname'>The name of the button</argument>
<argument name='ivalue'>The label of the button</argument>
<summary>Add a form submit button</summary>
<prose>Create a submit button for a form created with <functionref>addRemoteForm</functionref>. The button will be submitted when pressed as <code><variable>iname</variable>=<variable>ivalue</variable></code>. If <code>iname</code> is the empty string, then the value will not be submitted with the rest of the form.</prose>
<related><dataref>ControlInputType</dataref></related>
<related><functionref>addLocalControlInput</functionref></related>
<related><functionref>addRemoteForm</functionref></related>"
public ElementTree addRemoteControlInput(ElementTree parent, ControlInputType itype, String iname=createString(0), String ivalue=createString(0)) {
  if (!canContainInline(parent.name)) {
     throw(InvalidNesting("Can't add a form input inside "+parent.name));
  }
  input = mkElement("input");
  pushElement(parent,input);
  case itype of {
    InputSubmit -> module::setAttribute(input,"type","submit");
    if (iname != "") {
      module::setAttribute(input,"name",iname);
    }
    | InputReset -> module::setAttribute(input,"type","reset");
  }
  if (ivalue != "") {
    module::setAttribute(input,"value",ivalue);
  }

  return input;
}

"<argument name='parent'>The parent element</argument>
<argument name='legend'>The legend for the fieldset grouping the options</argument>
<argument name='iname'>The name of the option controls. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='options'>The options to select from</argument>
<argument name='allowmultiple'>This parameter is optional and defaults to <code>true</code>, which generates the option list as a set of checkboxes. If it is explicitly set to <code>false</code> then radio buttons are used instead.</argument>
<summary>Adds a set of checkboxes or radio buttons</summary>
<prose>Adds a set of checkboxes or radio buttons to a form. The meaning of the parameters is very similar or identical to the similar parameters for <functionref>addLabelledSelect</functionref>, but the appearance is very different. If the number of options is not large, this is generally considerably easier to use than a selection drop-down, although it does take up much more screen space.</prose>
<prose>When using this function to generate radio buttons, you should ensure that one of the options is initially selected, as browser behaviour when no option is selected is variable and often causes problems. You may need to add a \"no option selected\" radio button for initial selection in some circumstances.</prose>
<example>options = [
    SelectOption(\"Express delivery\",\"1\",true),
    SelectOption(\"Standard delivery\",\"2\",false),
    SelectOption(\"Slow delivery\",\"3\",false)
];
sel = addLabelledSelect(fieldset,\"Select a delivery speed\",\"choice\",
                        options,false);
/* // produces 
&lt;fieldset>&lt;legend>Select a delivery speed&lt;/legend>
  &lt;div>&lt;input type='checkbox' name='choice' value='1' id='Kay1' 
                    checked='checked'>
    &lt;label for='Kay1'>Express delivery&lt;/label>&lt;/div>
  &lt;div>&lt;input type='checkbox' name='choice' value='2' id='Kay2'>
    &lt;label for='Kay2'>Standard delivery&lt;/label>&lt;/div>
  &lt;div>&lt;input type='checkbox' name='choice' value='3' id='Kay3'>
    &lt;label for='Kay3'>Slow delivery&lt;/label>&lt;/div>
&lt;/fieldset>
*/ // exact markup may vary slightly to keep IDs unique in the document</example>
<related><dataref>SelectOption</dataref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addOption</functionref></related>"
// gives them all the same name for symmetry with <select (multiple)>
// survey engines may want some standard options arrays.
public ElementTree addOptionList(ElementTree parent, String legend, String iname, [SelectOption] options, Bool allowmultiple=true) {
  fset = addFieldset(parent,legend);
  if (allowmultiple) {
    itype = InputCheck;
  } else {
    itype = InputRadio;
    canmark = true;
  }
  for option in options { 
    div = addDivision(fset);
    if (option.value == "") {
      option.value = option.label;
    }
    control = addTextInput(div,itype,iname,option.value);
    if (option.chosen && (allowmultiple || canmark)) {
      module::setAttribute(control,"checked","checked");
      canmark = false;
    }
    label = appendInlineElement(div,FormLabel,option.label);
    doAssociateTandI(label,control);
  }
  return fset;
}

"<argument name='parent'>The parent element</argument>
<argument name='iname'>The name of the checkbox. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='option'>The value and label of the checkbox, and whether it is checked by default</argument>
<summary>Adds a labelled checkbox to a form.</summary>
<prose>Adds a single labelled checkbox to a form. Labelled controls are wrapped in a division, and so the <code>parent</code> may be the form itself.</prose>
<example>opt = SelectOption(\"Confirm action\",\"1\",false);
void(addOption(form,\"confirm\",opt));
/*
&lt;div>&lt;input type='checkbox' name='confirm' value='1' id='Kay1'>
  &lt;label for='Kay1'>Confirm action&lt;/label>&lt;/div>
*/</example>
<related><dataref>SelectOption</dataref></related>
<related><functionref>addLabelledInput</functionref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLabelledTextarea</functionref></related>
<related><functionref>addOptionList</functionref></related>"
public ElementTree addOption(ElementTree parent, String iname, SelectOption option) {
  div = addDivision(parent);
  if (option.value == "") {
    control = addTextInput(div,InputCheck,iname,option.label);
  } else {
    control = addTextInput(div,InputCheck,iname,option.value);
  }
  if (option.chosen) {
    module::setAttribute(control,"checked","checked");
  }
  label = appendInlineElement(div,FormLabel,option.label);
  doAssociateTandI(label,control);
  return div;
}

"<argument name='parent'>The parent element</argument>
<argument name='labeltext'>The text of the label</argument>
<argument name='itype'>The type of text input to add</argument>
<argument name='iname'>The name of the input. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='ivalue'>The initial value of the input. For text and password fields, this may be changed by the user in their web browser (someone attacking your application may of course change any value, regardless of the input type). This value is generally ignored by web browsers for file upload inputs. This value is optional, and defaults to the empty String.</argument>
<argument name='isize'>Optionally, the display size of the input element. Browser rules for determining this size vary - it is an approximate number of characters, but only approximate. Obviously browsers ignore this field for hidden fields and checkbox and radio inputs, and usually for file upload inputs too.</argument>
<summary>Adds a labelled text input</summary>
<prose>Adds a labelled text input to a form. Since the labelled input is enclosed within a division, the parent element may be the form itself. The parameters for this function are identical in use to the parameters of <functionref>addTextInput</functionref> with the same name.</prose>
<related><dataref>TextInputType</dataref></related>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLabelledTextarea</functionref></related>
<related><functionref>addOption</functionref></related>
<related><functionref>addTextInput</functionref></related>"
public ElementTree addLabelledInput(ElementTree parent, String labeltext, TextInputType itype, String iname, String ivalue=createString(0), Int isize=0) {
  div = addDivision(parent);
  label = appendInlineElement(div,FormLabel,labeltext);
  control = addTextInput(div,itype,iname,ivalue,isize);
  doAssociateTandI(label,control);
  return div;
}

"<argument name='parent'>The parent element</argument>
<argument name='name'>The name of the textarea. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='initialtext'>The initial contents of the textarea, if any. Thisparameter is optional, with the empty string as a default.</argument>
<argument name='rows'>The number of rows the fieldset has. This parameter may be omitted for a default of 5.</argument>
<argument name='cols'>The number of columns the fieldset has. This parameter may be omitted for a default of 40. Note that there is no browser-independent relationship between the number of columns in a text area, and the size of a normal text input, so getting them to be consistently the same width is not possible.</argument>
<summary>Adds a textarea</summary>
<prose>Adds a textarea to a form. Remember that textareas may not be added directly to the form - you need to add a block-level element such as <code>fieldset</code> to the form, and then add the textareas to that. Text areas are useful for entering large amounts of user input, or where the user input needs to contain new lines.</prose>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLabelledTextarea</functionref></related>
<related><functionref>addSelectElement</functionref></related>
<related><functionref>addTextInput</functionref></related>"
public ElementTree addTextarea(ElementTree parent, String name, String initialtext=createString(0), Int rows=5, Int cols=40) {
  if (!canContainInline(parent.name)) {
     throw(InvalidNesting("Can't add a form input inside "+parent.name));
  }
  if (rows < 1 || cols < 1) {
     throw(RequiresAttribute("Must have at least one row and column."));
  }
  ta = mkElement("textarea");
  module::setAttribute(ta,"name",name);
  if (initialtext != "") {
    doAddString(ta,initialtext);
  }
  module::setAttribute(ta,"rows",String(rows));
  module::setAttribute(ta,"cols",String(cols));
  pushElement(parent,ta);
  return ta;
}

"<argument name='parent'>The parent element</argument>
<argument name='name'>The name of the textarea. Remember that names starting with \"kaya_\" may be used by the Kaya standard library and should not be used directly by applications.</argument>
<argument name='labeltext'>The text of the label</argument>
<argument name='initialtext'>The initial contents of the textarea, if any. Thisparameter is optional, with the empty string as a default.</argument>
<argument name='rows'>The number of rows the fieldset has. This parameter may be omitted for a default of 5</argument>
<argument name='cols'>The number of columns the fieldset has. This parameter may be omitted for a default of 40. Note that there is no browser-independent relationship between the number of columns in a text area, and the size of a normal text input, so getting them to be consistently the same width is not possible.</argument>
<summary>Adds a labelled textarea</summary>
<prose>Adds a labelled textarea to a form. Since the labelled textarea is enclosed within a division, the parent element may be the form itself. Text areas are useful for entering large amounts of user input, or where the user input needs to contain new lines. The textarea and its label will be separated by a line break.</prose>
<related><functionref>addFieldset</functionref></related>
<related><functionref>addLabelledSelect</functionref></related>
<related><functionref>addLabelledInput</functionref></related>
<related><functionref>addOption</functionref></related>
<related><functionref>addTextarea</functionref></related>"
public ElementTree addLabelledTextarea(ElementTree parent, String name, String labeltext, String initialtext=createString(0), Int rows=5, Int cols=40) {
  div = addDivision(parent);
  label = appendInlineElement(div,FormLabel,labeltext);
  br = addLineBreak(div);
  control = addTextarea(div,name,initialtext,rows,cols);
  doAssociateTandI(label,control);
  return div;
}


/** Part 9: element retrieval **/


"<argument name='parent'>The element</argument>
<summary>Return sub-elements</summary>
<prose>Returns all (non-lazy) child elements of the chosen element. For example, using this on a list will return all list items.</prose>"
public [ElementTree] getChildren(ElementTree parent) {
  els = createArray(size(parent.elements));
  for elem in parent.elements {
    case elem of {
      SubElement(el) -> push(els,el);
      | _ -> ;
    }
  }
  return els;
}

"<summary>Make an anonymous block</summary>
<prose>Make an anonymous division. This is useful for use with <functionref>appendExisting</functionref> and <functionref>prependExisting</functionref>.</prose>
<example>ElementTree mkList([String] texts) {
    div = anonymousBlock();
    list = addList(div,Unordered,size(texts),texts);
    return list;
}
// can now do appendExisting(parent,mkList(texts));</example>
<related><functionref>appendExisting</functionref></related>
<related><functionref>prependExisting</functionref></related>"
public ElementTree anonymousBlock () {
  return mkElement("div");
}

"<argument name='dest'>The element to add to</argument>
<argument name='block'>The element to append</argument>
<summary>Append an existing element</summary>
<prose>Append an existing element as the final child of the specified element.</prose>
<example>p = addParagraph(parent,\"Hello world!\");
// is equivalent to
p = addParagraph(anonymousBlock(),\"Hello world!\");
appendExisting(parent,p);</example>
<related><functionref>anonymousBlock</functionref></related>
<related><functionref>appendGenerator</functionref></related>
<related><functionref>prependExisting</functionref></related>"
public Void appendExisting(ElementTree dest, ElementTree block) {
  if (!canContainException(dest.name,block.name)) {
    if (isBlock(block.name)) {
      if (!canContainBlock(dest.name)) {
	throw(InvalidNesting("Can't add a block-level element here"));
      }
    } else {
      if (!canContainInline(dest.name)) {
	throw(InvalidNesting("Can't add an inline element here"));
      }
    }
  }
  pushElement(dest,block);
}

"<argument name='dest'>The element to add to</argument>
<argument name='block'>The element to prepend</argument>
<summary>Prepend an existing element</summary>
<prose>Prepend an existing element as the first child of the specified element.</prose>
<related><functionref>anonymousBlock</functionref></related>
<related><functionref>appendExisting</functionref></related>
<related><functionref>prependGenerator</functionref></related>"
public Void prependExisting(ElementTree dest, ElementTree block) {
  if (!canContainException(dest.name,block.name)) {
    if (isBlock(block.name)) {
      if (!canContainBlock(dest.name)) {
	throw(InvalidNesting("Can't add a block-level element here"));
      }
    } else {
      if (!canContainInline(dest.name)) {
	throw(InvalidNesting("Can't add an inline element here"));
      }
    }
  }
  unshiftElement(dest,block);
}

"<argument name='dest'>The element to add to</argument>
<argument name='generator'>The generator to append</argument>
<summary>Append an element generator</summary>
<prose>Append an element generator as the final child of the specified element. This can be very useful in templating to ensure that only one distinct part of the template is in memory at once.</prose>
<related><functionref>appendExisting</functionref></related>
<related><functionref>prependGenerator</functionref></related>"
public Void appendGenerator(ElementTree dest, ElementTree() generator) {
  pushGenerator(dest,@generator);
}

"<argument name='dest'>The element to add to</argument>
<argument name='generator'>The generator to prepend</argument>
<summary>Prepend an element generator</summary>
<prose>Prepend an element generator as the first child of the specified element.</prose>
<related><functionref>appendGenerator</functionref></related>
<related><functionref>prependExisting</functionref></related>"
public Void prependGenerator(ElementTree dest, ElementTree() generator) {
  unshiftGenerator(dest,@generator);
}

"<argument name='parentsrc'>The parent block currently containing the element</argument>
<argument name='srcidx'>The position within that block of the element to move</argument>
<argument name='parentdest'>The block to move the element to</argument>
<argument name='destidx'>The position within the new block to move the element to.</argument>
<summary>Move a block</summary>
<prose>Move a block from one place to another. All children of the block being moved will move with it.</prose>"
public Void moveBlock(ElementTree parentsrc, Int srcidx, ElementTree parentdest, Int destidx) {
  if (srcidx >= size(parentsrc.elements)) {
    throw(OutOfBounds);
  }
  case parentsrc.elements[srcidx] of {
    CData(cd) -> inline = true; cdata = true; cname = "";
    | SubElement(el) -> inline = !isBlock(el.name); cdata = false; cname = el.name;
  }
  if (!canContainException(parentdest.name,cname)) {
    if (!inline) {
      if (!canContainBlock(parentdest.name)) {
	throw(InvalidNesting("Can't add a block-level element here"));
      }
    } else {
      if (!canContainInline(parentdest.name) && !(cdata && containsCDataOnly(parentdest.name))) {
	throw(InvalidNesting("Can't add an inline element here"));
      }
    }
  }
  case parentsrc.elements[srcidx] of {
    SubElement(el) -> addElementAt(parentdest,el,destidx);
    | CData(cd) -> addDataAt(parentdest,cd,destidx);
  }
}

private ElementTree mkElement(String name, Int expected=0) {
  return newElement(name,expected);
}


/** Part 10: Conversion from Strings */

"<argument name='location'>The location in the document to append the converted String</argument>
<argument name='input'>The String to convert</argument>
<argument name='safety'>The allowed elements and attributes</argument>
<argument name='doctype'>The document type of the input string. This is independent of the document type of the document and controls certain aspects of parsing - for example, if the doctype is <code>HTML4Strict</code>, the end tags for <code>p</code> or <code>td</code> elements may be omitted, and the code will be case-insensitive.</argument>
<summary>Convert a String to HTML</summary>
<prose>Convert a String to HTML. An Exception will be thrown if the parser cannot construct an unambiguous tree from the string, and it is considerably less forgiving than the parsers in most web browsers. (On the other hand, it's a bit more forgiving in places than a strict XML parser, especially if the <code>HTML4Strict</code> doctype is used). As usual, parsing with XHTML strictness will be quicker.</prose>
<prose>Read the <dataref>WhiteList</dataref> documentation for important information on safety when converting these Strings</prose>
<prose>If you need to read poor-quality HTML, you can use the <code>TagSoup</code> setting for the doctype parameter, but this will be somewhat slower - if you're writing the HTML yourself it's much more efficient to write good quality code in the first place. This parsing model rarely throws an Exception (though it is still possible on very poor code)</prose>
<example>str = \"&lt;p>This is a &lt;strong>crucial&lt;/strong> paragraph&lt;/p>\";
readFromString(parent,str,AllElements(Safe),HTML4Strict);</example>
<related><dataref>Doctype</dataref></related>
<related><dataref>WhiteList</dataref></related>
<related><functionref>readFromTemplate</functionref></related>
<related><functionref>string</functionref></related>"
public Void readFromString(ElementTree location, String input, WhiteList safety, Doctype doctype) {
  wl = whitelistHTML(safety);
  case doctype of {
    TagSoup -> ie = soupClosings; csen = false; tsoup = true; ae = isAlwaysEmpty;
    | HTML4Strict -> ie = impliedClosings; csen = false; tsoup = false; ae = isAlwaysEmpty;
    | XHTML1Strict -> ie = Dict::new(3,strHash); csen = true; tsoup = false; ae = createArray(1); // no implicit closing or ambiguous empty elements in XHTML
  }
  tmpbase = elementTree(input,"tmpbase",wl,ie,ae,Dict::new(2,strHash),csen,tsoup);
  els = getElements(tmpbase);
  for el in els {
    case el of {
      SubElement(sel) -> pushElement(location,sel);
      | CData(scd) -> pushData(location,scd);
    }
  }
}


"<argument name='location'>The location in the document to append the converted String</argument>
<argument name='input'>The String to convert</argument>
<argument name='safety'>The allowed elements and attributes</argument>
<argument name='doctype'>The document type of the input string. This is independent of the document type of the document and controls certain aspects of parsing - for example, if the doctype is <code>HTML4Strict</code>, the end tags for <code>p</code> or <code>td</code> elements may be omitted, and the code will be case-insensitive. As usual, parsing with XHTML strictness will be quicker. The <code>TagSoup</code> option is equivalent to <code>HTML4Strict</code> - templates should be good quality!</argument>
<argument name='templates'>A dictionary of templating functions. The key is the name of the templating tag, and the value is the function to call upon seeing that tag (which will be passed the templating tags attributes as a list of pairs).</argument>
<summary>Convert a HTML template to HTML</summary>
<prose>Convert a HTML template String to HTML. An Exception will be thrown if the parser cannot construct an unambiguous tree from the string, and it is considerably less forgiving than the parsers in most web browsers. (On the other hand, it's a bit more forgiving in places than a strict XML parser, especially if the <code>HTML4Strict</code> doctype is used).</prose>
<prose>A HTML template String contains empty tags with user-defined names. These tags are then replaced with the output of the function defined for that tag in the <code>templates</code> dictionary.</prose>
<prose>Read the <dataref>WhiteList</dataref> documentation for important information on safety when converting these Strings, and bear in mind that allowing user-supplied data to call templating functions is only as secure as the allowed templating functions.</prose>
<related><dataref>Doctype</dataref></related>
<related><dataref>WhiteList</dataref></related>
<related><functionref>readFromString</functionref></related>
<related><functionref>string</functionref></related>"
public Void readFromTemplate(ElementTree location, String input, WhiteList safety, Doctype doctype, Dict<String,ElementTree([(String,String)])> templates) {
  wl = whitelistHTML(safety);
  case doctype of {
    TagSoup ->  ie = impliedClosings; csen = false; ae = isAlwaysEmpty; // templates should be good quality!
    | HTML4Strict -> ie = impliedClosings; csen = false; ae = isAlwaysEmpty;
    | XHTML1Strict -> ie = Dict::new(3,strHash); csen = true; ae = createArray(1); // no implicit closing in XHTML
  }
  tmpbase = elementTree(input,"tmpbase",wl,ie,ae,templates,csen,false);
  els = getElements(tmpbase);
  for el in els {
    case el of {
      SubElement(sel) -> pushElement(location,sel);
      | CData(scd) -> pushData(location,scd);
      // allow top-level templates
      | SubTree(g) -> pushGenerator(location,g);
    }
  }
}



Dict<String,[String]> impliedClosings() {
  ics = Dict::new(17,strHash);
  add(ics,"td",["td","/tr","tr","/table","/tbody","/tfoot","/thead","tbody","tfoot"]);
  add(ics,"tr",["/tr","tr","/table","/tbody","/tfoot","/thead","tbody","tfoot"]);
  add(ics,"thead",["/table","tbody","tfoot"]);
  add(ics,"tbody",["/table","tbody"]);
  add(ics,"tfoot",["/table","tbody"]);
  add(ics,"li",["li","/ul","/ol"]);
  add(ics,"p",["address","blockquote","div","h1","h2","h3","h4","h5","h6","hr","p","pre","ul","ol","dl","table","form","fieldset","/address","/blockquote","/div","/form","/fieldset"]);
  add(ics,"dd",["dd","dt","/dl"]);
  add(ics,"dt",["dd","dt","/dl"]);
  return ics;
}

Dict<String,[String]> soupClosings() {
  ics = Dict::new(17,strHash);
  add(ics,"td",["td","/tr","tr","/table","/tbody","/tfoot","/thead","tbody","tfoot"]);
  add(ics,"tr",["/tr","tr","/table","/tbody","/tfoot","/thead","tbody","tfoot"]);
  add(ics,"thead",["/table","tbody","tfoot"]);
  add(ics,"tbody",["/table","tbody"]);
  add(ics,"tfoot",["/table","tbody"]);
  add(ics,"li",["li","/ul","/ol"]);
  add(ics,"ol",["ol","ul","/ul","p","/p","div","/div","h1","/h1","h2","/h2","h3","/h3","h4","/h4","h5","/h5","h6","/h6"]);
  add(ics,"ul",["ol","ul","/ol","p","/p","div","/div","h1","/h1","h2","/h2","h3","/h3","h4","/h4","h5","/h5","h6","/h6"]);
  add(ics,"p",["address","blockquote","div","h1","h2","h3","h4","h5","h6","hr","p","pre","ul","ol","dl","table","form","fieldset","/address","/blockquote","/div","/form","/fieldset","/ul","/ol"]);
  add(ics,"dd",["dd","dt","/dl"]);
  add(ics,"dt",["dd","dt","/dl"]);
  return ics;
}


HashSet<String> tagFormats() {
  // defaults to not breaking
  //  full = ["body","blockquote","ul","ol","hr","table","thead","tfoot","tbody","tr","form","fieldset","select","optgroup"];
  //  outer = ["h1","h2","h3","h4","h5","h6","p","address","div","pre","li","br","caption","td","th","legend","option","textarea"];
  tags = ["body","blockquote","ul","ol","hr","table","thead","tfoot","tbody","tr","form","fieldset","select","optgroup","h1","h2","h3","h4","h5","h6","p","address","div","pre","li","br","caption","td","th","legend","option","textarea"];
  ed = newHashSet(47,strHash);
  for i in [0..size(tags)-1] {
    add(ed,tags[i]);
  }
  return ed;
}

[String] genericAttribs = ["class","id","lang","dir","title"];

ConversionSafety getSafety(WhiteList w) {
  case w of {
    InlineOnly(s) -> return s;
    | AllElements(s) -> return s;
  }
}

Dict<String,[String]> whitelistHTML(WhiteList wconf) {
  case wconf of {
    CustomWhitelist(wl) -> return wl;
    | _ ->; // continue;
  }
  wl = Dict::new(79,strHash);
  if (wconf == Unchecked) {
    add(wl,"*",["*"]); // whitelist everything
    return wl;
  } else if (wconf == UltraSafe) {
    add(wl,createString(0),createArray(1)); // blacklist everything, act as a tag stripper
    return wl;
  }

  safeinline = ["b","i","em","strong","u","ins","del","code","kbd","samp","var","cite","dfn","abbr","big","small","sup","sub","q","span","br"];
  case getSafety(wconf) of {
    Safe -> ;
    | Unsafe -> concat(safeinline,["a","img"]);
    | _ -> concat(safeinline,["a","img","label","input","select","option","optgroup","textarea"]);
  }

  for name in safeinline {
    attribs = genericAttribs();
    case getSafety(wconf) of {
      Safe -> ;
      | _ -> push(attribs,"style");
    }
    case name of {
      "a" -> concat(attribs,["href"]); // only in Unsafe or worse as above
      | "img" -> concat(attribs,["src","alt","width","height"]); // likewise
      | "label" -> concat(attribs,["for"]);
      | "input" -> concat(attribs,["name","value","type","size"]);
      | "select" -> concat(attribs,["name","size","multiple"]);
      | "option" -> concat(attribs,["label","value"]);
      | "optgroup" -> concat(attribs,["label"]);
      | _ -> ;
    }
    add(wl,name,attribs);
  }
  case wconf of {
    InlineOnly(s) -> return wl;
    | _ -> ;
  }

  safeblock = ["p","pre","div","ol","li","ul","dl","dd","dt","h1","h2","h3","h4","h5","h6","address","blockquote","table","tr","td","th","tbody","tfoot","thead","caption","hr","col","colgroup"];
  case getSafety(wconf) of {
    // fieldset and legend aren't unsafe, just pointless without form.
    VeryUnsafe -> concat(safeblock,["form","fieldset","legend"]);
    | _ -> ;
  } 

  for name in safeblock {
    attribs = genericAttribs();
    case getSafety(wconf) of {
      Safe -> ;
      | _ -> concat(attribs,["style"]);
    }
    case name of {
      "form" -> concat(attribs,["action","method"]);
      | "td" -> concat(attribs,["colspan","rowspan","headers"]);
      | "th" -> concat(attribs,["colspan","rowspan","scope"]);
      | "table" -> concat(attribs,["summary"]);
      | _ -> ;
    }
    add(wl,name,attribs);
  }
  return wl;
}