/** -*-C-*-ish
    Kaya standard library
    Copyright (C) 2004, 2005 Edwin Brady

    This file is distributed under the terms of the GNU Lesser General
    Public Licence. See COPYING for licence.
*/
"<summary>HTTP requests</summary>
<prose>This module contains low-level functions for constructing and sending HTTP and HTTPS requests. If you are using HTTPS, then you should read the documentation for the <moduleref>TLS</moduleref> module about SSL certificates.</prose>"
module HTTP;

import WebCommon;
import Array;
import Net;
import Regex;
import Tuples;
import Strings;
import Builtins;

"<summary>The HTTP version</summary>
<prose>The HTTP version to use in communication. HTTP 1.1 (<code>HTTP11</code>) has advantages but may be harder to parse the responses of.</prose>"
public data HTTPversion = HTTP10 | HTTP11;

"<summary>Bad username for HTTP authentication</summary>
<prose>Usernames for HTTP authentication may not contain a ':'</prose>"
Exception BadUsername();

"<summary>Not a URL</summary>
<prose>The String for conversion did not appear to be a valid URL</prose>"
Exception NotAURL();


"<summary>URL information</summary>
<prose>This data type stores URL information.</prose>
<list>
<item><code>server</code> is the host to be contacted. Generally this should be the host name although for some sites the IP address may be used instead.</item>
<item><code>port</code> is the TCP port to use. Normally, 80 is used for HTTP and 443 for HTTPS</item>
<item><code>localpart</code> is the remainder of the URL (which must include the initial '/')</item>
<item><code>secure</code> is false for HTTP and true for HTTPS (note that it is possible for a HTTP server to run on port 443, so no assumptions will be made about this setting based on the port number)</item>
</list>
<related><functionref>getURL</functionref></related>
<related><functionref>parseURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public data HTTPURL(String server, Int port, String localpart, Bool secure);

"<argument name='url'>The String representing the URL</argument>
<summary>Convert a String to a HTTP URL</summary>
<prose>Parse a String solely containing an absolute URL into a <dataref>HTTPURL</dataref>. The String must begin with the protocol http:// or https://</prose>
<related><functionref>getURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public HTTPURL parseURL(String url) {
  re = compile("^https?://([^/]+)(/.*|$)",createArray(1));
  result = match(re,url);
  case result of {
    noMatch -> throw(NotAURL);
    | matches([_,server,local],_,_) -> secure = quickMatch("^https://",url);
    if (quickMatch(":",server)) {
      hdata = split(":",server);
      server = hdata[0];
      port = Int(hdata[1]);
    } else {
      if (secure) {
	port = 443;
      } else {
	port = 80;
      }
    }
    if (quickMatch("#",local)) {
      ldata = split("#",local);
      local = ldata[0];
    }
    return HTTPURL(server,port,local,secure);
    | _ -> throw(NotAURL); // shouldn't happen
  }
}

"<argument name='url'>A <dataref>HTTPURL</dataref>.</argument>
<argument name='headers'>A list of key/value pairs of extra HTTP headers (optional, defaulting to the empty list)</argument>
<argument name='version'>The HTTP version</argument>
<argument name='certfiles'>A list of SSL certificate files, each containing one or more PEM encoded certificates of trusted Certification Authorities, used for HTTPS connections. This list may be empty (the default), but this is much less secure. For HTTP connections, this parameter is ignored.</argument>
<summary>Retrieve a URL by HTTP.</summary>
<prose>Retrieve a URL by HTTP. The String returned will contain the HTTP response headers and response body exactly as sent by the server. Because Strings may not contain null bytes, this function is not suitable for retrieving binary data.</prose>
<related><functionref>basicAuthHeader</functionref></related>
<related><functionref>parseURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public String getURL(HTTPURL url, 
		     [(String,String)] headers = createArray(1),
		     HTTPversion version = HTTP10,
		     [String] certfiles=[]) 
{
    server = url.server;
    dir = url.localpart;
    port = url.port;
    secure = url.secure;

    h = connect(TCP, server, port, secure, certfiles);
    getURL(h,url,headers,version);
    urldata = recv(h);
    try {
      closeConnection(h);
    } catch(CloseError) {
      // ignore this error
    }
    return urldata;
}


"<argument name='h'>An established network connection.</argument>
<argument name='url'>A <dataref>HTTPURL</dataref>.</argument>
<argument name='headers'>A list of key/value pairs of extra HTTP headers (optional, defaulting to the empty list)</argument>
<argument name='version'>The HTTP version</argument>
<summary>Retrieve a URL by HTTP using an existing connection.</summary>
<prose>Retrieve a URL by HTTP using an existing connection. The connection will then be ready to receive response data.</prose>
<related><functionref>basicAuthHeader</functionref></related>
<related><functionref>parseURL</functionref></related>
<related><functionref>getURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public Void getURL(NetHandle h,
		   HTTPURL url, 
		   [(String,String)] headers = createArray(1),
		   HTTPversion version = HTTP10) {
    server = url.server;
    dir = url.localpart;
    port = url.port;
    
    case version of {
      HTTP10 -> hver = "HTTP/1.0";
      | HTTP11 -> hver = "HTTP/1.1";
    }
    send(h,"GET "+dir+" "+hver+"\nConnection: close\n");
    for x in headers {
	hd = x.fst;
	hv = x.snd;
	send(h,hd+": "+hv+"\n");
    }

    send(h,"Host:"+server+":"+port+"\n\n");

}



"<argument name='post'>The data to post, encoded as application/x-www-form-urlencoded data</argument>
<argument name='url'>A <dataref>HTTPURL</dataref>.</argument>
<argument name='headers'>A list of key/value pairs of extra HTTP headers (optional, defaulting to the empty list)</argument>
<argument name='version'>The HTTP version</argument>
<argument name='certfiles'>A list of SSL certificate files, each containing one or more PEM encoded certificates of trusted Certification Authorities, used for HTTPS connections. This list may be empty (the default), but this is much less secure. For HTTP connections, this parameter is ignored.</argument>
<summary>Post data to a URL by HTTP.</summary>
<prose>Post data to a URL by HTTP. The String returned will contain the HTTP response headers and response body exactly as sent by the server. Because Strings may not contain null bytes, this function is not suitable for requests that may return binary data.</prose>
<related><functionref>basicAuthHeader</functionref></related>
<related><functionref>getURL</functionref></related>
<related><functionref>parseURL</functionref></related>"
public String postURL(String post, HTTPURL url,
		      [(String,String)] headers = createArray(1),
		      HTTPversion version = HTTP10,
		      [String] certfiles = []) 
{
    server = url.server;
    dir = url.localpart;
    port = url.port;
    secure = url.secure;

    h = connect(TCP, server, port, secure, certfiles);
    postURL(h,post,url,headers,version);
    urldata = recv(h);
    try {
      closeConnection(h);
    } catch(CloseError) {
      // ignore this error
    }
    return urldata;
}

"<argument name='h'>An established network connection.</argument>
<argument name='post'>The data to post, encoded as application/x-www-form-urlencoded data</argument>
<argument name='url'>A <dataref>HTTPURL</dataref>.</argument>
<argument name='headers'>A list of key/value pairs of extra HTTP headers (optional, defaulting to the empty list)</argument>
<argument name='version'>The HTTP version</argument>
<summary>Post data to a URL by HTTP using an existing connection.</summary>
<prose>Post data to a URL by HTTP.  The connection will then be ready to receive response data.</prose>
<related><functionref>basicAuthHeader</functionref></related>
<related><functionref>getURL</functionref></related>
<related><functionref>parseURL</functionref></related>"
public Void postURL(NetHandle h, String post, HTTPURL url,
		    [(String,String)] headers = createArray(1),
		    HTTPversion version = HTTP10) {

    server = url.server;
    dir = url.localpart;
    port = url.port;

    case version of {
      HTTP10 -> hver = "HTTP/1.0";
      | HTTP11 -> hver = "HTTP/1.1";
    }
    send(h,"POST "+dir+" "+hver+"\nConnection: close\n");
    for x in headers {
	hd = x.fst;
	hv = x.snd;
	send(h,hd+": "+hv+"\n");
    }
    len = length(post);
    send(h,"Content-length: "+len+"\n");
    send(h,"Content-type: application/x-www-form-urlencoded\n");
    send(h,"Host:"+server+":"+port+"\n\n");
    send(h,post+"\n");
}

"<argument name='user'>The username</argument>
<argument name='pwd'>The password</argument>
<summary>Create a Basic authentication header</summary>
<prose>Creates an authentication header suitable for HTTP Basic Auth. The username may not contain a ':' character. Remember that this form of authentication is very insecure, and so should only be used over secure connections.</prose>
<related><functionref>getURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public (String,String) basicAuthHeader(String user, String pwd) {
  if (quickMatch(":",user)) {
    throw(BadUsername);
  }
  userpass = user+":"+pwd;
  upenc = base64Encode(userpass);
  return ("Authorization","Basic "+upenc);
}

"<argument name='cookies'>A list of pairs (in name,value order) of the cookies to send to the server with this request</argument>
<summary>Generate a cookie header</summary>
<prose>Generates a HTTP header for sending cookies.</prose>
<related><functionref>getURL</functionref></related>
<related><functionref>postURL</functionref></related>"
public (String,String) cookieHeader([(String,String)] cookies) {
  vals = [];
  for cookie in cookies {
    push(vals,urlEncode(cookie.fst)+"="+urlEncode(cookie.snd));
  }
  return ("Cookie",join(vals,";"));
}