/*
    mmorph, MULTEXT morphology tool
    Version 2.3, October 1995
    Copyright (c) 1994,1995 ISSCO/SUISSETRA, Geneva, Switzerland
    Dominique Petitpierre, <petitp@divsun.unige.ch>
*/
/*
    database.c

    handle the database storing and access modules

    Dominique Petitpierre,  Summer 94

*/

#include "user.h"
#include <sys/types.h>
#ifdef HAVE_LIMITS_H
#include <limits.h>
#endif

/*  redefine UINT_MAX for /usr/ucb/cc and SparcWorks compiler with option -Xs */
#if defined(sun) && defined(__unix) && defined(sparc) && ! defined(__STDC__)
#ifdef UINT_MAX
#undef UINT_MAX
#define UINT_MAX                 0xFFFFFFFF
#endif
#endif

#include <db_185.h>
#include <fcntl.h>
#include <netinet/in.h>
#include "mymalloc.h"

#ifdef FLOCK
#ifndef STDC_HEADERS
extern int  flock();

#endif
#endif

#define RECORD_CHUNK_SIZE    128

#define CHECK_RECORD_SIZE(size,incr,max) \
	    if (((size) += (incr)) > (max)) \
		fatal_error("error in database: inconsistant record");


extern DB  *dbopen();

static DB  *db_forms;
t_db_operation db_operation;

static u_record_data db_record;	/* storage for copying records from database */
static size_t db_record_size;
static size_t max_db_record_size;	/* allocated size of db_record */

/*
    open a hash database according to the operation to perform
*/

static DB  *
db_init(db_file_name, db_operation)
char       *db_file_name;
t_db_operation db_operation;

{
    char       *error_msg;
    DB         *db;
    HASHINFO    hashinfo;

    max_db_record_size = RECORD_CHUNK_SIZE;
    MY_STRALLOC(db_record.string, max_db_record_size);
    db_record_size = 0;
    hashinfo.bsize = 1024;	/* better for disk access, power of 2 */
    hashinfo.ffactor = 25;	/* average # of records per block of bsize */
    hashinfo.nelem = 1;	/* don't know how big */
    hashinfo.cachesize = 64 * 1024;	/* at least 6*bsize */
    hashinfo.hash = NULL;	/* default hash */
    hashinfo.lorder = 4321;	/* big endian */

    /* [Dec1996 - anieto@crea.rae.es] */	
    #if defined(linux)
    hashinfo.lorder = 1234;     /* little endian for linux on Intel x86 */
    #endif

    if (db_operation & Create) {
	db = dbopen(db_file_name,
		    O_CREAT | O_RDWR | O_TRUNC | O_EXCL,
		    DB_MODE, DB_HASH, &hashinfo);
	error_msg = "for creation, or file exists already";
    }
    else if (db_operation & Update) {
	/* TODO: find out if exclusive opening can be enforced */
	db = dbopen(db_file_name, O_RDWR, DB_MODE, DB_HASH,
		    &hashinfo);
	error_msg = "for updating,\nor file is in use";
    }
    else {	/* db_operation == Lookup */
	db = dbopen(db_file_name, O_RDONLY, DB_MODE,
		    DB_HASH, &hashinfo);
	error_msg = "for lookup";
    }
    if (db == NULL)
	fatal_error("cannot open file \"%s\" %s (errno=%d)",
		    db_file_name, error_msg, errno);
#ifdef FLOCK
    /*
       timid attempt to limit risks of corruption with locks. Does not work
       across NFS.
    */
    if (db_operation & (Create | Update)) {
	/* exclusive lock no one else is or will use the file */
	if (flock((db->fd) (db), LOCK_EX | LOCK_NB) < 0)
	    fatal_error("file %s is already in use (errno=%d)",
			db_file_name, errno);
    }
    else {	/* shared lock: only other lookups can be done */
	if (flock((db->fd) (db), LOCK_SH | LOCK_NB) < 0)
	    fatal_error("file %s is beeing updated (errno=%d)",
			db_file_name, errno);
    }
#endif
#ifdef LOCKF
    /*
       Attempt to limit risks of corruption with locks. should work across
       NFS. Does not work because lockf does not let you have read locks
       (shared locks)
    */
    if (db_operation & (Create | Update)) {
	/* exclusive lock no one else is or will use the file */
	if (lockf((db->fd) (db), F_TLOCK, 0L) < 0)
	    fatal_error("file %s is already in use (errno=%d)",
			db_file_name, errno);
    }
    else {	/* shared lock: only other lookups can be done */
	if (lockf((db->fd) (db), F_TEST, 0L) < 0)
	    fatal_error("file %s is beeing updated (errno=%d)",
			db_file_name, errno);
    }
#endif
    return (db);
}

/* wrapper to open db_forms */

void
db_forms_init(db_file_name, db_operation)
char       *db_file_name;
t_db_operation db_operation;
{
    db_forms = db_init(db_file_name, db_operation);
}


/*
    creates an external representation of the tfs,
    append it at the end of record->data and update record->size
    suitable to be stored in the database.
*/
static void
make_db_record(surface_lex, base_lex, tfs_index)
t_letter   *surface_lex;
t_letter   *base_lex;
int         tfs_index;

{
    u_record_data data;
    int         entry_length;
    int         base_lex_length;
    size_t      initial_size;

    /*
       TODO: squeeze into as little bits as possible. Use XDR to have an
       external representation
    */

    if (strcmp((char *) surface_lex, (char *) base_lex) == 0)
	base_lex_length = 0;	/* beware: this prevents zero length words */
    else
	base_lex_length = strlen((char *) base_lex);
    if (base_lex_length > UCHAR_MAX) {
	print_warning("base form string truncated to %d characters:\n\"",
		      UCHAR_MAX);
	print_string(logfile, base_lex);
	print_log("\"\n");
	base_lex_length = UCHAR_MAX;
    }
    entry_length = base_lex_length + sizeof(t_card) + 1;
    initial_size = db_record_size;	/* save initial value */
    db_record_size += entry_length;
    if (db_record_size > max_db_record_size) {
	max_db_record_size = db_record_size + RECORD_CHUNK_SIZE;
	MY_REALLOC(db_record.string, max_db_record_size, char);
    }
    data.string = db_record.string + initial_size;
    *(data.string++) = (char) base_lex_length;
    (void) strcpy((char *) data.string, (char *) base_lex);
    data.string += base_lex_length;
    T_CARD_TO_STRING(tfs_index, data.string);
}


static void
db_store_entry(db, surface_lex, surface_lex_length, base_lex, tfs_index)
DB         *db;
t_letter   *surface_lex;
int         surface_lex_length;
t_letter   *base_lex;
int         tfs_index;

{
    DBT         key;
    DBT         record;
    int         db_status;
    int         error;

    key.size = (size_t) surface_lex_length;
    key.data = (void *) surface_lex;
    db_status = (db->get) (db, &key, &record, 0);
    if (db_status < 0) {	/* is surface_lex already there? */
	error = errno;	/* save value */
	print_log("for search key: \"");
	print_string(logfile, surface_lex);
	fatal_error("error while reading database (errno=%d=%s)",
		    error,
		    (error < sys_nerr ? sys_errlist[error] : "?"));
    }
    if (db_status > 0) {	/* new lex */
	db_record_size = 0;
	make_db_record(surface_lex, base_lex, tfs_index);
	record.data = (void *) db_record.string;
	record.size = db_record_size;
	db_status = (db->put) (db, &key, &record, 0);
    }
    else {	/* lex exists in db */
	db_record_size = record.size;
	if (db_record_size > max_db_record_size) {
	    max_db_record_size = db_record_size + RECORD_CHUNK_SIZE;
	    MY_FREE(db_record.string);
	    MY_STRALLOC(db_record.string, max_db_record_size);
	}
	(void) memcpy((T_PTR) db_record.string, (T_PTR) record.data,
		      db_record_size);
	make_db_record(surface_lex, base_lex, tfs_index);
	record.data = (void *) db_record.string;
	record.size = db_record_size;
	db_status = (db->put) (db, &key, &record, 0);
    }
    if (db_status < 0) {
	error = errno;	/* save value */
	print_log("for search key: \"");
	print_string(logfile, surface_lex);
	fatal_error("\" error while writing database (errno=%d=%s)",
		    error,
		    (error < sys_nerr ? sys_errlist[error] : "?"));
    }
}

/* wrapper to db_store_entry */
void
db_store_form(surface_lex, surface_lex_length, base_lex, tfs_index)
t_letter   *surface_lex;
int         surface_lex_length;
t_letter   *base_lex;
int         tfs_index;

{
    db_store_entry(db_forms, surface_lex, surface_lex_length, base_lex,
		   tfs_index);
}

static void
db_lookup_entry(db, surface_lex, record)
DB         *db;
t_letter   *surface_lex;
DBT        *record;

{
    DBT         key;
    int         db_status;
    int         error;

    key.size = (size_t) strlen((char *) surface_lex);
    key.data = (void *) surface_lex;
    db_status = (db->get) (db, &key, record, 0);
    if (db_status < 0) {	/* is surface_lex there? */
	error = errno;	/* save value */
	print_log("for search key: \"");
	print_string(logfile, surface_lex);
	fatal_error("\" error while reading database (errno=%d=%s)",
		    error,
		    (error < sys_nerr ? sys_errlist[error] : "?"));

    }
    if (db_status > 0) {
	record->data = NULL;
	record->size = 0;
    }
}


static void
print_projection(segment_id, record, surface_lex)
t_segment_id segment_id;
DBT        *record;
t_letter   *surface_lex;

{
    u_record_data data;
    int         size;
    int         length;
    unsigned char base_lex_length;
    t_card      tfs_index;

    size = 0;
    data.string = (char *) record->data;
    while (size < record->size) {	/* for each sub record */
	base_lex_length = (unsigned char) *data.string;
	/* check that the string length is reasonable */
	length = base_lex_length + 1;
	CHECK_RECORD_SIZE(size, length + sizeof(t_card), record->size);
	if (want_segment_id)
	    print_out("%d\t\"", segment_id);
	else
	    print_out("\"");
	print_string(outfile, surface_lex);
	print_out("\" = \"");
	if (base_lex_length >= 1)
	    print_string_l(outfile, (t_letter *) data.string + 1,
			   (int) base_lex_length);
	else
	    print_string(outfile, surface_lex);
	print_out("\" ");
	data.string += length;
	STRING_TO_T_CARD(data.string, tfs_index);
	print_tfs_proj(&tfs_table[tfs_index]);
	data.string += sizeof(t_card);
    }
}


static void
print_projection_tbl(record, surface_lex)
DBT        *record;
t_letter   *surface_lex;

{
    u_record_data data;
    int         size;
    int         length;
    unsigned char base_lex_length;
    t_card      tfs_index;

    size = 0;
    data.string = (char *) record->data;
    while (size < record->size) {	/* for each sub record */
	if (size != 0)
	    print_out("%c", SUBFIELD_SEP);
	base_lex_length = (unsigned char) *data.string;
	/* check that the string length is reasonable */
	length = base_lex_length + 1;
	CHECK_RECORD_SIZE(size, length + sizeof(t_card), record->size);
	if (base_lex_length >= 1) {
	    print_string_l(outfile, (t_letter *) data.string + 1,
			   (int) base_lex_length);
	    print_out("%c", SUBSUBFIELD_SEP);
	}
	else if (surface_lex == NULL)
	    print_out("=%c", SUBSUBFIELD_SEP);
	else {
	    print_string(outfile, surface_lex);
	    print_out("%c", SUBSUBFIELD_SEP);
	}
	data.string += length;
	STRING_TO_T_CARD(data.string, tfs_index);
	print_tfs_proj_tbl(&tfs_table[tfs_index]);
	data.string += sizeof(t_card);
    }
}

t_boolean
db_forms_lookup(segment_id, surface_lex, tfs)
t_segment_id segment_id;
t_letter   *surface_lex;
s_tfs      *tfs;

{
    DBT         record;

    db_lookup_entry(db_forms, surface_lex, &record);
    /* not implemented: unify the tfs */
    if (record.data == NULL)
	return (FALSE);
    else {
	print_projection(segment_id, &record, surface_lex);
	return (TRUE);
    }
}

t_boolean
db_forms_lookup_tbl(surface_lex, append, decapitalized)
t_letter   *surface_lex;
t_boolean   append;
t_boolean   decapitalized;

{
    DBT         record;

    db_lookup_entry(db_forms, surface_lex, &record);
    /* not implemented: unify the tfs */
    if (record.data == NULL)
	return (FALSE);
    else {
	if (append)
	    print_out("%c", SUBFIELD_SEP);
	if (decapitalized)
	    print_projection_tbl(&record, surface_lex);
	else
	    print_projection_tbl(&record, (t_letter *) NULL);
	return (TRUE);
    }
}

/* wrapper for db->close */
void
db_forms_close()
{
#ifdef FLOCK
/*  sync and unlock not necessary: db->close takes care of that
    if ((db_forms->sync)(db_forms, 0) < 0)
        fatal_error("cannot flush database (errno=%d)",errno);
    if(flock((db_forms->fd)(db_forms), LOCK_UN|LOCK_NB) < 0)
        fatal_error("cannot unlock database (errno=%d)",errno);
*/
#endif
#ifdef LOCKF
    /* should test if there is still a lock */
    if ((db_forms->sync) (db_forms, 0) < 0)
	fatal_error("cannot flush database (errno=%d)", errno);
    lseek((db_forms->fd) (db_forms), (off_t) 0L, SEEK_SET);
    if (lockf((db_forms->fd) (db_forms), F_ULOCK, 0L) < 0)
	fatal_error("cannot unlock database (errno=%d)", errno);
#endif
    if ((db_forms->close) (db_forms) < 0)
	fatal_error("cannot close database (errno=%d=%s)", errno,
		    (errno < sys_nerr ? sys_errlist[errno] : "?"));
}

/*
    After all writing is completed, let other processes use
    the database for reading. (db_operation is Create or Update)
 */
void
db_forms_complete()
{
    if ((db_forms->sync) (db_forms, 0) < 0)
	fatal_error("cannot flush database (errno=%d=%s)", errno,
		    (errno < sys_nerr ? sys_errlist[errno] : "?"));
#ifdef FLOCK
    if (flock((db_forms->fd) (db_forms), LOCK_SH | LOCK_NB) < 0)
	fatal_error("cannot change lock on database (errno=%d)",
		    errno);
#endif
#ifdef LOCKF
    lseek((db_forms->fd) (db_forms), (off_t) 0L, SEEK_SET);
    if (lockf((db_forms->fd) (db_forms), F_ULOCK, 0L) < 0)
	fatal_error("cannot change lock on database (errno=%d)",
		    errno);
#endif
}

void
db_forms_dump()
{
    DBT         key;
    DBT         record;
    int         db_status;
    int         sum;
    int         sum_keys;
    int         key_card;
    int         size;
    int         max_key;

    sum = 0;
    key_card = 0;
    sum_keys = 0;
    max_key = 0;
    db_status = (db_forms->seq) (db_forms, &key, &record, R_FIRST);
    while (db_status == 0) {
	(void) memcpy((T_PTR) concatenation, (T_PTR) key.data, key.size);
	concatenation[key.size] = NUL_LETTER;
	/* TODO: count the internal records */
	print_projection((t_segment_id) key_card, &record, concatenation);
	size = key.size + record.size;
	max_key = MAX(max_key, size);
	sum += size;
	key_card++;
	sum_keys += key.size;
	db_status = (db_forms->seq) (db_forms, &key, &record, R_NEXT);
    }
    if (db_status < 0)
	fatal_error("error while reading next key in database (errno=%d=%s)",
		    errno, (errno < sys_nerr ? sys_errlist[errno] : "?"));
    if (debug & DEBUG_STAT) {
	max_key += 2 * sizeof(record.size);
	size = sizeof(record.size) * key_card;
	sum_keys += size;
	sum += 2 * size;
	print_log("%d keys, sum keys = %d, sum records = %d, total = %d\n",
		  key_card, sum_keys, sum - sum_keys, sum);
	print_log("%s %.1f, %s %.1f, %s %.1f, %s %d\n",
		  "average key =", (float) (sum_keys) / key_card,
		  "avg record =", (float) (sum - sum_keys) / key_card,
		  "avg entry =", (float) (sum) / key_card,
		  "max entry size =", max_key);
    }
}