/* Distributed Checksum Clearinghouse
 *
 * database cleaner
 *
 * Copyright (c) 2005 by Rhyolite Software
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND RHYOLITE SOFTWARE DISCLAIMS ALL
 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL RHYOLITE SOFTWARE
 * BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES
 * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
 * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
 * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
 * SOFTWARE.
 *
 * Rhyolite Software DCC 1.2.74-1.132 $Revision$
 */


#include "srvr_defs.h"
#include "dcc_ck.h"
#include <syslog.h>
#include <signal.h>

static DCC_EMSG dcc_emsg;

static DCC_WF dbclean_wf;
static DCC_CLNT_CTXT *ctxt;
static int flods_off;
static int dccd_unlocked;		/* dccd has been told to unlock	*/

static DCC_SRVR_NM srvr = DCC_SRVR_NM_DEF;
static DCC_CLNT_ID srvr_clnt_id = DCC_ID_INVALID;
static ID_TBL *srvr_clnt_tbl;
static u_char use_ipv6 = 0;
#ifdef USE_DBCLEAN_F
static u_char dbclean_db_mode = DB_OPEN_NO_MMAP;
#else
static u_char dbclean_db_mode = 0;
#endif

static const DB_MAGIC db_magic = DB_VERSION_DEF;
static DB_MAGIC old_db_magic;

static u_char dccd_started_us;
static u_char cleardb;			/* 1=clear the database */
static u_char repair;			/* 1=only repair the database */
static u_char standalone;		/* 1=don't even try talk to dccd */
static u_char quiet;			/* 1=don't announce results to stdout */
static u_char keep_white;		/* 1=do not rebuild whitelist */

static const char *homedir;
static u_char cur_db_created;
static const char *cur_db_nm = DB_DCC_NAME;
static DCC_PATH cur_hash_nm;
static int old_db_fd = -1;
static DB_HADDR old_db_hash_used;
static DB_NOKEEP_CKS old_db_nokeep_cks;
static DB_FLOD_THOLDS old_db_flod_tholds;
static u_int old_db_flags;
static DB_PTR old_db_pos,  new_db_csize;
static off_t new_db_fsize;
static u_int new_db_page_size, tgt_db_page_size;
static FLOD_MMAPS new_flod_mmaps;
static u_char adj_delay_pos;
static int lock_db_fd = -1;
static DCC_PATH lock_db_nm;
static u_char new_db_created;
static DCC_PATH new_db_nm;
static int new_db_fd = -1;
static u_char new_hash_created;
static DCC_PATH new_hash_nm;
static DCC_PATH old_db_nm;

static int expire_secs = -1;
static int def_expire_secs = DB_EXPIRE_SECS_DEF;
static int expire_spamsecs = -1;
static int def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF;
static int have_expire_parms = 0;
static DB_EX_TS new_ex_ts;
static DB_EX_SECS new_ex_secs;
static DCC_TS spamts[DCC_DIM_CKS];

static DB_HADDR new_hash_len;

static int expired_rcds, comp_rcds, expired_cks;
static int white_cks, kept_cks;

static struct timeval start;
static DCC_TS future_ts;

#define RESTART_DELAY	(60*5)

static time_t progress_rpt;
#define REPORT_INTERVAL		(5*60)
#define REPORT_INTERVAL_FAST	10
static u_char progress_rpt_started;
static int progress_rpt_percent;

static u_char write_new_flush(u_char);
static u_char write_new_rcd(const void *, int);
static void write_new_magic(const struct timeval *);
static void mk_nm(DCC_PATH, const char *, const char *);
static void unlink_whine(const char *);
static void rename_bail(const char *, const char *);
static u_char expire(DB_PTR);
static u_char copy_db(void);
static u_char catchup(void);
static void parse_white(void);
static void build_hash(void);
static void server_listening(DCC_AOPS, u_int32_t);
static void dccd_new_db(const char *);
static void dbclean_msg(const char *p, ...) PATTRIB(1,2);
static void finish(void);
static void deadman(int);
static void sigterm(int);


static void
usage(u_char die)
{
	const char str[] = {
		"usage: [-64dDfFNRPSVq] [-i id]"
		" [-a [server-addr][,server-port]] [-h homedir]\n"
		"   [-G on] [-s hash-size] [-e seconds] [-E spamsecs]\n"
		"   [-t type,allsecs,threshold,spamsecs]"
		" [-L ltype,facility.level]"};
	static u_char complained;

	/* its important to try to run, so don't give up unless necessary */
	if (die) {
		dcc_logbad(EX_USAGE, complained ? "giving up" : str);
	} else if (!complained) {
		dcc_error_msg("%s\ncontinuing", str);
		complained = 1;
	}
}


int NRATTRIB
main(int argc, char **argv)
{
	char hostname[MAXHOSTNAMELEN];
	const char *rest;
	char *duparg, *cntstr, *allstr, *spamstr, *p;
	int allsecs, spamsecs;
	DCC_TGTS tgts;
	u_char print_version = 0;
	u_long l;
	DCC_CK_TYPES type;
	struct stat sb;
	int i;

	gettimeofday(&start, 0);
	progress_rpt = start.tv_sec;
	dcc_timeval2ts(future_ts, &start, 24*60*60);

	dcc_syslog_init(1, argv[0], 0);

	if (DCC_DIM_CKS != DCC_COMP_DIM_CKS)
		dcc_logbad(EX_SOFTWARE,
			   "DCC_DIM_CKS != DCC_COMP_DIM_CKS;"
			   " check uses of both");

	while ((i = getopt(argc, argv,
			   "64dDfFNRPSVqi:a:h:G:s:e:E:t:L:")) != EOF) {
		switch (i) {
		case '6':
#ifndef NO_IPV6
			use_ipv6 = DCC_CLNT_INFO_FG_IPV6;
#endif
			break;
		case '4':
			use_ipv6 = 0;
			break;

		case 'd':
			if (db_debug++)
				++dcc_clnt_debug;
			break;

		case 'D':
			dccd_started_us = 1;
			break;

		case 'f':
			dbclean_db_mode &= ~DB_OPEN_NO_MMAP;
			break;


		case 'F':
			dbclean_db_mode |= DB_OPEN_NO_MMAP;
			break;

		case 'N':		/* make a new, clear database */
			cleardb = 1;
			standalone = 1;
			break;

		case 'R':
			repair = 1;
			break;

		case 'P':
			if (have_expire_parms > 0)
				dcc_logbad(EX_USAGE,
					   "do not use -P with -e, -E, or -t");
			have_expire_parms = -1;
			break;

		case 'S':
			standalone = 1;
			break;

		case 'V':
			fprintf(stderr, DCC_VERSION"\n");
			print_version = 1;
			break;

		case 'q':
			quiet = 1;
			break;

		case 'i':
			l = strtoul(optarg, &p, 0);
			if (*p != '\0'
			    || l < DCC_SRVR_ID_MIN
			    || l > DCC_SRVR_ID_MAX)
				dcc_logbad(EX_USAGE, "invalid DCC ID \"-i %s\"",
					   optarg);
			srvr_clnt_id = l;
			break;

		case 'a':
			rest = dcc_parse_nm_port(dcc_emsg, optarg, srvr.port,
						 hostname, sizeof(hostname),
						 &srvr.port, 0, 0,
						 0, 0);
			if (!rest)
				dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
			rest += strspn(rest, DCC_WHITESPACE);
			if (*rest != '\0')
				dcc_logbad(EX_USAGE,
					   "unrecognized port number in"
					   "\"-a %s\"", optarg);
			if (hostname[0] != '\0')
				BUFCPY(srvr.hostname, hostname);
			break;

		case 'h':
			homedir = optarg;
			break;

		case 'G':
			if (strcasecmp(optarg, "on"))
				usage(0);
			dcc_syslog_init(1, argv[0], " grey");
			if (have_expire_parms > 0)
				dcc_logbad(EX_USAGE,
					   "do not use -G with -e, -E, or -t");
			grey_on = 1;
			dcc_syslog_init(1, argv[0], " grey");
			have_expire_parms = -1;
			cur_db_nm = DB_GREY_NAME;
			break;

		case 's':		/* hash table size in entries */
			new_hash_len = strtoul(optarg, &p, 0);
			if (*p != '\0'
			    || new_hash_len < MIN_HASH_ENTRIES
			    || new_hash_len > MAX_HASH_ENTRIES)
				dcc_logbad(EX_USAGE,
					   "invalid database size \"%s\"",
					   optarg);
			break;

		case 'e':		/* expiration for non-bulk checksums */
			if (grey_on)
				dcc_logbad(EX_USAGE,
					   "-e cannot be used with -G");
			if (have_expire_parms < 0)
				dcc_logbad(EX_USAGE,
					   "-e cannot be used with -P");
			have_expire_parms = 1;
			expire_secs = dcc_get_secs(optarg, 0,
						   DB_EXPIRE_SECS_MIN,
						   DB_EXPIRE_SECS_MAX, -1);
			if (expire_secs < 0)
				dcc_logbad(EX_USAGE,
					   "invalid expiration seconds"
					   " \"-e %s\"",
					   optarg);
			break;

		case 'E':		/* expiration for bulk checksums */
			if (grey_on)
				dcc_logbad(EX_USAGE,
					   "-E cannot be used with -G");
			if (have_expire_parms < 0)
				dcc_logbad(EX_USAGE,
					   "-E cannot be used with -P");
			have_expire_parms = 1;
			expire_spamsecs = dcc_get_secs(optarg, 0,
						       DB_EXPIRE_SECS_MIN,
						       DB_EXPIRE_SECS_MAX, -1);
			if (expire_spamsecs < 0)
				dcc_logbad(EX_USAGE,
					   "invalid long term spam"
					   " expiration seconds"
					   " \"-E %s\"",
					   optarg);
			break;

		case 't':
			if (grey_on)
				dcc_logbad(EX_USAGE,
					   "-t cannot be used with -G");
			if (have_expire_parms < 0)
				dcc_logbad(EX_USAGE,
					   "-t cannot be used with -P");
			have_expire_parms = 1;
			duparg = dcc_strdup(optarg);
			allstr = strchr(duparg, ',');
			if (!allstr)
				dcc_logbad(EX_USAGE,
					   " missing comma in \"-t %s\"",
					   optarg);
			*allstr++ = '\0';
			cntstr = strchr(allstr, ',');
			if (!cntstr) {
				spamstr = 0;
			} else {
				*cntstr++ = '\0';
				spamstr = strchr(cntstr, ',');
				if (!spamstr)
					dcc_logbad(EX_USAGE,
						   "missing comma after"
						   " \"%s\" in \"-t %s\"",
						   cntstr, optarg);
				*spamstr++ = '\0';
			}
			type = dcc_str2type(duparg);
			if (!DCC_CK_OK_USER(type))
				dcc_logbad(EX_USAGE,
					   "unrecognized checksum type in"
					   " \"-t %s\"", optarg);
			allsecs = dcc_get_secs(allstr, 0,
					       DB_EXPIRE_SECS_MIN,
					       DB_EXPIRE_SECS_MAX, -1);
			if (allsecs < 0)
				dcc_logbad(EX_USAGE,
					   "invalid seconds \"%s\" in \"%s\"",
					   allstr, optarg);
			if (!cntstr) {
				tgts = DCC_TGTS_TOO_MANY;
				if (allsecs == 0
				    || DCC_CK_LONG_TERM(type)) {
					spamsecs = allsecs;
				} else {
					spamsecs = max(DB_EXPIRE_SPAMSECS_DEF,
						       allsecs);
				}
			} else {
				tgts = dcc_str2cnt(cntstr);
				if (tgts > DCC_TGTS_TOO_MANY || tgts <= 1)
					dcc_logbad(EX_USAGE,
						   "unrecognized count \"%s\""
						   " in \"-t %s\"",
						   cntstr, optarg);
				spamsecs = dcc_get_secs(spamstr, 0,
							DB_EXPIRE_SECS_MIN,
							DB_EXPIRE_SECS_MAX, -1);
				if (spamsecs < 0)
					dcc_logbad(EX_USAGE,
						   "invalid seconds"
						   " \"%s\" in \"%s\"",
						   spamstr, optarg);
				if ((spamsecs < allsecs && spamsecs != 0)
				    || (allsecs == 0 && spamsecs != 0))
					dcc_logbad(EX_USAGE,
						   "\"%s\""
						   " must not be smaller than"
						   " \"%s\" in \"%s\"",
						   spamstr, allstr, optarg);
			}
			dcc_free(duparg);
			new_ex_secs[type].all = allsecs;
			new_ex_secs[type].spam = spamsecs;
			new_ex_secs[type].clean_thold = tgts;
			break;

		case 'L':
			dcc_parse_log_opt(optarg);
			break;

		default:
			usage(0);
		}
	}
	argc -= optind;
	argv += optind;
	if (argc != 0)
		usage(1);

	if (srvr_clnt_id == DCC_ID_INVALID && !standalone) {
		if (print_version)
			exit(EX_OK);
		usage(1);
	}

	if (srvr.port == 0)
		srvr.port = DCC_GREY2PORT(grey_on);

	syslog(dcc_trace_priority, DCC_VERSION" %s %s",
	       repair ? "repairing" : "cleaning",
	       DCC_NM2PATH(cur_db_nm));

	dcc_clnt_unthread_init();
	dcc_wf_init(&dbclean_wf, 0, 0);

	atexit(finish);
	signal(SIGALRM, deadman);
	signal(SIGHUP, sigterm);
	signal(SIGTERM, sigterm);
	signal(SIGINT, sigterm);

	/* move to the target directory */
	srvr.clnt_id = srvr_clnt_id;
	if (!dcc_cdhome(dcc_emsg, homedir))
		dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	if (!standalone) {
		i = load_ids(dcc_emsg, &srvr_clnt_tbl, srvr_clnt_id);
		if (i < 0)
			dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
		else if (!i)
			dcc_error_msg("%s", dcc_emsg);
		strcpy(srvr.passwd, srvr_clnt_tbl->cur_passwd);
	}

	mk_nm(lock_db_nm, cur_db_nm, DB_LOCK_SUFFIX);
	mk_nm(cur_hash_nm, cur_db_nm, DB_HASH_SUFFIX);
	mk_nm(old_db_nm, cur_db_nm, "-old");
	mk_nm(new_db_nm, cur_db_nm, "-new");
	mk_nm(new_hash_nm, new_db_nm, DB_HASH_SUFFIX);

	/* exclude other instances of this program */
	lock_db_fd = dcc_lock_open(dcc_emsg, lock_db_nm, O_RDWR|O_CREAT,
				   DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0);
	if (lock_db_fd < 0) {
		dcc_logbad(dcc_ex_code, "%s: dbclean already running?",
			   dcc_emsg);
	} else {
		char pid[32];

		snprintf(pid, sizeof(pid), "%ld\n", (long)getpid());
		write(lock_db_fd, pid, strlen(pid));
	}

	/* create & the lock new database file */
	new_db_fd = dcc_lock_open(dcc_emsg, new_db_nm, O_RDWR|O_CREAT,
				  DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0);
	if (new_db_fd == -1)
		dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	if (0 > ftruncate(new_db_fd, 0))
		dcc_logbad(EX_IOERR, "truncate(%s,0): %s",
			   DCC_NM2PATH(new_db_nm), ERROR_STR());
	new_db_fsize = 0;
	new_db_created = 1;
	new_db_csize = DB_PTR_BASE;

	if (0 > stat(cur_db_nm, &sb)) {
		if (errno != ENOENT)
			dcc_logbad(EX_IOERR, "stat(%s): %s",
				   DCC_NM2PATH(cur_db_nm), ERROR_STR());
		/* empty a missing database */
		cleardb = 1;
		tgt_db_page_size = grey_on ? 1 : 0;
	} else {
		tgt_db_page_size = grey_on ? sb.st_size/4 : 0;
	}
	if (tgt_db_page_size < MIN_HASH_ENTRIES*sizeof(HASH_ENTRY)
	    && tgt_db_page_size != 0)
		tgt_db_page_size = MIN_HASH_ENTRIES*sizeof(HASH_ENTRY);
	new_db_page_size = db_get_page_size(0, tgt_db_page_size);
	write_new_magic(0);


	if (standalone) {
		u_char busy;

		/* open and lock the current database to ensure
		 * the daemon is not running */
		old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm, O_RDWR,
					  DCC_LOCK_OPEN_NOWAIT,
					  DCC_LOCK_ALL_FILE, &busy);
		if (busy)
			dcc_logbad(EX_USAGE, "database %s in use: %s",
				   DCC_NM2PATH(cur_db_nm), dcc_emsg);
		if (cleardb
		    && stat(cur_db_nm, &sb) >= 0)
			dcc_logbad(EX_USAGE, "%s already exists",
				   DCC_NM2PATH(cur_db_nm));

		/* create and lock the current database if it did not exist
		 * to ensure the daemon is not running */
		if (old_db_fd < 0) {
			old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm,
						  O_RDWR|O_CREAT,
						  DCC_LOCK_OPEN_NOWAIT,
						  DCC_LOCK_ALL_FILE, 0);
			if (old_db_fd < 0)
				dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
			cur_db_created = 1;
		}

	} else {
		/* Tell the daemon to start turning off the flooding
		 * so we can adjust its positions in FLOD_MMAP_NM
		 * Try very hard to reach it. */
		dcc_min_delay_us = DCC_MAX_RTT;
		ctxt = dcc_tmp_clnt_init(dcc_emsg, 0, &srvr, grey_on, use_ipv6);
		if (!ctxt)
			dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
		++flods_off;
		if (DCC_OP_OK != dcc_aop(dcc_emsg, ctxt, grey_on, -1,
					 DCC_AOP_FLOD, DCC_AOP_FLOD_SHUTDOWN,
					 0, 0, 0, 0, 0, 0))
			dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	}

	/* resolve white-listed host names before locking the database */
	parse_white();

	/* Tell the daemon to unlock the database between operations
	 * and insist it stop flooding. */
	if (!standalone) {
		DCC_OPS aop;
		DCC_ADMN_RESP_VAL check_buf;
		int check_buflen;

		/* give the daemon a chance to stop pumping the floods */
		for (;;) {
			check_buflen = sizeof(check_buf);
			aop = dcc_aop(dcc_emsg, ctxt, grey_on, -1,
				      DCC_AOP_FLOD, DCC_AOP_FLOD_CHECK, 0, 0, 0,
				      &check_buf, &check_buflen, 0);
			if (aop ==  DCC_OP_ERROR
			    || aop == DCC_OP_INVALID) {
				dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
			} else if (aop != DCC_OP_ADMN) {
				dcc_logbad(EX_UNAVAILABLE, "%s: %s",
					   dcc_op2str(DCC_OP_ADMN,
						      DCC_AOP_FLOD,
						      DCC_AOP_FLOD_CHECK),
					   check_buf.string);
			}
			i = flod_running(check_buf.string);
			if (i < 0)
				dcc_logbad(EX_PROTOCOL,
					   "%s: unrecognized \"%s\"",
					   dcc_op2str(DCC_OP_ADMN,
						      DCC_AOP_FLOD,
						      DCC_AOP_FLOD_CHECK),
					   check_buf.string);
			if (i == 0)
				break;
			if (time(0) > start.tv_sec+30) {
				if (flods_off < 2) {
					++flods_off;
					if (DCC_OP_OK != dcc_aop(dcc_emsg,
							ctxt, grey_on, -1,
							DCC_AOP_FLOD,
							DCC_AOP_FLOD_HALT, 0,
							0, 0, 0, 0, 0))
					    dcc_logbad(dcc_ex_code, "%s",
						       dcc_emsg);
					continue;
				}
				if (time(0) > start.tv_sec+45)
					dcc_logbad(EX_UNAVAILABLE,
						   "failed to stop floods: %s",
						   check_buf.string);
			}
			usleep(100*1000);
		}
		dccd_unlocked = 1;
		if (DCC_OP_OK != dcc_aop(dcc_emsg, ctxt, grey_on, -1,
					 DCC_AOP_DB_UNLOCK,
					 0, 0, 0, 0, 0, 0, 0))
			dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	}

	if (cleardb) {
		dcc_trace_msg(DCC_VERSION" %s database %s",
			      cur_db_created ? "creating" : "clearing",
			      DCC_NM2PATH(cur_db_nm));

	} else if (repair) {
		dcc_error_msg("explicit repair of %s", DCC_NM2PATH(cur_db_nm));

	} else {
		if (old_db_fd >= 0) {
			close(old_db_fd);
			old_db_fd = -1;
		}
		if (!db_open(0, cur_db_nm, 0,
			     DB_OPEN_RDONLY | dbclean_db_mode
			     | (standalone
				? DB_OPEN_LOCK_NOWAIT : DB_OPEN_LOCK_WAIT))) {
			/* If the hash table is sick, check timestamps only
			 * as much as no hash table allows.
			 * Then rebuild the hash table. */
			repair = 1;

		} else {
			old_db_flags = db_flags;
			/* save a handle on the old database to get
			 * reports that arrive while we expire it */
			old_db_fd = dup(db_fd);
			if (old_db_fd < 0)
				dcc_logbad(EX_OSERR, "dup(%s): %s",
					   DCC_NM2PATH(cur_db_nm),
					   ERROR_STR());

			old_db_hash_used = db_hash_used;
			old_db_nokeep_cks = db_nokeep_cks;

			/* read old and create new database file */
			if (!expire(db_csize)) {
				close(old_db_fd);
				old_db_fd = -1;
				old_db_hash_used = 0;
				repair = 1;
			}
		}

		if (repair)
			dcc_error_msg("repairing %s", DCC_NM2PATH(cur_db_nm));
	}

	/* if we are repairing the hash table (including now repairing
	 * after encountering problems while expiring),
	 * copy the current file with minimal expiring */
	if (repair && !cleardb) {
		if (!copy_db())
			exit(EX_UNAVAILABLE);
	}
	build_hash();

	/* we have the new database locked
	 *
	 * Preserve the current files as "*-old" and install the new files
	 */
	rename_bail(cur_db_nm, old_db_nm);
	rename_bail(new_hash_nm, cur_hash_nm);
	new_hash_created = 0;
	rename_bail(new_db_nm, cur_db_nm);
	new_db_created = 0;
	cur_db_created = 0;
	if (cleardb) {
		if (0 > unlink(FLOD_MMAP_PATH(grey_on))
		    && errno != ENOENT)
			dcc_error_msg("unlink(%s): %s",
				      FLOD_MMAP_PATH(grey_on), ERROR_STR());
		if (!db_close(0, 1))
			exit(EX_UNAVAILABLE);
		exit(EX_OK);
	}
	strcpy(new_db_nm, cur_db_nm);
	strcpy(new_hash_nm, cur_hash_nm);

	/* if the daemon was not running, we're finished */
	if (standalone) {
		/* install the flood positions if things are ok */
		if (flod_mmaps) {
			memcpy(flod_mmaps, &new_flod_mmaps,
			       sizeof(new_flod_mmaps));
			flod_unmap(0, 0);
		}
		if (!db_close(0, 1))
			exit(EX_UNAVAILABLE);
		exit(EX_OK);
	}

	/* try to finish as much disk I/O as we can to minimize stalling
	 * by dccd when we close the file and hand it over */
	if (!db_unload(dcc_emsg))
		dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	if (0 > fsync(db_fd))
		dcc_logbad(EX_IOERR, "fsync(%s): %s",
			   DCC_NM2PATH(db_nm), ERROR_STR());
	if (0 > fsync(db_hash_fd))
		dcc_logbad(EX_IOERR, "fsync(%s): %s",
			   DCC_NM2PATH(db_hash_nm), ERROR_STR());

	/* Copy any records from the old file to the new file that were
	 * added to the old file while we were creating the new file. */
	if (!catchup()) {
		write_new_magic(0);
		exit(EX_UNAVAILABLE);
	}

	/* tell the daemon to switch to the new database and stop unlocking
	 * the database on every operation.  This will leave the daemon
	 * stuck waiting for us to unlock the new database. */
	dccd_new_db("copy late arrivals");

	/* install the flood positions if things are ok */
	if (flod_mmaps) {
		memcpy(flod_mmaps, &new_flod_mmaps,
		       sizeof(new_flod_mmaps));
		flod_unmap(0, 0);
	}

	/* Copy any records from the old file to the new file in the
	 * race to tell the daemon to switch to the new file.
	 * The new file is still locked from build_hash().
	 * The daemon should be stuck waiting to open it in the
	 * DCC_AOP_DB_NEW request via the preceding dccd_new_db().
	 *
	 * Since the daemon has switched and probably cannot go back,
	 * ignore any errors */
	catchup();
	if (!db_close(0, 1))
		exit(EX_UNAVAILABLE);

	/* finish() will be called via exit() to tell the daemon to resume
	 * flooding if necessary.  However, in the normal case, we removed
	 * all counts against flooding before calling dccd_new_db() */
	 exit(EX_OK);
}



/* adjust output flood positions */
static DB_PTR
adj_mmap(void)
{
	FLOD_MMAP *mp;
	DB_PTR delta, new_pos;

	delta = new_db_csize - old_db_pos;
	new_pos = 0;
	for (mp = new_flod_mmaps.mmaps;
	     mp <= LAST(new_flod_mmaps.mmaps);
	     ++mp) {
		/* do nothing to marks we have already adjusted */
		if (mp->oflod_index >= 0)
			continue;
		if (mp->confirm_pos > old_db_pos) {
			/* note the next mark that will need adjusting
			 * but do not adjust it yet */
			if (new_pos == 0
			    || new_pos > mp->confirm_pos)
				new_pos = mp->confirm_pos;
		} else {
			/* adjust marks not past the current position */
			mp->confirm_pos += delta;
			mp->oflod_index = 0;
		}
	}
	if (adj_delay_pos) {
		if (new_flod_mmaps.delay_pos > old_db_pos) {
			if (new_pos == 0
			    || new_pos > new_flod_mmaps.delay_pos)
				new_pos = new_flod_mmaps.delay_pos;
		} else {
			new_flod_mmaps.delay_pos += delta;
			adj_delay_pos = 0;
		}
	}

	return new_pos;
}



static void
deadman(int s UATTRIB)
{
	dcc_logbad(EX_IOERR, "db_lock() timed out; dccd restarted?");
}



/* Get the leading report for a checksum
 *	Leave db_sts.rcd2 pointing at the record. */
static DB_RCD_CK *
get_lead(DCC_CK_TYPES type,
	 const DCC_SUM sum)
{
	DB_RCD_CK *lead_ck;
	DB_FOUND db_result;

	/* we must lock the file to keep the daemon from changing the
	 * internal hash table links */
	if (!DB_IS_LOCKED()) {
		alarm(60*60);		/* don't stall for more than an hour */
		if (0 > db_lock(0))
			return 0;
		/* cheat and don't turn off the alarm, since we ought
		 * to be back here long before an hour has passed */
	}

	dcc_emsg[0] = '\0';
	db_result = db_lookup(dcc_emsg, type, sum, 0, MAX_HASH_ENTRIES,
			      &db_sts.hash, &db_sts.rcd2, &lead_ck);
	if (DB_FOUND_IT == db_result)
		return lead_ck;

	dcc_error_msg("hash lookup for %s from "L_HPAT" at "L_HPAT" = %d: %s",
		      dcc_type2str_err(type, 0, 1), old_db_pos,
		      db_sts.rcd2.s.rptr, db_result, dcc_emsg);
	return 0;
}



static void  PATTRIB(1,2)
dbclean_msg(const char *p, ...)
{
	va_list args;

	va_start(args, p);
	if (quiet) {
		vsyslog(dcc_trace_priority, p, args);
	} else {
		dcc_vtrace_msg(p, args);
	}
	va_end(args);
}



static void
report_progress(u_char force_time,
		const char *s1, DB_PTR done, const char *s2, DB_PTR total)
{
	int secs, interval;
	struct timeval tv;
	float percent;

	/* don't start progress reporting at the end */
	if (!total)
		percent = 100.0;
	else
		percent = (done*100.0)/total;
	if (!progress_rpt_started
	    && percent > 30.0)
		return;

	gettimeofday(&tv, 0);
	secs = tv.tv_sec - progress_rpt;
	interval = ((db_debug > 1)
		    ? REPORT_INTERVAL_FAST
		    : REPORT_INTERVAL);
	if (secs >= interval
	    || (force_time && progress_rpt_percent != 100)) {
		progress_rpt_started = 1;
		progress_rpt_percent = percent;
		secs = tv.tv_sec - start.tv_sec;
		secs /= interval;
		secs = secs*interval;
		progress_rpt = start.tv_sec + secs;
		if (db_debug > 1)
			dbclean_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%"
				    "\tdb_mmaps=%d hash=%d",
				    s1, done, total, s2, progress_rpt_percent,
				    db_stats.db_mmaps, db_stats.hash_mmaps);
		else
			dbclean_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%",
				    s1, done, total, s2, progress_rpt_percent);
	}
}



/* copy the existing flag file, discard junk and old entries */
static u_char				/* 1=done 0=database broken */
expire(DB_PTR old_db_csize)
{
#define EXPIRE_BAIL() {alarm(0); flod_unmap(0, 0); db_close(0, 0); return 0;}

	DCC_TS ts;
	u_char old_ok;
	DB_RCD rcd, new, new1;
	const DB_RCD_CK *rcd_ck, *rcd_ck2;
	DB_RCD_CK *new_ck;
	DCC_TGTS tgts_raw, ck_tgts, ld_tgts;
	u_char needed, obs_lvl, timely;
	int old_num_cks, new_num_cks, nokeep_cks;
	DB_PTR min_confirm_pos, next_adj_pos;
	FLOD_MMAP *mp;
	struct timeval new_sn, now;
	DCC_CK_TYPES prev_type, type, type2;
	int rcd_len;
	u_int new_len;
	DB_RCD_CK *lead_ck;
	struct stat sb;
	int progress_rpt_cnt;
	int i;

	/* Compute default durations
	 *  Assume the hash table will be 50% of the size of the database */
	i = (((db_csize*3)/2)*60) / db_max_rss;
	if (i > 60
	    && db_ex_secs[DCC_CK_FUZ2].all != 0
	    && db_ex_secs[DCC_CK_FUZ2].spam != 0) {
		def_expire_secs = (db_ex_secs[DCC_CK_FUZ2].all*60)/i;
		def_expire_secs -= def_expire_secs % (60*60);
		if (def_expire_secs < DB_EXPIRE_SECS_DEF_MIN)
			def_expire_secs = DB_EXPIRE_SECS_DEF_MIN;

		def_expire_spamsecs = (db_ex_secs[DCC_CK_FUZ2].spam*60)/i;
		def_expire_spamsecs -= def_expire_spamsecs % (24*60*60);
		if (def_expire_spamsecs < DB_EXPIRE_SPAMSECS_DEF_MIN)
			def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF_MIN;
#if DCC_DB_MBYTE == 0 && !defined(HAVE_PHYSMEM_TOTAL) && !defined(HAVE__SC_PHYS_PAGES) && !defined(HAVE_HW_PHYSMEM)
		if (def_expire_secs == DB_EXPIRE_SECS_DEF_MIN
		    || def_expire_spamsecs == DB_EXPIRE_SPAMSECS_DEF_MIN)
			dbclean_msg("no way to determine available RAM;"
				    " rebuild with ./configure with_db_memory");
#endif
		}
	if (expire_secs < 0 || expire_spamsecs < 0) {
		if (expire_secs < 0) {
			expire_secs = def_expire_secs;
			if (expire_secs > expire_spamsecs
			    && expire_spamsecs > 0)
				expire_secs = expire_spamsecs;
		}
		if (expire_spamsecs < 0)
			expire_spamsecs = max(def_expire_spamsecs, expire_secs);
		if (expire_secs != DB_EXPIRE_SECS_DEF
		    || expire_spamsecs != DB_EXPIRE_SPAMSECS_DEF)
			syslog(dcc_trace_priority,
			       "reducing default -e %d hours  -E %d days",
			       expire_secs / (60*60),
			       expire_spamsecs / (24*60*60));
	}

#ifdef DCC_DBCLEAN_ADJ_EPOCH     /* for testing */
	/* generate DCC_DBCLEAN_ADJ_EPOCH from something like
	 * date -j -f '%m/%d/%y %T' '02/27/04 15:30:24' '+%s'
	 */
	{
	time_t adj = time(0) - DCC_DBCLEAN_ADJ_EPOCH;
	if (expire_secs > adj)
		expire_secs -= adj;
	if (expire_spamsecs > adj)
		expire_spamsecs -= adj;
	}
#endif

	if (expire_spamsecs > 0 && expire_spamsecs < expire_secs)
		dcc_logbad(EX_USAGE,
			   "long term spam expiration -E"
			   " must be longer than -e");

	expired_rcds = 0;
	expired_cks = 0;
	kept_cks = white_cks;
	progress_rpt_cnt = 0;
	progress_rpt_started = 0;

	/* Compute thresholds for records we keep.
	 * Use the values from the previous use of dbclean as defaults
	 * unless they are bogus */
	old_ok = 0;
	dcc_secs2ts(ts, start.tv_sec);
	type = DCC_CK_TYPE_FIRST;
	for (;;) {
		DB_EX_SEC *th = &db_ex_secs[type];

		if (th->clean_thold <= 0
		    || th->clean_thold > DCC_TGTS_TOO_MANY
		    || th->spam <= 0
		    || th->spam > DB_EXPIRE_SECS_MAX
		    || th->all <= 0
		    || th->all > DB_EXPIRE_SECS_MAX) {
			if (db_debug)
				dbclean_msg("bad old threshold for %s",
					    dcc_type2str_err(type, 0, 1));
			break;
		}
		if (DCC_TS_NEWER_TS(db_ex_ts[type].all, ts)) {
			if (db_debug)
				dbclean_msg("bad old timestamp for %s",
					    dcc_type2str_err(type, 0, 1));
			break;
		}
		if (++type > DCC_CK_TYPE_LAST) {
			old_ok = 1;	/* the old values are ok */
			break;
		}
	}

	for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) {
		DB_EX_SEC *new_th = &new_ex_secs[type];

		/* always keep server-ID declarations one week */
		if (type == DCC_CK_SRVR_ID) {
			new_th->all = 7*24*60*60;
			new_th->spam = 7*24*60*60;
			new_th->clean_thold = 1;
			dcc_secs2ts(spamts[type], start.tv_sec - 7*24*60*60);
			memcpy(new_ex_ts[type].all, spamts[type],
			       sizeof(new_ex_ts[type].all));
			continue;
		}

		if (new_th->clean_thold == 0) {
			/* we have no explicit settings for this checksum type.
			 * Use the dccd bulk thresholds by default */
			if (grey_on) {
				new_th->clean_thold = DCC_TGTS_TOO_MANY;
			} else if (db_flod_tholds[type] <= DCC_TGTS_TOO_MANY
				   && db_flod_tholds[type] > 0) {
				new_th->clean_thold = db_flod_tholds[type];
			} else {
				new_th->clean_thold = BULK_THRESHOLD;
			}
			/* if we have no global expiriation parameters
			 * and the old durations are valid, use them */
			if (have_expire_parms < 0 && old_ok) {
				new_th->all = db_ex_secs[type].all;
				new_th->spam = db_ex_secs[type].spam;
			} else if (grey_on) {
				if (type == DCC_CK_GREY_TRIPLE) {
					new_th->all = DEF_GREY_WINDOW;
					new_th->spam = DEF_GREY_WHITE;
				} else if (type == DCC_CK_BODY
					   || type == DCC_CK_GREY_MSG) {
					new_th->all = DEF_GREY_WINDOW;
					new_th->spam = DEF_GREY_WINDOW;
				} else {
					new_th->all = 1;
					new_th->spam = 1;
				}
			} else {
				new_th->all = expire_secs;
				new_th->spam = (DCC_CK_LONG_TERM(type)
							? expire_spamsecs
							: expire_secs);
			}
		}

		dcc_secs2ts(spamts[type], start.tv_sec - new_th->spam);

		/* use previous time limits if they are ok & newer */
		dcc_secs2ts(ts, start.tv_sec - new_th->all);
		if (!old_ok
		    || DCC_TS_NEWER_TS(ts, db_ex_ts[type].all)) {
			memcpy(new_ex_ts[type].all, ts,
			       sizeof(new_ex_ts[type].all));
		} else {
			memcpy(new_ex_ts[type].all, db_ex_ts[type].all,
			       sizeof(new_ex_ts[type].all));
		}
	}

	/* put the thresholds into the new file */
	write_new_magic(0);

	/* if we are running as root,
	 * don't change the owner of the database */
	if (getuid() == 0) {
		if (0 > fstat(old_db_fd, &sb))
			dcc_logbad(EX_IOERR, "fstat(%s): %s",
				   DCC_NM2PATH(old_db_nm), ERROR_STR());
		if (0 > fchown(new_db_fd, sb.st_uid, sb.st_gid))
			dcc_logbad(EX_IOERR, "fchown(%s,%d,%d): %s",
				   DCC_NM2PATH(new_db_nm),
				   (int)sb.st_uid, (int)sb.st_gid,
				   ERROR_STR());
	}

	if (DB_PTR_BASE != lseek(old_db_fd, DB_PTR_BASE, SEEK_SET))
		dcc_logbad(EX_IOERR, "lseek(%s,%d): %s",
			   DCC_NM2PATH(cur_db_nm), DB_PTR_BASE,
			   ERROR_STR());
	read_rcd_invalidate();

	flod_mmap(0, db_sn, 0, 1, 1);
	if (flod_mmaps)
		memcpy(&new_flod_mmaps, flod_mmaps, sizeof(new_flod_mmaps));
	min_confirm_pos = new_flod_mmaps.delay_pos;
	next_adj_pos = DB_PTR_BASE;
	for (mp = new_flod_mmaps.mmaps;
	     mp <= LAST(new_flod_mmaps.mmaps);
	     ++mp) {
		if (mp->hostname[0] == '\0') {
			mp->oflod_index = 0;
		} else {
			mp->oflod_index = -1;
			if (min_confirm_pos > mp->confirm_pos)
				min_confirm_pos = mp->confirm_pos;
		}
	}
	adj_delay_pos = (new_flod_mmaps.delay_pos != 0) ? 1 : 0;

	/* preserve database-purged flag */
	new_sn.tv_sec = 0;
	new_sn.tv_usec = 0;
	if (!cleardb)
		dcc_ts2timeval(&new_sn, db_sn);
	if (new_sn.tv_sec != 0)
		new_sn = start;
	dcc_timeval2ts(new_flod_mmaps.sn, &new_sn, 0);

	/* copy the old file to the new, discarding old data as we go */
	for (old_db_pos = DB_PTR_BASE;
	     old_db_pos < old_db_csize;
	     old_db_pos += rcd_len) {
		if (--progress_rpt_cnt <= 0) {
			report_progress(0, "  processed",
					old_db_pos/1000000,
					"Mbytes", old_db_csize/1000000);
			progress_rpt_cnt = 1000;
		}

		if (old_db_pos == next_adj_pos)
			next_adj_pos = adj_mmap();

		rcd_len = read_rcd(0, &rcd,
				   old_db_fd, old_db_pos, cur_db_nm);
		if (rcd_len <= 0) {
			if (rcd_len == 0)
				dcc_error_msg("unexpected EOF in %s at "L_HPAT
					      " instead of "L_HPAT,
					      DCC_NM2PATH(cur_db_nm),
					      old_db_pos,
					      old_db_csize);
			/* ask our neighbors to fix our database */
			memset(&new_sn, 0, sizeof(new_sn));
			old_db_pos = old_db_csize;
			break;
		}

		/* skip end-of-page padding */
		if (rcd_len == sizeof(rcd)-sizeof(rcd.cks))
			continue;

		if (DB_RCD_ID(&rcd) == DCC_ID_WHITE) {
			/* skip whitelist entries if whitelist source ok */
			if (!keep_white)
				continue;
			/* Refresh whitelist entries if source is bad */
			dcc_timeval2ts(rcd.ts, &start, 0);
		}

		old_num_cks = DB_NUM_CKS(&rcd);

		/* expire or throw away deleted reports */
		tgts_raw = DB_TGTS_RCD_RAW(&rcd);
		if (tgts_raw == 0) {
			++expired_rcds;
			expired_cks += old_num_cks;
			continue;
		}

		if (DCC_TS_NEWER_TS(rcd.ts, future_ts)) {
			dcc_error_msg("discarding report at "L_HPAT
				      " from the future %s",
				      old_db_pos,
				      dcc_ts2str_err(rcd.ts));
			++expired_rcds;
			expired_cks += old_num_cks;
			continue;
		}


		needed = 0;
		obs_lvl = 0;
		timely = 1;
		nokeep_cks = 0;
		new_len = sizeof(new)-sizeof(new.cks);
		memcpy(&new, &rcd, new_len);
		new.fgs_num_cks &= ~DB_CK_MASK;
		new_ck = new.cks;
		for (prev_type = DCC_CK_INVALID, rcd_ck = rcd.cks;
		     rcd_ck < &rcd.cks[old_num_cks];
		     prev_type = type, ++rcd_ck) {
			type = DB_CK_TYPE(rcd_ck);
			if (!DCC_CK_OK_DB(type)) {
				dcc_error_msg("discarding %s checksum at "
					      L_HPAT,
					      dcc_type2str_err(type, 0, 1),
					      old_db_pos);
				++expired_cks;
				new.fgs_num_cks |= DB_RCD_FG_TRIM;
				new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
				continue;
			}

			if (type <= prev_type
			    && prev_type != DCC_CK_FLOD_PATH) {
				dcc_error_msg("discarding out of order %s"
					      " checksum at "L_HPAT,
					      dcc_type2str_err(type, 0, 1),
					      old_db_pos);
				++expired_cks;
				new.fgs_num_cks |= DB_RCD_FG_TRIM;
				new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
				continue;
			}

			/* Silently discard pure junk from other servers,
			 * provided it is junk by default */
			if (DB_TEST_NOKEEP(db_nokeep_cks, type)
			    && DB_DEF_NOKEEP(type)
			    && DB_RCD_ID(&rcd) != DCC_ID_WHITE) {
				++expired_cks;
				continue;
			}

			/* Keep paths except on old records or records that
			 * have been trimmed or compressed.
			 * Do not remove paths from server-ID declarations. */
			if (type == DCC_CK_FLOD_PATH) {
				if (DB_RCD_TRIMMED(&new)
				    || DB_RCD_ID(&new) == DCC_ID_COMP)
					continue;
				rcd_ck2 = rcd_ck+1;
				for (;;) {
					type2 = DB_CK_TYPE(rcd_ck2);
					if (type2 == DCC_CK_SRVR_ID
					    || !DCC_TS_OLDER_TS(rcd.ts,
							new_ex_ts[type2].all)) {
					    /* keep this path since this report
					     * is a server-ID declaration
					     * or not old */
					    *new_ck = *rcd_ck;
					    new_len += sizeof(*rcd_ck);
					    ++new_ck;
					    ++new.fgs_num_cks;
					    ++nokeep_cks;
					    break;
					}
					if (++rcd_ck2>=&rcd.cks[old_num_cks]) {
					    /* we are discarding this path */
					    new.fgs_num_cks |= DB_RCD_FG_TRIM;
					    new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
					    break;
					}
				}
				continue;
			}

			/* throw this checksum away if it is extremely old */
			if (DCC_TS_OLDER_TS(rcd.ts, spamts[type])) {
				++expired_cks;
				new.fgs_num_cks |= DB_RCD_FG_TRIM;
				new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
				continue;
			}

			/* Throw away delete requests after they've
			 * been sent off */
			if (tgts_raw == DCC_TGTS_DEL
			    && old_db_pos < min_confirm_pos) {
				++expired_cks;
				new.fgs_num_cks |= DB_RCD_FG_TRIM;
				new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
				continue;
			}

			if (!DCC_TS_OLDER_TS(rcd.ts, new_ex_ts[type].all)) {
				/* This report is recent.
				 * However, obsolete or junk checksums
				 * don't make the report needed */
				if (DB_TEST_NOKEEP(db_nokeep_cks, type)
				    && DB_RCD_ID(&rcd) != DCC_ID_WHITE) {
					++nokeep_cks;
				} else if (DB_CK_OBS(rcd_ck)) {
					if (obs_lvl < db_ck_fuzziness[type]) {
					    obs_lvl = db_ck_fuzziness[type];
					    needed = 0;
					}
				} else {
					if (obs_lvl <= db_ck_fuzziness[type]) {
					    obs_lvl = db_ck_fuzziness[type];
					    needed = 1;
					}
				}

			} else {
				/* This checksum is old.
				 * Throw away delete requests
				 * and other servers' checksums we don't like */
				if (tgts_raw == DCC_TGTS_DEL
				    || DB_TEST_NOKEEP(db_nokeep_cks, type)) {
					++expired_cks;
					new.fgs_num_cks |= DB_RCD_FG_TRIM;
					new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
					continue;
				}
				/* Throw away old obsolete checksums
				 * and entire reports if the fuzziest
				 * checksum is obsolete */
				if (DB_CK_OBS(rcd_ck)) {
					if (obs_lvl < db_ck_fuzziness[type]) {
					    obs_lvl = db_ck_fuzziness[type];
					    needed = 0;
					}
					++expired_cks;
					new.fgs_num_cks |= DB_RCD_FG_TRIM;
					new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
					continue;
				}

				/* The checksum is old enough to compress,
				 * but not old enough to delete, so mark
				 * the record as eligible for splitting. */
				timely = 0;

				/* old summaries are unneeded */
				if (DB_RCD_SUMRY(&rcd))
					continue;

				/* Discard this checksum if it does not
				 * contribute to a compelling total.
				 * To determine the total,
				 * we must have a hash table */
				ck_tgts = DB_TGTS_CK(rcd_ck);
				if (!repair
				    &&  ck_tgts < new_ex_secs[type
							].clean_thold) {
					lead_ck = get_lead(type, rcd_ck->sum);
					if (!lead_ck)
					    EXPIRE_BAIL();
					/* discard the checksum if its ultimate
					 * total is low or if it reaches
					 * spam after this report. */
					ld_tgts = DB_TGTS_CK(lead_ck);
					if (ld_tgts < new_ex_secs[type
							].clean_thold
					    || (ld_tgts == DCC_TGTS_TOO_MANY
						&& (DB_TGTS_CK(rcd_ck)
						    != DCC_TGTS_TOO_MANY))) {
					    ++expired_cks;
					    new.fgs_num_cks |= DB_RCD_FG_TRIM;
					    new.fgs_num_cks &= ~DB_RCD_FG_DELAY;
					    continue;
					}
				}

				/* since we did not delete this checksum,
				 * we need the record */
				if (obs_lvl <= db_ck_fuzziness[type]) {
					obs_lvl = db_ck_fuzziness[type];
					needed = 1;
				}
			}

			/* Keep this checksum if we decide the whole report
			 * is needed. */
			*new_ck = *rcd_ck;
			new_len += sizeof(*rcd_ck);
			++new_ck;
			++new.fgs_num_cks;
		}

		if (!repair && !standalone && DB_IS_LOCKED()) {
			gettimeofday(&now, 0);
			if (db_locked.tv_sec != now.tv_sec
			    || db_locked.tv_usec + 500*1000 < now.tv_usec) {
				if (!db_unlock(0))
					EXPIRE_BAIL();
			}
		}

		/* if none of its checksums are needed,
		 * then discard the entire record */
		if (!needed) {
			expired_cks += DB_NUM_CKS(&new);
			++expired_rcds;
			continue;
		}

		/* Put the record into the new file.
		 * If all of the record is recent, it contains 1 checksum,
		 * or if all of its totals are the same, then simply add it.
		 * Otherwise, divide in into separate records of one
		 * checksum so each checksum can be independently compressed.
		 * Optimize a little by putting the first checksums with
		 * a common sum into one record. */
		new_num_cks = DB_NUM_CKS(&new);
		kept_cks += new_num_cks - nokeep_cks;
		if (!timely
		    || DB_RCD_ID(&new) == DCC_ID_COMP
		    || DB_RCD_TRIMMED(&new)) {
			new_ck = new.cks;

			/* skip the checksums that have the same total
			 * as the first checksum and leave them with the
			 * original report */
			new_ck = new.cks;
			tgts_raw = DB_TGTS_CK(new_ck);
			i = new_num_cks;
			while (--i > 0) {
				++new_ck;
				if (DB_TGTS_CK(new_ck) != tgts_raw)
					break;
			}

			/* put the remaining checksums into their own records */
			while (--i >= 0) {
				memcpy(&new1, &new,
				       sizeof(new1)-sizeof(new1.cks));
				new1.cks[0] = *new_ck;
				new1.fgs_num_cks = 1;
				new1.srvr_id_auth = DCC_ID_COMP;
				if (!write_new_rcd(&new1,
						   sizeof(new1)
						   -sizeof(new1.cks)
						   +sizeof(new1.cks[0])))
					EXPIRE_BAIL();
				new_len -= sizeof(new.cks[0]);
				new.fgs_num_cks = DB_NUM_CKS(&new)-1;
				new.srvr_id_auth = DCC_ID_COMP;
			}
		}

		/* write the rest (or all) of the record */
		if (!write_new_rcd(&new, new_len))
			EXPIRE_BAIL();
	}
	write_new_flush(1);
	alarm(0);

	/* do final adjustment of the positions */
	adj_mmap();
	/* force them to be right if the system crashed with the
	 * flod.map file on the disk more up to date and so after the
	 * database file on the disk */
	for (mp = new_flod_mmaps.mmaps;
	     mp <= LAST(new_flod_mmaps.mmaps);
	     ++mp) {
		if (mp->hostname[0] != '\0'
		    && mp->confirm_pos > new_db_csize)
			mp->confirm_pos = new_db_csize;
	}

	i = db_close(0, 1);
	write_new_magic(&new_sn);
	report_progress(1, "  processed",
			old_db_pos/1000000,
			"Mbytes", old_db_csize/1000000);
	dbclean_msg("expired %d records and %d checksums in %s",
		    expired_rcds, expired_cks,
		    DCC_NM2PATH(cur_db_nm));
	return i;
}



/* copy the database copy while doing minimal expiring */
static u_char
copy_db(void)
{
	/* do not lock the old database because the daemon must continue
	 * to answer requests */
	if (old_db_fd < 0) {
		old_db_fd = open(cur_db_nm, O_RDONLY, 0);
		if (old_db_fd == -1)
			dcc_logbad(EX_IOERR, "open(%s): %s",
				   DCC_NM2PATH(cur_db_nm), ERROR_STR());
	}

	if (!read_magic(dcc_emsg, &old_db_magic, old_db_fd, cur_db_nm))
		dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
	if (memcmp(old_db_magic.s.version, db_magic.s.version,
		   sizeof(old_db_magic.s.version)))
		dcc_logbad(EX_IOERR, "%s has the wrong magic \"%.*s\"",
			   DCC_NM2PATH(cur_db_nm),
			   ISZ(old_db_magic.s.version), old_db_magic.s.version);

	memcpy(db_sn, old_db_magic.s.sn, sizeof(db_sn));
	memcpy(&db_ex_ts, &old_db_magic.s.ex_ts,
	       sizeof(db_ex_ts));
	memcpy(&db_ex_secs, &old_db_magic.s.ex_secs,
	       sizeof(db_ex_secs));
	old_db_nokeep_cks = db_nokeep_cks = old_db_magic.s.nokeep_cks;
	memcpy(db_flod_tholds, old_db_magic.s.flod_tholds,
	       sizeof(db_flod_tholds));
	memcpy(old_db_flod_tholds, old_db_magic.s.flod_tholds,
	       sizeof(old_db_flod_tholds));
	old_db_flags = db_flags = old_db_magic.s.flags;

	return expire(old_db_magic.s.db_csize);
}



/* Copy any records from the old file to the new file that were
 * added to the old file while we were creating the new file. */
static u_char
catchup(void)
{
	DB_RCD rcd;
	int rcd_len;
	u_char result;
	int count, old_count;

	/* Because the old file should still be unlocked, the daemon
	 * will have been keeping its magic number block accurate */
	result = 1;
	count = 0;
	do {
		old_count = count;
		if (!read_magic(dcc_emsg, &old_db_magic,
				old_db_fd, old_db_nm)) {
			dcc_error_msg("%s", dcc_emsg);
			result = 0;
			break;
		}
		if (old_db_magic.s.db_csize < old_db_pos) {
			dcc_error_msg("%s mysteriously truncated",
				      DCC_NM2PATH(old_db_nm));
			result = 0;
			break;
		}
		if ((off_t)old_db_pos != lseek(old_db_fd, old_db_pos,
					       SEEK_SET)) {
			dcc_error_msg("lseek(%s, "L_HPAT"): %s",
				      DCC_NM2PATH(old_db_nm), old_db_pos,
				      ERROR_STR());
			result = 0;
			break;
		}
		read_rcd_invalidate();
		while (old_db_pos < old_db_magic.s.db_csize) {
			rcd_len = read_rcd(0, &rcd,
					   old_db_fd, old_db_pos, old_db_nm);
			if (rcd_len <= 0) {
				if (rcd_len == 0)
					dcc_error_msg("premature EOF in %s"
						      " at "L_HPAT
						      " instead of "L_HPAT,
						      DCC_NM2PATH(old_db_nm),
						      old_db_pos,
						      old_db_magic.s.db_csize);
				result = 0;
				break;
			}
			/* If something bad happens, we may not be able to
			 * go back to the old file.  Carry on to get as much
			 * data as we can although we know the dccd daemon
			 * may croak when we release it */
			if (!db_add_rcd(0, &rcd)) {
				result = 0;
				break;
			}
			old_db_pos += rcd_len;
			++count;
		}
	} while (result && old_count != count);

	if (count > 0 && db_debug >= 1)
		dbclean_msg("added %d late reports", count);

	return result;
}



/* try to compress old reports */
static void
compress_old(void)
{
	DB_PTR prev;
	DB_RCD_CK *new_ck, *prev_ck;
	int new_ck_num, prev_ck_num;
	DCC_TGTS new_tgts, prev_tgts;
	DCC_CK_TYPES new_type, prev_type;
	DCC_SRVR_ID new_srvr, prev_srvr;

	/* Before spending the time to map a preceding checksum,
	 * find at least one checksum worth keeping and that might
	 * be combined or compressed with its predecessor.
	 * Don't bother checking if this report has been compressed
	 * or trimmed */
	new_ck_num = DB_NUM_CKS(db_sts.rcd.d.r),
	new_ck = db_sts.rcd.d.r->cks;
	if (DB_RCD_TRIMMED(db_sts.rcd.d.r))
		new_srvr = DCC_ID_COMP;
	else
		new_srvr = DB_RCD_ID(db_sts.rcd.d.r);
	for (;;) {
		if (--new_ck_num < 0)
			return;
		/* all of the checksums in this record must be old */
		new_type = DB_CK_TYPE(new_ck);
		if (new_srvr != DCC_ID_COMP
		    && !DB_TEST_NOKEEP(db_nokeep_cks, new_type)
		    && !DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts,
					new_ex_ts[DB_CK_TYPE(new_ck)].all))
			return;
		/* you can compress 2 reports only if you have 2 */
		prev = DB_PTR_EX(new_ck->prev);
		if (prev != DB_PTR_NULL)
			break;
		++new_ck;
	}

	/* having picked a checksum, map the record with its predecessor */
	prev_ck = db_map_rcd_ck(dcc_emsg, &db_sts.rcd2,
				prev, DB_CK_TYPE(new_ck));
	if (!prev_ck)
		dcc_logbad(dcc_ex_code, "%s", dcc_emsg);

	/* The current and previous records must be old
	 * and contain the same useful checksums. */
	new_ck_num = DB_NUM_CKS(db_sts.rcd.d.r);
	new_ck = db_sts.rcd.d.r->cks;
	if (DB_RCD_TRIMMED(db_sts.rcd2.d.r))
		prev_srvr = DCC_ID_COMP;
	else
		prev_srvr = DB_RCD_ID(db_sts.rcd2.d.r);
	prev_ck_num = DB_NUM_CKS(db_sts.rcd2.d.r);
	prev_ck = db_sts.rcd2.d.r->cks;
	for (;;) {
		/* we must run out of checksums in the two reports at the
		 * same time */
		if (prev_ck_num == 0 || new_ck_num == 0) {
			if (prev_ck_num == new_ck_num)
				break;
			return;
		}

		/* ignore paths */
		prev_type = DB_CK_TYPE(prev_ck);
		if (DB_TEST_NOKEEP(db_nokeep_cks, prev_type)) {
			--prev_ck_num;
			++prev_ck;
			continue;
		}
		new_type = DB_CK_TYPE(new_ck);
		if (DB_TEST_NOKEEP(db_nokeep_cks, new_type)) {
			--new_ck_num;
			++new_ck;
			continue;
		}

		/* because the checksums are ordered,
		 * give up at the first difference in checksums */
		if (new_type != prev_type
		    || memcmp(new_ck->sum, prev_ck->sum, sizeof(new_ck->sum)))
			return;

		/* Give up at the first recent and valuable checksum. */
		if ((new_srvr != DCC_ID_COMP
		     && !DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts,
					 new_ex_ts[new_type].all))
		    || (prev_srvr != DCC_ID_COMP
			&& !DCC_TS_OLDER_TS(db_sts.rcd2.d.r->ts,
					    new_ex_ts[new_type].all)))
			return;

		--prev_ck_num;
		++prev_ck;
		--new_ck_num;
		++new_ck;
	}

	/* The current and previous records are compatiable.
	 * Add the count of the previous record to the current record
	 * and mark the previous record useless.
	 * The individual totals in the current record are already correct,
	 * so postpone worrying about the deleted record. */
	new_tgts = DB_TGTS_RCD_RAW(db_sts.rcd.d.r);
	if (new_tgts < DCC_TGTS_TOO_MANY) {
		prev_tgts = DB_TGTS_RCD(db_sts.rcd2.d.r);
		if (prev_tgts > DCC_TGTS_TOO_MANY)
			return;
		if (prev_tgts == DCC_TGTS_TOO_MANY) {
			new_tgts = DCC_TGTS_TOO_MANY;
		} else {
			new_tgts += prev_tgts;
			if (new_tgts > DCC_TGTS_TOO_MANY)
				new_tgts = DCC_TGTS_TOO_MANY;
		}
		DB_TGTS_RCD_SET(db_sts.rcd.d.r, new_tgts);
	}

	/* mark one of the records to be deleted next time */
	DB_TGTS_RCD_SET(db_sts.rcd2.d.r, 0);

	db_sts.rcd.d.r->srvr_id_auth = DCC_ID_COMP;
	db_sts.rcd.d.r->fgs_num_cks = DB_NUM_CKS(db_sts.rcd.d.r);
	/* use the newest timestamp */
	if (DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts, db_sts.rcd2.d.r->ts))
		memcpy(db_sts.rcd.d.r->ts, db_sts.rcd2.d.r->ts,
		       sizeof(db_sts.rcd.d.r->ts));
	++comp_rcds;
}



/* write a parsed whitelist checksum */
static int
white_write(DCC_EMSG emsg, DCC_WF *wf UATTRIB, const char *fnm, int lineno,
	    DCC_CK_TYPES type, DCC_SUM sum, DCC_TGTS tgts)
{
	DB_RCD rcd;
	int rcd_len;

	/* ignore checksums that clients are never supposed to send
	 * to the server */
	switch (type) {
	case DCC_CK_INVALID:
	case DCC_CK_ENV_TO:
	case DCC_CK_GREY_MSG:
	case DCC_CK_GREY_TRIPLE:
	case DCC_CK_SRVR_ID:
		dcc_pemsg(EX_DATAERR, emsg,
			  "%s checksum%s cannot be used in a dccd whitelist",
			  dcc_type2str_err(DCC_CK_ENV_TO, 0, 0),
			  fnm_lineno(fnm, lineno));
		return 0;

	case DCC_CK_IP:
	case DCC_CK_ENV_FROM:
	case DCC_CK_FROM:
	case DCC_CK_MESSAGE_ID:
	case DCC_CK_RECEIVED:
	case DCC_CK_SUB:
	case DCC_CK_BODY:
	case DCC_CK_FUZ1:
	case DCC_CK_FUZ2:
		break;			/* these are ok */
	}

	/* greylist whitelist entries cannot involve blacklisting
	 * and use DCC_TGTS_OK2 to signal whitelisting because that
	 * is what greylist clients expect */
	if (grey_on) {
		/* silently ignore anything except whitelisting */
		if (tgts != DCC_TGTS_OK)
			return 1;
		tgts = DCC_TGTS_OK2;
	}

	memset(&rcd, 0, sizeof(rcd));
	dcc_timeval2ts(rcd.ts, &start, 0);
	rcd.srvr_id_auth = DCC_ID_WHITE;
	rcd.fgs_num_cks = 1;
	rcd.cks[0].type_fgs = type;
	rcd_len = sizeof(rcd) - sizeof(rcd.cks) + sizeof(rcd.cks[0]);
	DB_TGTS_RCD_SET(&rcd, tgts);
	memcpy(rcd.cks[0].sum, sum, sizeof(rcd.cks[0]));

	++white_cks;
	return write_new_rcd(&rcd, rcd_len) ? 1 : -1;
}



/* Add the whitelist of certified non-spam senders and otherwise
 *	start the database */
static void
parse_white(void)
{
	FILE *white_file;
	const char *white_nm;

	white_cks = 0;

	white_nm = grey_on ? "grey_whitelist" : "whitelist";
	if (!keep_white) {
		white_file = fopen(white_nm, "r");
		if (!white_file) {
			/* worry only if the file exists but can't be used */
			if (errno != ENOENT) {
				dcc_error_msg("fopen(%s): %s",
					      DCC_NM2PATH(white_nm),
					      ERROR_STR());
				keep_white = 1;
			}
		} else {
			if (!dcc_parse_whitefile(0, &dbclean_wf,
						 white_nm, white_file,
						 0, white_write, 0, 0)) {
				keep_white = 1;
			}
			fclose(white_file);
		}
	}
	if (keep_white) {
		/* If the whitelist was bad, purge the new database of
		 * the bad new whitelist.  We will use the existing
		 * whitelist */
		write_new_flush(1);
		new_db_csize = DB_PTR_BASE;
		if (0 > ftruncate(new_db_fd, DB_PTR_BASE))
			dcc_logbad(EX_IOERR, "truncate(%s, %d): %s",
				   DCC_NM2PATH(new_db_nm), DB_PTR_BASE,
				   ERROR_STR());
		new_db_fsize = DB_PTR_BASE;
		white_cks = 0;
	}

	/* update the counts in the database file */
	write_new_magic(0);
}



/* rebuild the hash table and the totals and links within the database file
 *	finish with the file locked */
static void
build_hash(void)
{
	DB_PTR rcd_pos;
	DB_HADDR haddr_window, haddr_lo, haddr_hi;
	int pass, total_passes;
	u_int rcd_len;
	int rcd_cks, rcd_sums, rcds, sums;
	const DB_RCD_CK *rcd_ck;
	int progress_rpt_cnt;
	DB_HADDR guess_hash_len;

	db_buf_init(new_db_page_size);

	if (!new_hash_len) {
		/* Try to choose a hash table size now so that when it
		 * is next time to rebuild after 24 hours of incoming
		 * checksums, the alpha or load factor will still be 0.9.
		 * We probably ran 24 hours ago, so the old hash size
		 * is a good estimate of the size tomorrow. */
		new_hash_len = old_db_hash_used;

		/* Take what turns out to be a good guess if we do not
		 * have a good measure of the old hash table size. */
		guess_hash_len = kept_cks+white_cks;
		guess_hash_len += guess_hash_len/10;
		if (new_hash_len < guess_hash_len)
			new_hash_len = guess_hash_len;

		/* go for alpha 0.9 in 24 hours */
		new_hash_len += new_hash_len/10;

		if (new_hash_len > db_max_hash_entries)
			dbclean_msg("default hash table size of %d"
				    " larger than maximum %d",
				    new_hash_len, db_max_hash_entries);

		if (new_hash_len < MIN_HASH_ENTRIES)
			new_hash_len = MIN_HASH_ENTRIES;
		if (new_hash_len < DEF_HASH_ENTRIES
		    && !grey_on)
			new_hash_len = DEF_HASH_ENTRIES;
	}

	/* Open and lock the new database */
	unlink(new_hash_nm);
	new_hash_created = 1;
	if (!db_open(0, new_db_nm, new_hash_len,
		     DB_OPEN_LOCK_NOWAIT | dbclean_db_mode)) {
		dcc_logbad(dcc_ex_code, "could not start database %s",
			   DCC_NM2PATH(new_db_nm));
	}
	db_nokeep_cks = old_db_nokeep_cks;
	memcpy(db_flod_tholds, old_db_flod_tholds, sizeof(db_flod_tholds));

	/* add every record in the database file to the hash table and
	 * fix its accumulated counts and reverse links */
	comp_rcds = 0;
	sums = 0;
	rcds = 0;
	progress_rpt_cnt = 0;
	progress_rpt_started = 0;
	haddr_window = db_hash_page_len*((db_buf_total*3)/4);
	total_passes = (db_hash_len+haddr_window-1)/haddr_window;
	for (haddr_lo = 0, pass = 1;
	     haddr_lo < db_hash_len;
	     haddr_lo = haddr_hi, ++pass) {
		if (haddr_lo > db_hash_len-haddr_window)
			haddr_hi = MAX_HASH_ENTRIES;
		else
			haddr_hi = haddr_lo+haddr_window;
		for (rcd_pos = DB_PTR_BASE;
		     rcd_pos < db_csize;
		     rcd_pos += rcd_len) {
			/* skip reports crossing page bounardies */
			if (rcd_pos%db_page_size > db_page_max) {
				rcd_len = DB_RCD_PAD;
				continue;
			}
			if (--progress_rpt_cnt <= 0) {
				report_progress(0, "  hash rebuilt",
						sums/total_passes,
						"checksums", kept_cks);
				progress_rpt_cnt = 1000;
			}

			db_end_pg_num = rcd_pos / db_page_size;
			if (!db_map_rcd(0, &db_sts.rcd, rcd_pos, &rcd_len)) {
				dcc_logbad(dcc_ex_code,
					   "hash build failed reading"
					   " record at "L_HPAT,
					   rcd_pos);
			}

			/* skip end of page padding */
			if (db_sts.rcd.d.r->fgs_num_cks == 0)
				continue;

			++rcds;

			/* count the checksums we'll link in this record */
			rcd_cks = DB_NUM_CKS(db_sts.rcd.d.r);
			rcd_sums = 0;
			for (rcd_ck = db_sts.rcd.d.r->cks;
			     rcd_ck < &db_sts.rcd.d.r->cks[rcd_cks];
			     ++rcd_ck) {
				if (!DB_TEST_NOKEEP(db_nokeep_cks,
						    DB_CK_TYPE(rcd_ck)))
					++rcd_sums;
			}
			sums += rcd_sums;

			if (!db_link_rcd(0, haddr_lo, haddr_hi)) {
				dcc_logbad(dcc_ex_code,
					   "failed to relink record at "L_HPAT,
					   rcd_pos);
			}

			compress_old();
		}
	}

	report_progress(1, "  hash rebuilt", sums/total_passes,
			"checksums", kept_cks);
	dbclean_msg("hashed %d records containing %d checksums,"
		    " compressed %d records",
		    rcds, sums/total_passes, comp_rcds);
	dbclean_msg("%d hash entries total, %d or %d%% used",
		    db_hash_len,
		    HASH_LEN_EXT(db_hash_used),
		    (HASH_LEN_EXT(db_hash_used)*100)
		    / HASH_LEN_EXT(db_hash_len));
}



static u_char
write_new_db(const void *buf, int buflen, off_t pos, u_char fatal)
{
	int i;

	if (pos != lseek(new_db_fd, pos, SEEK_SET)) {
		if (fatal) {
			dcc_logbad(EX_IOERR, "lseek(%s, 0): %s",
				   DCC_NM2PATH(new_db_nm), ERROR_STR());
		} else {
			dcc_error_msg("lseek(%s, 0): %s",
				      DCC_NM2PATH(new_db_nm), ERROR_STR());
		}
		return 0;
	}

	i = write(new_db_fd, buf, buflen);
	if (i == buflen) {
		if (new_db_fsize < pos+buflen)
			new_db_fsize = pos+buflen;
		return 1;
	}

	if (fatal) {
		if (i < 0)
			dcc_logbad(EX_IOERR, "write(%s): %s",
				   DCC_NM2PATH(new_db_nm), ERROR_STR());
		else
			dcc_logbad(EX_IOERR, "write(%s)=%d instead of %d",
				   DCC_NM2PATH(new_db_nm), i, buflen);
	} else {
		if (i < 0)
			dcc_error_msg("write(%s): %s",
				      DCC_NM2PATH(new_db_nm), ERROR_STR());
		else
			dcc_error_msg("write(%s)=%d instead of %d",
				      DCC_NM2PATH(new_db_nm), i, buflen);
	}
	return 0;
}



static u_char write_new_db_buf[64*1024];
static u_int write_new_db_buflen = 0;
static DB_PTR write_new_base;

static u_char
write_new_flush(u_char fatal)
{
	u_char result = 1;

	if (write_new_db_buflen != 0
	    && !write_new_db(write_new_db_buf, write_new_db_buflen,
			     write_new_base, fatal))
		result = 0;

	write_new_base = new_db_csize;
	write_new_db_buflen = 0;
	return result;
}


static u_char
write_new_buf(const void *buf, int buflen)
{
	if (write_new_db_buflen + buflen > ISZ(write_new_db_buf)
	    && !write_new_flush(0))
		return 0;

	memcpy(&write_new_db_buf[write_new_db_buflen], buf, buflen);
	write_new_db_buflen += buflen;
	return 1;
}



/* add a record to the new file */
static u_char
write_new_rcd(const void *buf, int buflen)
{
	const u_char zeros[128] = {0};
	DB_PTR new_page_num;
	u_char result;
	int pad, i;

	/* pad accross page boundaries */
	new_page_num = (new_db_csize + buflen)/new_db_page_size;
	if (new_page_num != new_db_csize/new_db_page_size) {
		pad = new_page_num*new_db_page_size - new_db_csize;
		pad = ((pad + DB_RCD_PAD-1) / DB_RCD_PAD) * DB_RCD_PAD;
		do {
			i = sizeof(zeros);
			if (i > pad)
				i = pad;
			if (!write_new_buf(zeros, i))
				return 0;
			pad -= i;
			new_db_csize += i;
		} while (pad != 0);
	}

	result = write_new_buf(buf, buflen);
	new_db_csize += buflen;
	return result;
}



/* write the magic string at the head of the database file */
static void
write_new_magic(const struct timeval *sn)
{
	DB_MAGIC new_magic;
	DCC_CK_TYPES type;
	DCC_TGTS thold;
	int i;

	write_new_flush(1);

	new_magic = db_magic;
	new_magic.s.db_csize = new_db_csize;
	if (grey_on)
		new_magic.s.flags |= DB_MAGIC_ST_GREY;
	if (repair || dccd_started_us) {
		if (old_db_flags & DB_MAGIC_ST_SELF_CLEAN)
			new_magic.s.flags |= DB_MAGIC_ST_SELF_CLEAN2;
		new_magic.s.flags |= DB_MAGIC_ST_SELF_CLEAN;
	}

	for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) {
		thold = new_ex_secs[type].clean_thold;
		if (thold != 0) {
			new_magic.s.ex_secs[type]. clean_thold = thold;
			new_magic.s.ex_secs[type].all = new_ex_secs[type].all;
			new_magic.s.ex_secs[type].spam = new_ex_secs[type].spam;
			memcpy(&new_magic.s.ex_ts[type], &new_ex_ts[type],
			       sizeof(new_magic.s.ex_ts[type]));
		} else {
			new_magic.s.ex_secs[type].all = def_expire_secs;
			new_magic.s.ex_secs[type].spam = (DCC_CK_LONG_TERM(type)
							? def_expire_spamsecs
							: def_expire_secs);
		}
	}
	if (sn) {
		dcc_timeval2ts(new_magic.s.sn, sn, 0);
		new_magic.s.nokeep_cks = old_db_nokeep_cks;

	} else {
		/* Use a zero serial number as a flag that the database
		 * has been purged and the flooding peers should rewind. */
		new_magic.s.nokeep_cks = def_nokeep_cks();
	}
	new_magic.s.page_size = new_db_page_size;
	write_new_db(&new_magic, sizeof(new_magic), 0, 1);

	/* ensure that the last page of the file is complete */
	if (new_db_page_size != 0) {
		i = new_db_fsize % new_db_page_size;
		if (i != 0) {
			if (!db_extend(dcc_emsg, new_db_fd, new_db_nm,
				       new_db_fsize + (new_db_page_size - i),
				       new_db_fsize))
				dcc_logbad(dcc_ex_code, "%s", dcc_emsg);
			new_db_fsize += (new_db_page_size - i);
		}
	}
}



static void
mk_nm(DCC_PATH tgt, const char *src1, const char *src2)
{
	if (ISZ(DCC_PATH) <= snprintf(tgt, sizeof(DCC_PATH),
				      "%s%s", src1, src2))
		dcc_logbad(EX_NOINPUT, "too long DB name \"%s%s\"",
			   src1, src2);
}



static void
unlink_whine(const char *nm)
{
	if (0 > unlink(nm))
		dcc_error_msg("unlink(%s): %s",
			      DCC_NM2PATH(cur_db_nm), ERROR_STR());
}



static void
rename_bail(const char *from, const char *to)
{
	if (0 > rename(from, to))
		dcc_logbad(EX_IOERR, "rename(%s, %s): %s",
			   DCC_NM2PATH(from), to, ERROR_STR());
}



/* try to wait until the server is willing to listen */
static void
server_listening(DCC_AOPS aop, u_int32_t val1)
{
	struct timeval s, now;
	DCC_OPS result;
	DCC_ADMN_RESP_VAL check_buf;
	int check_buflen;
	u_char first;

	gettimeofday(&s, 0);
	first = 1;
	for (;;) {
		/* Ask about status to ensure the server has caught up
		 * and to wait the full measure.  We wait the full
		 * measure instead of giving up early if the server
		 * is persistently sick with the args to dcc_clnt_rdy */
		dcc_ctxts_lock();
		dcc_clnt_rdy(dcc_emsg, ctxt,
			     (grey_on ? DCC_CLNT_FG_GREY : 0)
			     | DCC_CLNT_FG_NO_FAIL);
		dcc_ctxts_unlock();

		check_buflen = sizeof(check_buf);
		result = dcc_aop(dcc_emsg, ctxt, grey_on, -1,
				 DCC_AOP_FLOD, DCC_AOP_FLOD_CHECK, 0, 0, 0,
				 &check_buf, &check_buflen, 0);
		if (result == DCC_OP_ADMN)
			return;

		gettimeofday(&now, 0);

		/* eventually give up and try to open the new database */
		if (db_debug || first
		    || now.tv_sec > s.tv_sec + RESTART_DELAY
		    || now.tv_sec < s.tv_sec) {
			if (result ==  DCC_OP_ERROR
			    || result == DCC_OP_INVALID) {
				dcc_error_msg("%s", dcc_emsg);
			} else {
				dcc_error_msg("%s for %s: %s",
					      dcc_op2str(DCC_OP_ADMN,
							DCC_AOP_FLOD,
							DCC_AOP_FLOD_CHECK),
					      dcc_op2str(DCC_OP_ADMN,
							aop,
							val1),
					      check_buf.string);
			}
			if (now.tv_sec > s.tv_sec + RESTART_DELAY
			    || now.tv_sec < s.tv_sec)
				return;
		}
		sleep(5);
		first = 0;
	}
}



/* tell the daemon to switch to the new database */
static void
dccd_new_db(const char *msg)
{
	if (!dccd_unlocked)
		return;

	/* Send a round of NOPs and ask about status to ensure the server
	 * has dealt with requests that arrived while we had the database
	 * locked and otherwise caught up.  We want to try to ensure that
	 * the server is listening when we re-open the database so that
	 * it does not leave flooding off.
	 * On some systems with lame mmap() support including BSD/OS, the
	 * the daemon can stall for minutes in close(). */
	server_listening(DCC_AOP_DB_NEW, 0);

	dccd_unlocked = 0;
	if (DCC_OP_OK == dcc_aop(dcc_emsg, ctxt, grey_on, -1, DCC_AOP_DB_NEW,
				 0, 0, 0, 0, 0, 0, 0)) {
		/* the daemon assumes a resumption of flooding with
		 * a new database */
		if (flods_off > 0)
			--flods_off;
	} else {
		/* cannot be a fatal error,
		 * lest we leave the database broken */
		dcc_error_msg("%s: %s", msg, dcc_emsg);
	}
}



static void
finish(void)
{
	int bailing = 0;

	/* delete the new files */
#ifndef DCC_DBCLEAN_KEEP_NEW
	if (new_db_created) {
		unlink_whine(new_db_nm);
		new_db_created = 0;
		bailing = -1;
	}
	/* we don't really know if the new hash file was created,
	 * so don't worry about problems */
	if (new_hash_created) {
		unlink_whine(new_hash_nm);
		new_hash_created = 0;
		bailing = -1;
	}
#endif
	if (cur_db_created) {
		unlink_whine(cur_db_nm);
		unlink(cur_hash_nm);
		cur_db_created = 0;
		bailing = -1;
	}

	if (new_db_fd >= 0) {
		if (0 > close(new_db_fd))
			dcc_error_msg("close(%s): %s",
				      DCC_NM2PATH(new_db_nm), ERROR_STR());
		new_db_fd = -1;
	}
	if (old_db_fd >= 0) {
		if (0 > close(old_db_fd))
			dcc_error_msg("close(%s): %s",
				      DCC_NM2PATH(old_db_nm), ERROR_STR());
		old_db_fd = -1;
	}
	flod_unmap(0, 0);

	if (lock_db_fd >= 0) {
		unlink_whine(lock_db_nm);
		close(lock_db_fd);
		lock_db_fd = -1;
	}

	/* release the daemon, but if the database is still open, it's bad */
	db_close(0, bailing);
	dccd_new_db("finish");

	while (flods_off > 0) {
		server_listening(DCC_AOP_FLOD, DCC_AOP_FLOD_RESUME);
		--flods_off;
		if (DCC_OP_OK != dcc_aop(dcc_emsg, ctxt, grey_on, -1,
					 DCC_AOP_FLOD, DCC_AOP_FLOD_RESUME, 0,
					 0, 0, 0, 0, 0)) {
			dcc_error_msg("%s", dcc_emsg);
		}
	}
}



/* terminate with a signal */
static void
sigterm(int s)
{
	dcc_error_msg("interrupted");
	exit(-s);
}