/*
    xml_output.c - XML output dor The WebDruid

    Copyright (C) 2003-2004 Fabien Chevalier (fabien@juliana-multimedia.com)

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version, and provided that the above
    copyright and permission notice is included with all distributed
    copies of this or derived software.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
*/

#include "config.h"

/* ensure sys/types */
#ifndef _SYS_TYPES_H
#include <sys/types.h>
#endif

/* some systems need this */
#ifdef HAVE_MATH_H
#include <math.h>
#endif

#include <time.h>

/* SunOS 4.x Fix */
#ifndef CLK_TCK
#define CLK_TCK _SC_CLK_TCK
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "webdruid.h"
#include "lang.h"
#include "hashtab.h"
#include "linklist.h"
#include "sengine.h"
#include "utils.h"

/* TODO: delete this when it is time...*/
#include "preserve.h"

/*********************************************/
/*  local defines                            */
/*********************************************/

/*
   We won't output more than LIST_MAX items
   in our sorted lists, unless dump_xxx or
   all_xxx is specified
*/
#define LIST_MAX 250


/*********************************************/
/*  local variables                          */
/*********************************************/

/* sort arrays */

static UNODEPTR *u_array      = NULL;                /* Sort array for URL's     */
static HNODEPTR *h_array      = NULL;                /* hostnames (sites)        */
static RNODEPTR *r_array      = NULL;                /* referrers                */
static ANODEPTR *a_array      = NULL;                /* user agents              */
static INODEPTR *i_array      = NULL;                /* ident strings (username) */
static u_long   a_ctr         = 0;                   /* counter for sort array   */

/*********************************************/
/*  local functions                          */
/*********************************************/

static int  common_header(FILE * out_fp);        /* write xml header file         */
static int  write_settings(FILE * out_fp);       /* write common settings in file */
static void output_setting_string(FILE * out_fp, /* write string setting          */
          const char *name, const char *value);

static u_long  load_url_array(  UNODEPTR *);               /* load URL array      */
static u_long  load_site_array( HNODEPTR *);               /* load Site array     */
static u_long  load_ref_array(  RNODEPTR *);               /* load Refs array     */
static u_long  load_agent_array(ANODEPTR *);               /* load Agents array   */
static u_long  load_ident_array(INODEPTR *);               /* load ident array    */
static u_long  load_search_array(SNODEPTR array[],
                  SNODEPTR source_htab[]);
static int qs_url_cmph( const void*, const void*);         /* compare by hits     */
static int qs_url_cmpk( const void*, const void*);         /* compare by kbytes   */
static int qs_url_cmpn( const void*, const void*);         /* compare by entrys   */
static int qs_url_cmpx( const void*, const void*);         /* compare by exits    */
static int qs_site_cmph(const void*, const void*);         /* compare by hits     */
static int qs_site_cmpk(const void*, const void*);         /* compare by kbytes   */
static int qs_ref_cmph( const void*, const void*);         /* compare by hits     */
static int qs_agnt_cmph(const void*, const void*);         /* compare by hits     */
static int qs_ident_cmph(const void*, const void*);        /* compare by hits     */
static int qs_snode_cmp_count(const void *cp1, const void *cp2);    /* compare by hits     */
static int qs_sengines_cmp_count(const void *cp1, const void *cp2);    /* compare by hits     */


/*********************************************/
/*  Writes global statistics                 */
/*********************************************/

int write_history_xml()
{
   int i;
   int year = 0;
   const char *xml_fname = "history.xml";
   FILE * out_fp;

   if (verbose>1)
     printf("%s\n",_("Generating shared XML document..."));

   /* fill in filename */

   /* first, open the file */
   if ( (out_fp = open_out_file_in_dir("xml", xml_fname)) == NULL )
    return 1;

    /* Let's gogogo... */

   if(common_header(out_fp))
   {
      fclose(out_fp);
      return 1;
   };

   fprintf(out_fp, "<report xmlns=\"http://www.webdruid.org/history\">\n");

   if(write_settings(out_fp))
   {
      fclose(out_fp);
      return 1;
   };

   /* Write some months/years values */

   for(i = 0; i < 12; i++)
   {
      if(year != hist_year[i])
      {
         if(year)
         {
            fprintf(out_fp, "</year>\n");
         }

         year = hist_year[i];

         fprintf(out_fp, "<year value=\"%04d\">\n", year);
      }
      if(hist_month[i]){
         fprintf(out_fp, "<month value=\"%02d\"/>\n", hist_month[i]);
      }
   }

   fprintf(out_fp, "</year>\n");

   fprintf(out_fp, "</report>\n");

   fclose(out_fp);
   return 0;
}

/*********************************************/
/*  Writes monthly statistics                */
/*********************************************/

int write_month_xml()
{
   unsigned long i, j;
   char xml_fname[256];           /* filename storage area...       */
   FILE * out_fp;
   struct sengine *sp_sorted_table; /* contains the array of array of sorted search phrases */

   if (verbose>1)
     printf("%s %s %d...\n",_("Generating XML document for"), _(l_month[cur_month-1]), cur_year);

   /* fill in filename */
   snprintf(xml_fname,sizeof(xml_fname),"monthly_%04d_%02d.xml",
            cur_year,cur_month);

   /* first, open the file */
   if ( (out_fp = open_out_file_in_dir("xml", xml_fname)) == NULL )
    return 1;

   /* Let's gogogo... */

   if(common_header(out_fp))
   {
      fclose(out_fp);
      return 1;
   };

   fprintf(out_fp, "<report xmlns=\"http://www.webdruid.org/stats\">\n");

   if(write_settings(out_fp))
   {
      fclose(out_fp);
      return 1;
   };

   /* Keep track of what kind of report it is... */
   fprintf(out_fp, "<grain>month</grain>\n");
   /* Keep track of what month we are... */
   fprintf(out_fp, "<year>%04d</year>\n", cur_year);
   fprintf(out_fp, "<month>%02d</month>\n", cur_month);

   /* Do URL related stuff here, sorting appropriately                      */
   if ((a_ctr = load_url_array(NULL)) != 0)
   {
      if ( (u_array = malloc(sizeof(UNODEPTR)*(a_ctr))) != 0 )
      {
         unsigned long max_output; /* number of items to output */
         unsigned long total_hits = 0;
         unsigned long total_kilobytes = 0;
         unsigned long total_exit = 0;
         unsigned long total_entry = 0;

         a_ctr = load_url_array(u_array);        /* load up our sort array        */

         for(i = 0; i < a_ctr; i++)
         {
            total_hits += u_array[i]->count;
            total_kilobytes += u_array[i]->xfer/1024;
            if (u_array[i]->flag == OBJ_REG)
            {
               if (u_array[i]->entry != 0)
                  total_entry += u_array[i]->entry;
               if (u_array[i]->exit != 0)
                  total_exit += u_array[i]->exit;
            }
         }

         fprintf(out_fp, "<urls>\n");

         fprintf(out_fp, "<count value=\"%lu\"/>\n",
            a_ctr);

         fprintf(out_fp, "<sums hits=\"%lu\" kilobytes=\"%lu\" entries=\"%lu\" exits=\"%lu\"/>\n",
            total_hits, total_kilobytes, total_entry, total_exit);

         if(all_urls || dump_urls)
         {
            fprintf(out_fp, "<list key=\"hits\" all=\"yes\">\n");
            max_output = a_ctr;
         }
         else
         {
            fprintf(out_fp, "<list key=\"hits\">\n");
            max_output = MIN(LIST_MAX, a_ctr);
         }

         qsort(u_array, a_ctr, sizeof(UNODEPTR), qs_url_cmph);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<url name=\"%s\" hits=\"%lu\" kilobytes=\"%.0f\" entries=\"%lu\" exits=\"%lu\"/>\n",
            u_array[i]->string, u_array[i]->count, u_array[i]->xfer/1024, u_array[i]->entry, u_array[i]->exit);
         }

         fprintf(out_fp, "</list>\n");

         /* Top URL's (by kbytes)         */
         fprintf(out_fp, "<list key=\"kilobytes\">\n");
         qsort(u_array,a_ctr,sizeof(UNODEPTR),qs_url_cmpk);
         max_output = MIN(LIST_MAX, a_ctr);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<url name=\"%s\" hits=\"%lu\" kilobytes=\"%.0f\" entries=\"%lu\" exits=\"%lu\"/>\n",
            u_array[i]->string, u_array[i]->count, u_array[i]->xfer/1024, u_array[i]->entry, u_array[i]->exit);
         }
         fprintf(out_fp, "</list>\n");

         /* Top Entry Pages               */
         fprintf(out_fp, "<list key=\"entries\">\n");
         qsort(u_array, a_ctr, sizeof(UNODEPTR), qs_url_cmpn);
         for(i = 0, j = 0; i < a_ctr && j < LIST_MAX; i++)
         {
            if(u_array[i]->flag != OBJ_HIDE)
            {
               fprintf(out_fp, "<url name=\"%s\" hits=\"%lu\" kilobytes=\"%.0f\" entries=\"%lu\" exits=\"%lu\"/>\n",
               u_array[i]->string, u_array[i]->count, u_array[i]->xfer/1024, u_array[i]->entry, u_array[i]->exit);
               j++;
            }
         }
         fprintf(out_fp, "</list>\n");

         /* Top Exit Pages               */
         fprintf(out_fp, "<list key=\"exits\">\n");
         qsort(u_array, a_ctr, sizeof(UNODEPTR), qs_url_cmpx);
         for(i = 0; i < a_ctr && j < LIST_MAX; i++)
         {
            if(u_array[i]->flag != OBJ_HIDE)
            {
               fprintf(out_fp, "<url name=\"%s\" hits=\"%lu\" kilobytes=\"%.0f\" entries=\"%lu\" exits=\"%lu\"/>\n",
               u_array[i]->string, u_array[i]->count, u_array[i]->xfer/1024, u_array[i]->entry, u_array[i]->exit);
               j++;
            }
         }
         fprintf(out_fp, "</list>\n");

         free(u_array);

         fprintf(out_fp, "</urls>\n");
      }
      else if (verbose) fprintf(stderr,"%s [u_array]\n",_("Can't allocate enough memory, Top URLs disabled!")); /* err */
   }

   /* do hostname (sites) related stuff here, sorting appropriately...      */
   if ((a_ctr = load_site_array(NULL)) != 0)
   {
      if ( (h_array = malloc(sizeof(HNODEPTR)*(a_ctr))) != 0 )
      {
         unsigned long max_output; /* number of items to output */
         unsigned long total_hits = 0;
         unsigned long total_files = 0;
         unsigned long total_visits = 0;
         unsigned long total_kilobytes = 0;

         a_ctr = load_site_array(h_array);       /* load up our sort array        */

         for(i = 0; i < a_ctr; i++)
         {
            total_hits += h_array[i]->count;
            total_files += h_array[i]->files;
            total_kilobytes += h_array[i]->xfer/1024;
            total_visits += h_array[i]->visit;
         }

         fprintf(out_fp, "<sites>\n");

         fprintf(out_fp, "<count value=\"%lu\"/>\n",
            a_ctr);

         fprintf(out_fp, "<sums hits=\"%lu\" files=\"%lu\" kilobytes=\"%lu\" visits=\"%lu\"/>\n",
            total_hits, total_files, total_kilobytes, total_visits);

         if(all_sites || dump_sites)
         {
            fprintf(out_fp, "<list key=\"hits\" all=\"yes\">\n");
            max_output = a_ctr;
         }
         else
         {
            fprintf(out_fp, "<list key=\"hits\">\n");
            max_output = MIN(LIST_MAX, a_ctr);
         }

         qsort(h_array,a_ctr,sizeof(HNODEPTR),qs_site_cmph);
         for(i = 0; i < max_output; i++)
         {
               fprintf(out_fp, "<site hostname=\"%s\" hits=\"%lu\" files=\"%lu\" kilobytes=\"%.0f\" visits=\"%lu\"/>\n",
               h_array[i]->string, h_array[i]->count, h_array[i]->files, h_array[i]->xfer/1024, h_array[i]->visit);
         }

         fprintf(out_fp, "</list>\n");

         /* Top Sites table (by kbytes)   */
         fprintf(out_fp, "<list key=\"kilobytes\">\n");
         qsort(h_array,a_ctr,sizeof(HNODEPTR),qs_site_cmpk);
         max_output = MIN(LIST_MAX, a_ctr);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<site hostname=\"%s\" hits=\"%lu\" files=\"%lu\" kilobytes=\"%.0f\" visits=\"%lu\"/>\n",
            h_array[i]->string, h_array[i]->count, h_array[i]->files, h_array[i]->xfer/1024, h_array[i]->visit);
         }
         fprintf(out_fp, "</list>\n");

         free(h_array);

         fprintf(out_fp, "</sites>\n");
      }
      else if (verbose) fprintf(stderr,"%s [h_array]\n",_("Can't allocate enough memory, Top Sites disabled!")); /* err */
   }

   /* do referrer related stuff here, sorting appropriately...              */
   if ((a_ctr = load_ref_array(NULL)) != 0)
   {
      if ((r_array = malloc(sizeof(RNODEPTR)*(a_ctr))) != 0)
      {
         unsigned long max_output; /* number of items to output */
         unsigned long total_hits = 0;

         a_ctr = load_ref_array(r_array);       /* load up our sort array        */

         for(i = 0; i < a_ctr; i++)
            total_hits += r_array[i]->count;

         fprintf(out_fp, "<referrers>\n");

         fprintf(out_fp, "<count value=\"%lu\"/>\n",
            a_ctr);

         fprintf(out_fp, "<sums hits=\"%lu\"/>\n", total_hits);

         if(all_refs || dump_refs)
         {
            fprintf(out_fp, "<list key=\"hits\" all=\"yes\">\n");
            max_output = a_ctr;
         }
         else
         {
            fprintf(out_fp, "<list key=\"hits\">\n");
            max_output = MIN(LIST_MAX, a_ctr);
         }

         qsort(r_array, a_ctr, sizeof(RNODEPTR), qs_ref_cmph);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<referrer url=\"%s\" hits=\"%lu\"/>\n",
            r_array[i]->string, r_array[i]->count);
         }
         fprintf(out_fp, "</list>\n");

         free(r_array);

         fprintf(out_fp, "</referrers>\n");
     }
        else if (verbose) fprintf(stderr,"%s [r_array]\n",_("Can't allocate enough memory, Top Referrers disabled!")); /* err */
   }

   /* do ident (username) related stuff here, sorting appropriately...      */
   if((a_ctr = load_ident_array(NULL)) != 0)
   {
      if ((i_array = malloc(sizeof(RNODEPTR)*(a_ctr))) != 0)
      {
         unsigned long max_output; /* number of items to output */
         unsigned long total_hits = 0;

         a_ctr = load_ident_array(i_array);       /* load up our sort array        */

         for(i = 0; i < a_ctr; i++)
            total_hits += i_array[i]->count;

         fprintf(out_fp, "<usernames>\n");

         fprintf(out_fp, "<count value=\"%lu\"/>\n",
            a_ctr);

         fprintf(out_fp, "<sums hits=\"%lu\"/>\n", total_hits);

         if(all_users || dump_users)
         {
            fprintf(out_fp, "<list key=\"hits\" all=\"yes\">\n");
            max_output = a_ctr;
         }
         else
         {
            fprintf(out_fp, "<list key=\"hits\">\n");
            max_output = MIN(LIST_MAX, a_ctr);
         }

         qsort(i_array, a_ctr, sizeof(INODEPTR), qs_ident_cmph);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<username url=\"%s\" hits=\"%lu\"/>\n",
            i_array[i]->string, i_array[i]->count);
         }
         fprintf(out_fp, "</list>\n");

         free(i_array);

         fprintf(out_fp, "</usernames>\n");
     }
         else if (verbose) fprintf(stderr,"%s [i_array]\n",_("Can't allocate enough memory, Top Usernames disabled!")); /* err */
   }

   /* do user agent related stuff here, sorting appropriately...            */
   if ((a_ctr = load_agent_array(NULL)) != 0)
   {
      if ((a_array = malloc(sizeof(ANODEPTR)*(a_ctr))) != 0)
      {
         unsigned long max_output; /* number of items to output */
         unsigned long total_hits = 0;

         a_ctr = load_agent_array(a_array);       /* load up our sort array        */

         for(i = 0; i < a_ctr; i++)
            total_hits += a_array[i]->count;

         fprintf(out_fp, "<agents>\n");

         fprintf(out_fp, "<count value=\"%lu\"/>\n",
            a_ctr);

         fprintf(out_fp, "<sums hits=\"%lu\"/>\n", total_hits);

         if(all_agents || dump_agents)
         {
            fprintf(out_fp, "<list key=\"hits\" all=\"yes\">\n");
            max_output = a_ctr;
         }
         else
         {
            fprintf(out_fp, "<list key=\"hits\">\n");
            max_output = MIN(LIST_MAX, a_ctr);
         }

         qsort(a_array, a_ctr, sizeof(ANODEPTR), qs_agnt_cmph);
         for(i = 0; i < max_output; i++)
         {
            fprintf(out_fp, "<username url=\"%s\" hits=\"%lu\"/>\n",
            a_array[i]->string, a_array[i]->count);
         }
         fprintf(out_fp, "</list>\n");

         free(a_array);

         fprintf(out_fp, "</agents>\n");
     }
         else if (verbose) fprintf(stderr,"%s [a_array]\n",_("Can't allocate enough memory, Top User Agents disabled!")); /* err */
   }

   /* let's malloc it -- and do it's initialisation with a copy of sp_table */
   sp_sorted_table = malloc(sizeof(struct sengine) * n_sengines);

   if(sp_sorted_table == NULL)
   {
      if (verbose)
         fprintf(stderr,"[generate_sengines_info] %s\n",_("Can't allocate enough memory, Search engines report disabled!")); /* err */
   }
   else
   {
      unsigned long total_searches = 0;

      memcpy(sp_sorted_table, sp_table, sizeof(struct sengine) * n_sengines);

      /* compute the number of searches for all engines */
      for(i=0; i<n_sengines; i++)
         total_searches += sp_table[i].count;

      /* Let's fill our array of array */
      for(i=0; i<n_sengines; i++)
      {
         if(sp_table[i].count)
         {
            /* Do load the array */
            u_long size = load_search_array(NULL, sp_table[i].s_htab);

            sp_sorted_table[i].s_htab = malloc((size + 1) * sizeof(SNODEPTR));
            if(sp_sorted_table[i].s_htab == NULL) /* no more memory */
            {
               if (verbose)
                  fprintf(stderr,"[generate_sengines_info] %s\n",_("Can't allocate enough memory, Search engines report will be incomplete!")); /* err */
               /* force it to be ignored */
               sp_sorted_table[i].s_htab = NULL;
               sp_sorted_table[i].count = 0;
            }
            else
            {
               /* if memory allocated, fill it and sort it */
               load_search_array(sp_sorted_table[i].s_htab, sp_table[i].s_htab);
               qsort(sp_sorted_table[i].s_htab, size, sizeof(SNODEPTR), qs_snode_cmp_count);
               /* end array with null */
               sp_sorted_table[i].s_htab[size] = NULL;
            }
         }
         else
            sp_sorted_table[i].s_htab = NULL;
      }

      /* now we sort the array of array */

      qsort(sp_sorted_table, n_sengines, sizeof(struct sengine), qs_sengines_cmp_count);

      fprintf(out_fp, "<sengines>\n");
      fprintf(out_fp, "<sums hits=\"%lu\"/>\n", total_searches);
      /* iterate through sengines */
      for(i = 0; i < n_sengines; i++)
      {
         if(sp_sorted_table[i].count != 0)
         {
            int j;
            LISTPTR lptr;

            fprintf(out_fp, "<sengine name=\"%s\">\n", sp_sorted_table[i].display_name);
            fprintf(out_fp, "<sums hits=\"%lu\"/>\n", sp_sorted_table[i].count);

            /* iterate through search phrases */
            for(j = 0; j < LIST_MAX && sp_sorted_table[i].s_htab[j] != NULL; j++)
            {
               fprintf(out_fp, "<search phrase=\"%s\">\n",
                  sp_sorted_table[i].s_htab[j]->string);
               fprintf(out_fp, "<sums hits=\"%lu\"/>\n",
                  sp_sorted_table[i].s_htab[j]->count);
               /* iterate through search urls */
               lptr = sp_sorted_table[i].s_htab[j]->urls;
               while(lptr != NULL)
               {
                  fprintf(out_fp, "<url name=\"%s\"/>\n", (char *)lptr->item);
                  lptr = lptr->next;
               }
               fprintf(out_fp, "</search>\n");
            }
            fprintf(out_fp, "</sengine>\n");
         }
      }
      fprintf(out_fp, "</sengines>\n");

      /* cleanup */
      for(i=0; i < n_sengines; i++)
         free(sp_sorted_table[i].s_htab);

      free(sp_sorted_table);
   }

   fprintf(out_fp, "</report>\n");

   fclose(out_fp);                        /* close the file                 */
   return 0;                              /* done...                        */
}


/*********************************************/
/*  WRITE_SETTINGS                           */
/*********************************************/

/*
   Outputs settings
*/

static int write_settings(FILE * out_fp)
{
   /* Record WebDruid's settings */
   fprintf(out_fp, "<settings>\n");

   /* Give hostname */
   output_setting_string(out_fp, "HostName", hname);

   /* Give version */
   output_setting_string(out_fp, "Version", version);

   fprintf(out_fp, "</settings>\n");

   return 0;
}

/*********************************************/
/*  COMMON_HEADER                            */
/*********************************************/

/*
   Outputs header in xml file
*/

static int common_header(FILE * out_fp)
{
   time_t curtime = time(0);

   fprintf(out_fp, "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
   fprintf(out_fp,"<!-- Generated by The WebDruid  v %s       -->\n",
      version);
   fprintf(out_fp,"<!-- (http://www.webdruid.org)                -->\n");
   fprintf(out_fp,"<!--                                          -->\n");
   fprintf(out_fp,"<!-- Copyright 2003-2004 Fabien Chevalier     -->\n");
   fprintf(out_fp,"<!-- (fabien@juliana-multimedia.com)          -->\n");
   fprintf(out_fp,"<!--                                          -->\n");
   fprintf(out_fp,"<!-- Distributed under the GNU GPL  Version 2 -->\n");
   fprintf(out_fp,"<!--                                          -->\n");
   fprintf(out_fp,"<!--                                          -->\n");
   fprintf(out_fp,"<!-- *** Generated: %s *** -->\n\n", ctime(&curtime));

   return 0;
}

/*********************************************/
/*  output_setting_string  - straightforward */
/*********************************************/

static void output_setting_string(FILE * out_fp, const char *name, const char *value)
{
   fprintf(out_fp, "<setting name=\"%s\"><value>%s</value></setting>\n", name, value);
}

/*********************************************/
/* LOAD_SITE_ARRAY - load up the sort array  */
/*********************************************/

static u_long load_site_array(HNODEPTR *pointer)
{
   HNODEPTR hptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0;i<MAXHASH;i++)
   {
      hptr=sm_htab[i];
      while (hptr!=NULL)
      {
         if (pointer==NULL) ctr++;       /* fancy way to just count 'em    */
         else *(pointer+ctr++)=hptr;     /* otherwise, really do the load  */
         hptr=hptr->next;
      }
   }
   return ctr;   /* return number loaded */
}

/*********************************************/
/* LOAD_URL_ARRAY - load up the sort array   */
/*********************************************/

static u_long load_url_array(UNODEPTR *pointer)
{
   UNODEPTR uptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0;i<MAXHASH;i++)
   {
      uptr=um_htab[i];
      while (uptr!=NULL)
      {
         if (pointer==NULL) ctr++;       /* fancy way to just count 'em    */
         else *(pointer+ctr++)=uptr;     /* otherwise, really do the load  */
         uptr=uptr->next;
      }
   }
   return ctr;   /* return number loaded */
}

/*********************************************/
/* LOAD_REF_ARRAY - load up the sort array   */
/*********************************************/

static u_long load_ref_array(RNODEPTR *pointer)
{
   RNODEPTR rptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0;i<MAXHASH;i++)
   {
      rptr=rm_htab[i];
      while (rptr!=NULL)
      {
         if (pointer==NULL) ctr++;       /* fancy way to just count 'em    */
         else *(pointer+ctr++)=rptr;     /* otherwise, really do the load  */
         rptr=rptr->next;
      }
   }
   return ctr;   /* return number loaded */
}

/*********************************************/
/* LOAD_AGENT_ARRAY - load up the sort array */
/*********************************************/

static u_long load_agent_array(ANODEPTR *pointer)
{
   ANODEPTR aptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0;i<MAXHASH;i++)
   {
      aptr=am_htab[i];
      while (aptr!=NULL)
      {
         if (pointer==NULL) ctr++;       /* fancy way to just count 'em    */
         else *(pointer+ctr++)=aptr;     /* otherwise, really do the load  */
         aptr=aptr->next;
      }
   }
   return ctr;   /* return number loaded */
}

/*********************************************/
/* LOAD_IDENT_ARRAY - load up the sort array */
/*********************************************/

static u_long load_ident_array(INODEPTR *pointer)
{
   INODEPTR iptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0;i<MAXHASH;i++)
   {
      iptr=im_htab[i];
      while (iptr!=NULL)
      {
         if (pointer==NULL) ctr++;       /* fancy way to just count 'em    */
         else *(pointer+ctr++)=iptr;     /* otherwise, really do the load  */
         iptr=iptr->next;
      }
   }
   return ctr;   /* return number loaded */
}

/**********************************************/
/* LOAD_SEARCH_ARRAY - load up the sort array */
/**********************************************/

static u_long load_search_array(SNODEPTR array[], SNODEPTR source_htab[])
{
   SNODEPTR sptr;
   int      i;
   u_long   ctr = 0;

   /* load the array */
   for (i=0; i<MAXSPHASH; i++)
   {
      sptr=source_htab[i];
      while (sptr!=NULL)
      {
         if (array==NULL) ctr++;     /* fancy way to just count 'em   */
         else array[ctr++]=sptr;     /* otherwise, really do the load */
         sptr=sptr->next;
      }
   }
   return ctr;   // return number loaded*/
   return 0;
}

/*********************************************/
/* QS_SITE_CMPH - QSort compare site by hits */
/*********************************************/

static int qs_site_cmph(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(HNODEPTR *)cp1)->count;
   t2=(*(HNODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by hostname instead */
   return strcmp( (*(HNODEPTR *)cp1)->string,
                  (*(HNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_SITE_CMPK - QSort cmp site by bytes    */
/*********************************************/

static int qs_site_cmpk(const void *cp1, const void *cp2)
{
   double t1, t2;
   t1=(*(HNODEPTR *)cp1)->xfer;
   t2=(*(HNODEPTR *)cp2)->xfer;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if xfer bytes are the same, we sort by hostname instead */
   return strcmp( (*(HNODEPTR *)cp1)->string,
                  (*(HNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_URL_CMPH - QSort compare URL by hits   */
/*********************************************/

static int qs_url_cmph(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(UNODEPTR *)cp1)->count;
   t2=(*(UNODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by url instead */
   return strcmp( (*(UNODEPTR *)cp1)->string,
                  (*(UNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_URL_CMPK - QSort compare URL by bytes  */
/*********************************************/

static int qs_url_cmpk(const void *cp1, const void *cp2)
{
   double t1, t2;
   t1=(*(UNODEPTR *)cp1)->xfer;
   t2=(*(UNODEPTR *)cp2)->xfer;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if xfer bytes are the same, we sort by url instead */
   return strcmp( (*(UNODEPTR *)cp1)->string,
                  (*(UNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_URL_CMPN - QSort compare URL by entry  */
/*********************************************/

static int qs_url_cmpn(const void *cp1, const void *cp2)
{
   double t1, t2;
   t1=(*(UNODEPTR *)cp1)->entry;
   t2=(*(UNODEPTR *)cp2)->entry;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if xfer bytes are the same, we sort by url instead */
   return strcmp( (*(UNODEPTR *)cp1)->string,
                  (*(UNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_URL_CMPX - QSort compare URL by exit   */
/*********************************************/

static int qs_url_cmpx(const void *cp1, const void *cp2)
{
   double t1, t2;
   t1=(*(UNODEPTR *)cp1)->exit;
   t2=(*(UNODEPTR *)cp2)->exit;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if xfer bytes are the same, we sort by url instead */
   return strcmp( (*(UNODEPTR *)cp1)->string,
                  (*(UNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_REF_CMPH - QSort compare Refs by hits  */
/*********************************************/

static int qs_ref_cmph(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(RNODEPTR *)cp1)->count;
   t2=(*(RNODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by referrer URL instead */
   return strcmp( (*(RNODEPTR *)cp1)->string,
                  (*(RNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_AGNT_CMPH - QSort cmp Agents by hits   */
/*********************************************/

static int qs_agnt_cmph(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(ANODEPTR *)cp1)->count;
   t2=(*(ANODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by agent string instead */
   return strcmp( (*(ANODEPTR *)cp1)->string,
                  (*(ANODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_IDENT_CMPH - QSort cmp ident by hits   */
/*********************************************/

static int qs_ident_cmph(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(INODEPTR *)cp1)->count;
   t2=(*(INODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, sort by ident (username) string instead */
   return strcmp( (*(INODEPTR *)cp1)->string,
                  (*(INODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_SNODE_CMP_COUNT                        */
/*********************************************/

static int qs_snode_cmp_count(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=(*(SNODEPTR *)cp1)->count;
   t2=(*(SNODEPTR *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by search string instead */
   return strcmp( (*(SNODEPTR *)cp1)->string,
                  (*(SNODEPTR *)cp2)->string );
}

/*********************************************/
/* QS_SENGINES_CMP_COUNT                     */
/*********************************************/

static int qs_sengines_cmp_count(const void *cp1, const void *cp2)
{
   u_long  t1, t2;
   t1=((struct sengine *)cp1)->count;
   t2=((struct sengine *)cp2)->count;
   if (t1!=t2) return (t2<t1)?-1:1;
   /* if hits are the same, we sort by search string instead */
   return strcmp( ((struct sengine *)cp1)->display_name,
                  ((struct sengine *)cp2)->display_name );
}