#!/usr/bin/perl
# BackupPC_refCountUpdate: Pool reference count updater
#
# DESCRIPTION
#
#   BackupPC_refCountUpdate checks the pool reference counts
#
#   Usage: BackupPC_refCountUpdate
#
# AUTHOR
#   Craig Barratt  <cbarratt@users.sourceforge.net>
#
# COPYRIGHT
#   Copyright (C) 2001-2020  Craig Barratt
#
#   This program is free software: you can redistribute it and/or modify
#   it under the terms of the GNU General Public License as published by
#   the Free Software Foundation, either version 3 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#========================================================================
#
# Version 4.4.0, released 20 Jun 2020.
#
# See http://backuppc.sourceforge.net.
#
#========================================================================

use strict;
no utf8;

use lib "__INSTALLDIR__/lib";

use Getopt::Std;
use Fcntl qw(:mode);
use File::Path;
use Data::Dumper;

use BackupPC::Lib;
use BackupPC::XS;
use BackupPC::DirOps qw( :BPC_DT_ALL );

select(STDOUT); $| = 1;

my %opts;
if ( !getopts("cFfnmpsvh:r:P:o:", \%opts) || @ARGV != 0 || !(defined($opts{h}) || $opts{m}) ) {
    print <<EOF;
Usage:
  BackupPC_refCountUpdate -h HOST [-c] [-f] [-F] [-o N] [-p] [-v]
      With no other args, updates count db on backups with poolCntDelta files
      and computes the host's total reference counts.  Also builds refCnt for
      any >=4.0 backups without refCnts.
        -f     - do an fsck on this HOST, which involves a rebuild of the
                 last two backup refCnts.  poolCntDelta files are ignored.
                 Also forces fsck if requested by needFsck flag files
                 in TopDir/pc/HOST/refCnt.  Equivalent to -o 2.
        -F     - rebuild all the >=4.0 per-backup refCnt files for this
                 host.  Equivalent to -o 3.
        -c     - compare current count db to new db before replacing
        -o N   - override \$Conf{RefCntFsck}.
        -p     - don't show progress
        -v     - verbose
    Notes: in case there are legacy (ie: <=4.0.0alpha3) unapplied poolCntDelta
    files in TopDir/pc/HOST/refCnt then the -f flag is turned on.

  BackupPC_refCountUpdate -m [-f] [-p] [-c] [-r N-M] [-s] [-v] [-P phase]
        -m       Updates main count db, based on each HOST
        -f     - do an fsck on all the hosts, ignoring poolCntDelta files,
                 and replacing each host's count db.  Will wait for backups
                 to finish if any are running.
        -F     - rebuild all the >=4.0 per-backup refCnt files.
        -p     - don't show progress
        -c     - clean pool files
        -r N-M - process a subset of the main count db, 0 <= N <= M <= 255
        -s     - prints stats
        -v     - verbose
        -P phase Phase from 0..15 each time we run BackupPC_nightly.  Used
                 to compute exact pool size for portions of the pool based
                 on the phase and \$Conf{PoolSizeNightlyUpdatePeriod}.
EOF
    exit(1);
}

my $ErrorCnt    = 0;
my $refCntStart = 0;
my $refCntEnd   = 127;

my $EmptyMD5 = pack("H*", "d41d8cd98f00b204e9800998ecf8427e");

die("BackupPC::Lib->new failed\n") if ( !(my $bpc = BackupPC::Lib->new) );
my $TopDir = $bpc->TopDir();
my $BinDir = $bpc->BinDir();
my $Hosts  = $bpc->HostInfoRead();
my @Hosts  = sort(keys(%$Hosts));

if ( $opts{h} ne "" ) {
    if ( !defined($Hosts->{$opts{h}}) ) {
        print(STDERR "BackupPC_refCountUpdate: host $opts{h} doesn't exist\n");
        exit(1);
    }
    $bpc->ConfigRead($opts{h});
}
my %Conf      = $bpc->Conf();
my $PoolStats = {};

if ( $opts{v} ) {
    $Conf{XferLogLevel} = 8;
    BackupPC::XS::Lib::logLevelSet(8);
}

#
# Write new-style attrib files (<= 4.0.0beta3 uses old-style), which are 0-length
# files with the digest encoded in the file name (eg: attrib_md5HexDigest). We
# can still read the old-style files, and we upgrade them as we go.
#
BackupPC::XS::Attrib::backwardCompat(0, 0);

print("__bpc_pidStart__ $$\n") if ( !$opts{p} );

if ( defined($opts{h}) ) {
    updateHostPoolCnt($opts{h}, $opts{f}, $opts{F}, $opts{c});
} elsif ( $opts{m} ) {
    if ( $opts{r} =~ /^(\d+)-(\d+)$/ && 0 <= $1 && $1 <= $2 && $2 <= 255 ) {
        $refCntStart = int($1 / 2);
        $refCntEnd   = int($2 / 2);
    } elsif ( defined($opts{r}) ) {
        print(STDERR "BackupPC_refCountUpdate: -r arguments should be N-M where 0 <= N <= M <= 255\n");
        print("__bpc_pidEnd__ $$\n") if ( !$opts{p} );
        exit(1);
    }

    #
    # update all the pool count changes for each host
    #
    if ( $opts{f} || $opts{F} ) {
        updateAllHostPoolCnt($opts{f}, $opts{F});
    }

    #
    # rebuild the overall pool count database
    #
    updateTotalRefCnt();

    #
    # clean files from the pool that are no longer referenced,
    # and mark candidate files for future cleaning.  This prints
    # stats too, so if not cleaning, we need to print stats, if
    # requested.
    #
    if ( $opts{c} ) {
        cleanPoolFiles();
    } elsif ( $opts{s} ) {
        statsPrint();
    }
}

print("BackupPC_refCountUpdate total errors: $ErrorCnt\n") if ( $ErrorCnt > 0 || $Conf{XferLogLevel} >= 2 );

print("__bpc_pidEnd__ $$\n") if ( !$opts{p} );
exit($ErrorCnt ? 1 : 0);

#
# For a given host, either update the poolCnt files using the poolCntDelta files,
# or if $forceFsck, completely rebuild the poolCnt files from scratch.
#
sub updateHostPoolCnt
{
    my($host, $forceLast2Fsck, $forceFullFsck, $compareCurr) = @_;
    my $pcDir        = "$TopDir/pc/$host";
    my $refCntDir    = "$pcDir/refCnt";
    my $errorCntSave = $ErrorCnt;
    my @fsckFiles;
    my @Backups    = $bpc->BackupInfoRead($host);
    my $startClock = time;

    mkdir($refCntDir, 0770) if ( !-d $refCntDir );

    #
    # Grab the lock to make sure no dumps start.  Try a non-blocking request
    # first, so we can warn the user if we have to wait.
    #
    my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 0);
    if ( $lockFd < 0 ) {
        print("Backup running on host $host.... waiting\n");
        $lockFd = BackupPC::XS::DirOps::lockRangeFile("$refCntDir/LOCK", 0, 1, 1);
    }
    if ( $lockFd < 0 ) {
        print("Unable to get lock on $refCntDir/LOCK... skipping $host\n");
        $ErrorCnt++;
        return;
    }

    #
    # Clean up the HOST/refCnt directory to see if an fsck is needed
    #
    foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $refCntDir)} ) {
        unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ );
        if ( $e->{name} =~ /^t?poolCntDelta/ ) {
            #
            # legacy <= 4.0.0beta3 code left poolCntDelta files in the host's refCnt
            # directory, so remove them and do a last2Fsck
            #
            unlink("$refCntDir/$e->{name}");
            $forceLast2Fsck = 1;
        }
        next if ( $e->{name} !~ /^needFsck/ );
        unlink("$refCntDir/$e->{name}");
        $forceLast2Fsck = 1;
    }

    #
    # Apply policy in $Conf{RefCntFsck} (potentially overridden by -o N)
    #   0: no additional fsck
    #   1: do an fsck on the last backup if it is from a full backup (handled below)
    #   2: do an fsck on the last two backups always
    #   3: do a full fsck on all the backups
    #
    my $ConfRefCntFsck = $opts{o} =~ /^\d+$/ ? $opts{o} : $Conf{RefCntFsck};
    if ( $ConfRefCntFsck == 2 ) {
        print("BackupPC_refCountUpdate: doing fsck on last two backups on $host since \$ConfRefCntFsck == 2\n");
        $forceLast2Fsck = 1;
    } elsif ( $ConfRefCntFsck == 3 ) {
        print("BackupPC_refCountUpdate: doing fsck on all backups on $host since \$ConfRefCntFsck == 3\n");
        $forceFullFsck = 1;
    }

    #
    # Figure out which backups to rebuild or apply to deltas
    #
    my $bkupList = [];
    for ( my $i = 0 ; $i < @Backups ; $i++ ) {
        my $bkupNum       = $Backups[$i]{num};
        my $compress      = $Backups[$i]{compress};
        my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt";
        my($gotFsck, $gotDelta, $gotPoolCnt, $gotNoPoolCntOk);

        next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $bkupRefCntDir) );

        if ( !-d $bkupRefCntDir ) {
            print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since $bkupRefCntDir doesn't exist\n");
            mkdir($bkupRefCntDir, 0770);
            push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compress});
            next;
        }
        my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir);
        foreach my $e ( @$entries ) {
            unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ );
            if ( $e->{name} =~ /^needFsck/ || $e->{name} =~ /^tpoolCntDelta/ ) {
                unlink("$bkupRefCntDir/$e->{name}");
                $gotFsck = 1;
            }
            $gotDelta       = 1 if ( $e->{name} =~ /^poolCntDelta/ );
            $gotPoolCnt     = 1 if ( $e->{name} =~ /^poolCnt\./ );
            $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ );
        }
        print(  "BackupPC_refCountUpdate: host $host #$bkupNum: gotFsck = $gotFsck, gotDelta = $gotDelta,"
              . " gotPoolCnt = $gotPoolCnt, gotNoPoolCntOk = $gotNoPoolCntOk\n")
          if ( $Conf{XferLogLevel} >= 4 );
        #
        # Apply policy in $ConfRefCntFsck:
        #   0: no additional fsck
        #   1: do an fsck on the last backup if it is from a full backup
        #   2: do an fsck on the last two backups always (handled above)
        #   3: do a full fsck on all the backups (handled above)
        #
        if ( $ConfRefCntFsck == 1 && $i == @Backups - 1 && $Backups[$i]{type} eq "full" ) {
            print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum (full) since \$ConfRefCntFsck == 1\n");
            $gotFsck = 1;
        }

        if ( !$gotPoolCnt && !$gotFsck && !$gotNoPoolCntOk ) {
            #
            # This shouldn't happen - we should have some pool files, so force an fsck
            #
            print("BackupPC_refCountUpdate: doing fsck on $host #$bkupNum since there are no poolCnt files\n");
            $gotFsck = 1;
        }
        if ( $forceFullFsck || ($forceLast2Fsck && $i >= @Backups - 2) || $gotFsck ) {
            push(@$bkupList, {bkupNum => $bkupNum, fsck => 1, compress => $compress});
        } elsif ( $gotDelta ) {
            push(@$bkupList, {bkupNum => $bkupNum, fsck => 0, compress => $compress});
        }
    }

    while ( (my $b = shift(@$bkupList)) ) {
        my $errorCntSave0 = $ErrorCnt;
        if ( processOneHostBackup($host, $b->{bkupNum}, $b->{fsck}, $b->{compress}) && !$b->{fsck} ) {
            print(
                "BackupPC_refCountUpdate: given errors, redoing host $host #$b->{bkupNum} with fsck (reset errorCnt to $errorCntSave0)\n"
            );
            $ErrorCnt = $errorCntSave0;
            unshift(@$bkupList, {bkupNum => $b->{bkupNum}, fsck => 1, compress => $b->{compress}});
        }
    }

    #
    # Now add up all the per-backup counts to get the per-host totals
    #
    my $errorCntSave0 = $ErrorCnt;
    print("BackupPC_refCountUpdate: computing totals for host $host\n") if ( $Conf{XferLogLevel} >= 2 );
    my $needFsck     = updateHostRefCounts($host);
    my $bkupRedoDone = {};

    while ( $needFsck ) {
        my $listText = join(", ", @$needFsck);
        print(
            "BackupPC_refCountUpdate: due to errors, doing fsck on host $host backups $listText; (reset errorCnt to $errorCntSave0)\n"
        );
        $ErrorCnt = $errorCntSave0;
        foreach my $bkupNum ( @$needFsck ) {
            next if ( $bkupRedoDone->{$bkupNum} );
            my $compress = 1;
            for ( my $i = 0 ; $i < @Backups ; $i++ ) {
                next if ( $Backups[$i]{num} != $bkupNum );
                $compress = $Backups[$i]{compress};
                last;
            }
            processOneHostBackup($host, $bkupNum, 1, $compress);
            $bkupRedoDone->{$bkupNum} = 1;
        }
        $needFsck = updateHostRefCounts($host);
    }

    #
    # and rename the new count files to replace the current ones
    #
    renameNewCntFiles($host, $refCntDir, $compareCurr, "total");

    #
    # Now give the lock back
    #
    BackupPC::XS::DirOps::unlockRangeFile($lockFd);
    printf(
        "BackupPC_refCountUpdate: host %s got %d errors (took %d secs)\n",
        $host,
        $ErrorCnt - $errorCntSave,
        time - $startClock
    );
    $bpc->flushXSLibMesgs();
}

sub processOneHostBackup
{
    my($host, $bkupNum, $fsck, $compress) = @_;
    my $pcDir         = "$TopDir/pc/$host";
    my $bkupRefCntDir = "$pcDir/$bkupNum/refCnt";
    my $errorCntSave0 = $ErrorCnt;
    my $fd;

    #
    # add a placeholder fsck request, in case we die unexpectedly
    #
    open($fd, ">", "$bkupRefCntDir/needFsck.refCountUpdate") && close($fd);

    print("BackupPC_refCountUpdate: processing host $host #$bkupNum (fsck = $fsck)\n") if ( $Conf{XferLogLevel} >= 2 );

    if ( $fsck ) {
        my $entries = BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir);
        #
        # Remove any delta files since we are rebuilding
        #
        foreach my $e ( @$entries ) {
            unlink("$bkupRefCntDir/$e->{name}") if ( $e->{name} =~ /^t?poolCntDelta/ || $e->{name} =~ /^poolCntNew/ );
        }
        my $deltaInfo = BackupPC::XS::DeltaRefCnt::new("$pcDir/$bkupNum");
        print("__bpc_progress_state__ refCnt #$bkupNum\n") if ( !$opts{p} );
        my($err, $inodeMax) = BackupPC::XS::DirOps::refCountAllInodeMax("$pcDir/$bkupNum", $compress, 1, $deltaInfo);
        $ErrorCnt += $err;
        $deltaInfo->flush();
        $bpc->flushXSLibMesgs();
        #
        # Update inodeLast for this backup if it is larger than what's stored in backups
        #
        my @backups = $bpc->BackupInfoRead($host);
        for ( my $i = 0 ; $i < @backups ; $i++ ) {
            next if ( $backups[$i]{num} != $bkupNum || $backups[$i]{inodeLast} > $inodeMax );
            $inodeMax += 2;
            print(
                "BackupPC_refCountUpdate: $host #$bkupNum inodeLast set to $inodeMax (was $backups[$i]{inodeLast})\n");
            $backups[$i]{inodeLast} = $inodeMax;
            BackupPC::Storage->backupInfoWrite($pcDir, $bkupNum, $backups[$i], 1);
            $bpc->BackupInfoWrite($host, @backups);
            last;
        }
    }
    print("__bpc_progress_state__ cntUpdate #$bkupNum\n") if ( !$opts{p} );
    updateHostDelta2Cnt($host, $bkupRefCntDir, $fsck, 0, $bkupNum);
    $bpc->flushXSLibMesgs();

    #
    # delete the noPoolCntOk if there are pool files, or create one
    # if there are none
    #
    my($gotNoPoolCntOk, $gotPoolCnt);
    foreach my $e ( @{BackupPC::DirOps::dirRead($bpc, $bkupRefCntDir)} ) {
        $gotPoolCnt     = 1 if ( $e->{name} =~ /^poolCnt\./ );
        $gotNoPoolCntOk = 1 if ( $e->{name} =~ /^noPoolCntOk/ );
    }
    if ( $gotPoolCnt && $gotNoPoolCntOk ) {
        unlink("$bkupRefCntDir/noPoolCntOk");
    } elsif ( !$gotPoolCnt ) {
        open(my $fd, ">", "$bkupRefCntDir/noPoolCntOk") && close($fd);
    }
    unlink("$bkupRefCntDir/needFsck.refCountUpdate");
    return 1 if ( $errorCntSave0 != $ErrorCnt );
}

#
# Process all the per-backup ref counts to create the total ref count for the host.
# Returns undef on success, or a listref of backup numbers that need fscks.
#
sub updateHostRefCounts
{
    my($host) = @_;

    my @Backups       = $bpc->BackupInfoRead($host);
    my $refCntDestDir = "$TopDir/pc/$host/refCnt";
    my $needFsck      = {};

    print("__bpc_progress_state__ sumUpdate\n") if ( !$opts{p} );
    for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
        print("__bpc_progress_fileCnt__ $refCntFile/128\n") if ( !$opts{p} && ($refCntFile & 0x7) == 0 );
        for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
            my $count = BackupPC::XS::PoolRefCnt::new();
            my $countDirty;
            my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDestDir, $compress, $refCntFile * 2);
            for ( my $i = 0 ; $i < @Backups ; $i++ ) {
                my $bkupNum       = $Backups[$i]{num};
                my $refCntBkupDir = "$TopDir/pc/$host/$bkupNum/refCnt";
                next if ( $Backups[$i]{version} eq "" || ($Backups[$i]{version} =~ /^[23]\./ && !-d $refCntBkupDir) );
                my $poolCntHostFile = sprintf("%s/poolCnt.%d.%02x", $refCntBkupDir, $compress, $refCntFile * 2);
                next if ( !-f $poolCntHostFile );

                my $countHost = BackupPC::XS::PoolRefCnt::new();
                if ( $countHost->read($poolCntHostFile) ) {
                    print("Can't open pool count file $poolCntHostFile\n");
                    $ErrorCnt++;
                    next;
                }

                my($d, $c);
                my $idx = 0;
                while ( 1 ) {
                    ($d, $c, $idx) = $countHost->iterate($idx);
                    last if ( !defined($d) );

                    if ( $c < 0 ) {
                        printf(
                            "BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d); will do fsck on #%d\n",
                            $host, $poolCntHostFile, unpack("H*", $d),
                            $c, $bkupNum
                        );
                        $needFsck->{$bkupNum}++;
                        $ErrorCnt++;
                    } elsif ( $c > 0 ) {
                        $count->incr($d, $c);
                        $countDirty = 1;
                    }
                }
            }
            if ( $countDirty && $count->write($poolCntFileNew) ) {
                print("Can't write new host pool count file $poolCntFileNew\n");
                $ErrorCnt++;
            }
            $count = undef;
            $bpc->flushXSLibMesgs();
        }
    }
    if ( keys(%$needFsck) ) {
        return [sort({ $a <=> $b } keys(%$needFsck))];
    }
}

#
# Process all the delta files in the given directory and update the poolCnt files
#
sub updateHostDelta2Cnt
{
    my($host, $refCntDir, $forceFsck, $compareCurr, $bkupNum) = @_;
    my $gotDeltas;

    #
    # Read all the poolCntDelta files, and update the host's poolCnt files
    # (or overwrite them if $forceFsck)
    #
    my $entries = BackupPC::DirOps::dirRead($bpc, $refCntDir);
    foreach my $e ( @$entries ) {
        unlink("$refCntDir/$e->{name}") if ( $e->{name} =~ /^poolCntNew/ );
    }
    foreach my $e ( @$entries ) {
        next if ( $e->{name} !~ /^poolCntDelta/ );
        my $deltaFileName = "$refCntDir/$e->{name}";
        my $compress      = 1;
        $compress = $1 if ( $deltaFileName =~ m{/poolCntDelta_(\d)_} );
        $gotDeltas++;
        updateHostPoolOneDelta2Cnt($host, $refCntDir, $deltaFileName, !$forceFsck, $compress, $bkupNum);
    }
    renameNewCntFiles($host, $refCntDir, $compareCurr, "#$bkupNum") if ( $gotDeltas );
}

#
# Rename all the new pool files, replacing the old ones
#
sub renameNewCntFiles
{
    my($host, $refCntDir, $compareCurr, $statusText) = @_;

    print("__bpc_progress_state__ rename $statusText\n") if ( !$opts{p} );
    for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
        for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
            my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, $compress, $refCntFile * 2);
            my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x",    $refCntDir, $compress, $refCntFile * 2);
            if ( $compareCurr ) {
                $ErrorCnt += poolCountHostNewCompare($host, $compress, $poolCntFileCur, $poolCntFileNew);
            }
            if ( -f $poolCntFileNew ) {
                if ( !rename($poolCntFileNew, $poolCntFileCur) ) {
                    print("BackupPC_refCountUpdate: can't rename $poolCntFileNew to $poolCntFileCur ($!)\n");
                    unlink($poolCntFileNew);
                    $ErrorCnt++;
                    next;
                }
            } elsif ( -f $poolCntFileCur && !unlink($poolCntFileCur) ) {
                print("BackupPC_refCountUpdate: can't unlink $poolCntFileCur ($!)\n");
                $ErrorCnt++;
                next;
            }
        }
    }
}

#
# Apply a single pool delta file to the host's new count database.
#
sub updateHostPoolOneDelta2Cnt
{
    my($host, $refCntDir, $deltaFileName, $accumCurr, $compress, $bkupNum) = @_;

    my $count = BackupPC::XS::PoolRefCnt::new();
    if ( $count->read($deltaFileName) ) {
        print("BackupPC_refCountUpdate: can't read pool count delta file $deltaFileName\n");
        $ErrorCnt++;
        return;
    }
    my($delta, $d, $c, $entryCnt);
    my $idx = 0;
    while ( 1 ) {
        ($d, $c, $idx) = $count->iterate($idx);
        last if ( !defined($d) );
        $delta->[vec($d, 0, 8) >> 1]{$d} = $c;
        $entryCnt++;
    }
    $count = undef;
    print("BackupPC_refCountUpdate: processing host $host #$bkupNum deltaFile $deltaFileName with $entryCnt entries\n")
      if ( $Conf{XferLogLevel} >= 2 );

    for ( my $refCntFile = 0 ; $refCntFile < 128 ; $refCntFile++ ) {
        my $poolCntFileNew = sprintf("%s/poolCntNew.%d.%02x", $refCntDir, $compress, $refCntFile * 2);
        my $poolCntFileCur = sprintf("%s/poolCnt.%d.%02x",    $refCntDir, $compress, $refCntFile * 2);
        if ( !defined($delta->[$refCntFile]) || !%{$delta->[$refCntFile]} ) {
            next if ( -f $poolCntFileNew || !-f $poolCntFileCur );
        }
        #
        # Read the existing count
        #
        my $count = BackupPC::XS::PoolRefCnt::new();
        if ( -f $poolCntFileNew ) {
            if ( $count->read($poolCntFileNew) ) {
                print("BackupPC_refCountUpdate: can't open new pool count file $poolCntFileNew\n");
                $ErrorCnt++;
                next;
            }
        } else {
            #
            # Read in the existing counts if we are accumulating.
            #
            if ( $accumCurr && -f $poolCntFileCur && $count->read($poolCntFileCur) ) {
                print("BackupPC_refCountUpdate: can't open cur pool count file $poolCntFileCur\n");
                $ErrorCnt++;
                next;
            }
        }
        #
        # Apply the deltas and write the file back; it's an error for any count to
        # be negative.  Remove zero counts.
        #
        foreach my $d ( keys(%{$delta->[$refCntFile]}) ) {
            my $c = $count->incr($d, $delta->[$refCntFile]{$d});
            $count->delete($d) if ( $c == 0 );
            next               if ( $c >= 0 );
            printf(
                "BackupPC_refCountUpdate: got a negative count for %s processing %s; redoing #%d\n",
                unpack("H*", $d),
                $deltaFileName, $bkupNum
            );
            $ErrorCnt++;
            last;
        }
        if ( $count->write($poolCntFileNew) ) {
            print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFileNew\n");
            $ErrorCnt++;
            next;
        }
    }
    $delta = {};
    if ( unlink($deltaFileName) != 1 ) {
        print("BackupPC_refCountUpdate: can't unlink $deltaFileName ($!)\n");
        $ErrorCnt++;
    }
}

#
# Compare a new pool count file with the existing one.  Returns an error count.
# After the compare, renames the new pool count to replace the existing
# one.
#
sub poolCountHostNewCompare
{
    my($host, $compress, $poolCntFileCur, $poolCntFileNew) = @_;
    my $errorCnt = 0;

    my $countNew = BackupPC::XS::PoolRefCnt::new();
    my $count    = BackupPC::XS::PoolRefCnt::new();

    if ( -f $poolCntFileNew && $countNew->read($poolCntFileNew) ) {
        print("BackupPC_refCountUpdate: can't open new pool count file $poolCntFileNew\n");
        $errorCnt++;
    }
    if ( -f $poolCntFileCur && $count->read($poolCntFileCur) ) {
        print("BackupPC_refCountUpdate: can't open current pool count file $poolCntFileCur\n");
        $errorCnt++;
    }

    #
    # Compare the new and existing counts
    #
    my($digest, $cnt);
    my $idx = 0;

    while ( 1 ) {
        ($digest, $cnt, $idx) = $count->iterate($idx);
        last if ( !defined($digest) );

        if ( !defined($countNew->get($digest)) && $cnt > 0 ) {
            printf("BackupPC_refCountUpdate: host %s digest.%d %s count is %d, but should be 0\n",
                $host, $compress, unpack("H*", $digest), $cnt);
            $errorCnt++;
            next;
        }
        if ( $cnt != $countNew->get($digest) ) {
            printf(
                "BackupPC_refCountUpdate: host %s digest.%d %s count is %d, but should be %d\n",
                $host, $compress, unpack("H*", $digest),
                $cnt, $countNew->get($digest)
            );
            $errorCnt++;
        }
        $countNew->delete($digest);
    }

    $idx = 0;
    while ( 1 ) {
        ($digest, $cnt, $idx) = $countNew->iterate($idx);
        last if ( !defined($digest) );
        printf(
            "BackupPC_refCountUpdate: host %s digest.%d %s count missing, but should be %d\n",
            $host, $compress,
            unpack("H*", $digest),
            $countNew->get($digest)
        );
        $errorCnt++;
    }

    return $errorCnt;
}

sub updateAllHostPoolCnt
{
    my($forceLast2Fsck, $forceFullFsck) = @_;

    foreach my $host ( @Hosts ) {
        updateHostPoolCnt($host, $forceLast2Fsck, $forceFullFsck, 0);
    }
}

#
# Accumulate all the host poolRefCnt files to create the
# overall pool reference counts
#
sub updateTotalRefCnt
{
    return if ( $ErrorCnt );

    print("__bpc_progress_state__ refCnt pool\n") if ( !$opts{p} );
    for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
        my $poolName = $compress ? "cpool4" : "pool4";
        for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) {
            #
            # Count the number of pool directories
            #
            print("__bpc_progress_fileCnt__ cntUpdate $refCntFile/$refCntEnd\n") if ( !$opts{p} );
            my $poolDir = sprintf("%s/%02x", $compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}, $refCntFile * 2);
            next if ( !-d $poolDir );
            $PoolStats->{$poolName}[$refCntFile]{dirCnt}++;
            my $entries = BackupPC::DirOps::dirRead($bpc, $poolDir);
            foreach my $e ( @$entries ) {
                next if ( $e->{name} !~ /^[\da-f][\da-f]$/ );
                $PoolStats->{$poolName}[$refCntFile]{dirCnt}++;
            }

            #
            # For each host update the per-host count
            #
            my $poolCntFile = "$poolDir/poolCnt";
            my $count       = BackupPC::XS::PoolRefCnt::new();
            my $countCopy   = BackupPC::XS::PoolRefCnt::new();
            my $countCurr   = BackupPC::XS::PoolRefCnt::new();

            $countCurr->read($poolCntFile) if ( -f $poolCntFile );
            foreach my $host ( @Hosts ) {
                my $refCntDir   = "$TopDir/pc/$host/refCnt";
                my $hostCntFile = sprintf("%s/poolCnt.%d.%02x", $refCntDir, $compress, $refCntFile * 2);
                my $countHost   = BackupPC::XS::PoolRefCnt::new();
                $countHost->read($hostCntFile) if ( -f $hostCntFile );
                my($d, $c);
                my $idx = 0;

                while ( 1 ) {
                    ($d, $c, $idx) = $countHost->iterate($idx);
                    last if ( !defined($d) );

                    if ( $c < 0 ) {
                        printf("BackupPC_refCountUpdate: host %s (%s) digest %s has negative count (%d)\n",
                            $host, $hostCntFile, unpack("H*", $d), $c);
                        $ErrorCnt++;
                    }
                    if ( !defined($countCurr->get($d)) ) {
                        #
                        # add stats for the new pool file
                        #
                        my $poolFile = $bpc->MD52Path($d, $compress);
                        my @s        = stat($poolFile);
                        if ( @s ) {
                            my $nBlks = $s[12];
                            $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks;
                            if ( $c > 0 ) {
                                if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) {
                                    print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n");
                                    $ErrorCnt++;
                                }
                            }
                        }
                    } elsif ( $countCurr->get($d) == 0 && $c > 0 ) {
                        #
                        # remove S_IXOTH flag since this digest is now referenced
                        #
                        my $poolFile = $bpc->MD52Path($d, $compress);
                        my @s        = stat($poolFile);
                        if ( @s ) {
                            if ( ($s[2] & S_IXOTH) && chmod(0444, $poolFile) != 1 ) {
                                print("BackupPC_refCountUpdate: can't chmod 0444 $poolFile\n");
                                $ErrorCnt++;
                            }
                        }
                    }
                    $count->incr($d, $c);
                    $countCopy->incr($d, $c);
                    $countCurr->incr($d, $c);    # make sure we only count the new pool file once.
                }
            }

            #
            # Add entries for any files in the existing pool count that aren't already in count/countCopy.
            #
            if ( -f $poolCntFile ) {
                my($d, $c);
                my $idx = 0;

                while ( 1 ) {
                    ($d, $c, $idx) = $countCurr->iterate($idx);
                    last if ( !defined($d) );
                    next if ( defined($count->get($d)) );
                    $count->incr($d, 0);
                    $countCopy->incr($d, 0);
                }
            }

            #
            # Scan the pool to add any missing files that have a zero count
            #
            for ( my $subDir = 0 ; $subDir < 128 ; $subDir++ ) {
                my $poolSubDir = sprintf("%s/%02x", $poolDir, $subDir * 2);
                my $entries    = BackupPC::DirOps::dirRead($bpc, $poolSubDir);
                foreach my $e ( @$entries ) {
                    next if ( $e->{name} eq "." || $e->{name} eq ".." );
                    if ( $e->{name} !~ /^[\da-f]{32,48}$/ && $e->{name} !~ /lock/i ) {
                        print("BackupPC_refCountUpdate: unknown pool file $poolSubDir/$e->{name} removed\n");
                        unlink("$poolSubDir/$e->{name}");
                        next;
                    }
                    my $d  = pack("H*", $e->{name});
                    my $b2 = vec($d, 0, 16);
                    if ( $refCntFile != (($b2 >> 8) & 0xfe) / 2 || $subDir != (($b2 >> 0) & 0xfe) / 2 ) {
                        print("BackupPC_refCountUpdate: unexpected pool file $poolSubDir/$e->{name} removed\n");
                        unlink("$poolSubDir/$e->{name}");
                        next;
                    }
                    #
                    # check a small random subset of the pool file's actual md5 against its file name
                    #
                    if ( rand(100) < $Conf{PoolNightlyDigestCheckPercent} ) {
                        my $md5 = BackupPC::XS::FileDigest::digest("$poolSubDir/$e->{name}", $compress);
                        if ( substr($d, 0, 16) ne $md5 ) {
                            printf(
                                "BackupPC_refCountUpdate: ERROR pool file %s has digest %s instead of %s\n",
                                "$poolSubDir/$e->{name}",
                                unpack("H*", $md5),
                                unpack("H*", substr($d, 0, 16))
                            );
                        } else {
                            print("BackupPC_refCountUpdate: pool file $poolSubDir/$e->{name} md5 digest verified\n")
                              if ( $Conf{XferLogLevel} >= 5 );
                        }
                    }
                    if ( !defined($count->get($d)) ) {
                        my @s = stat("$poolSubDir/$e->{name}");
                        print("BackupPC_refCountUpdate: adding pool file $e->{name} with count 0 (size $s[7])\n")
                          if ( $Conf{XferLogLevel} >= 5 );
                        #
                        # add stats for new pool file
                        #
                        my $nBlks = $s[12];
                        $PoolStats->{$poolName}[$refCntFile]{blkCnt} += $nBlks;
                        $count->incr($d, 0);
                    } else {
                        $countCopy->delete($d);
                    }
                }
            }

            #
            # Compute the full pool size periodically, in case the relative +/- above
            # doesn't exactly track.
            #
            # Normally we only update relative changes to the pool size (when a new pool
            # file is added, or an old one is deleted), which is a lot more efficient.
            #
            # So decide when to do a full pool size scan.  $refCntFile goes from 0..127,
            # phase ($opts{P}) is 0..15 and $Conf{PoolSizeNightlyUpdatePeriod} is
            # 0, 1, 2, 4, 8, 16.
            #
            my $fullPoolScan;
            if ( $Conf{PoolSizeNightlyUpdatePeriod} > 0 && defined($opts{P}) ) {
                $fullPoolScan = (int($refCntFile / 8) % $Conf{PoolSizeNightlyUpdatePeriod}) ==
                  ($opts{P} % $Conf{PoolSizeNightlyUpdatePeriod});
            }

            #            print("BackupPC_refCountUpdate: computing full $poolName size for $refCntFile (phase = $opts{P},"
            #                . " \$Conf{PoolSizeNightlyUpdatePeriod} = $Conf{PoolSizeNightlyUpdatePeriod})\n")
            #                                if ( $fullPoolScan );

            my $blkCnt        = 0;    # size of pool files
            my $fileCntRep    = 0;    # total number of pool files with repeated md5 checksums
                                      # (ie: digest > 16 bytes; first instance isn't counted)
            my $fileRepMax    = 0;    # worse case chain length of pool files that have repeated
                                      # checksums (ie: max(NNN) for all digests xxxxxxxxxxxxxxxxNNN)
            my $fileLinkMax   = 0;    # maximum number of links on a pool file
            my $fileLinkTotal = 0;    # total number of links on entire pool
            my($digest, $cnt);
            my $idx         = 0;
            my $poolFileCnt = 0;

            while ( 1 ) {
                ($digest, $cnt, $idx) = $count->iterate($idx);
                last if ( !defined($digest) );
                $poolFileCnt++;
                $fileLinkTotal += $cnt;
                $fileLinkMax = $cnt if ( $fileLinkMax < $cnt && $digest ne $EmptyMD5 );
                if ( $fullPoolScan ) {
                    my $poolFile = $bpc->MD52Path($digest, $compress);
                    my @s        = stat($poolFile);
                    $blkCnt += $s[12] if ( @s );
                }
                next if ( length($digest) <= 16 );
                my $ext = $bpc->digestExtGet($digest);
                $fileCntRep++;
                $fileRepMax = $ext if ( $fileRepMax < $ext );
            }
            $PoolStats->{$poolName}[$refCntFile]{blkCnt}        = $blkCnt if ( $fullPoolScan );
            $PoolStats->{$poolName}[$refCntFile]{fileCnt}       = $poolFileCnt;
            $PoolStats->{$poolName}[$refCntFile]{fileLinkMax}   = $fileLinkMax;
            $PoolStats->{$poolName}[$refCntFile]{fileLinkTotal} = $fileLinkTotal;
            $PoolStats->{$poolName}[$refCntFile]{fileCntRep}    = $fileCntRep;
            $PoolStats->{$poolName}[$refCntFile]{fileRepMax}    = $fileRepMax;

            #
            # Remove zero counts on pool files that don't exist.  Report pool files
            # that have non-zero counts and are missing.
            #
            $idx = 0;
            while ( 1 ) {
                ($digest, $cnt, $idx) = $countCopy->iterate($idx);
                last if ( !defined($digest) );
                if ( $cnt == 0 ) {
                    $count->delete($digest);
                    next;
                }
                next if ( $digest eq $EmptyMD5 );
                my $digestStr = unpack("H*", $digest);
                print("BackupPC_refCountUpdate: missing pool file $digestStr count $cnt\n");
                $ErrorCnt++;
            }

            if ( $count->write("$poolCntFile.$$") ) {
                print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFile.$$\n");
                unlink("$poolCntFile.$$");
                $ErrorCnt++;
                return;
            } elsif ( !rename("$poolCntFile.$$", $poolCntFile) ) {
                print("BackupPC_refCountUpdate: can't rename $poolCntFile.$$ to $poolCntFile ($!)\n");
                unlink("$poolCntFile.$$");
                $ErrorCnt++;
                return;
            }
            $bpc->flushXSLibMesgs();
        }
    }
}

sub cleanPoolFiles
{
    for ( my $compress = 0 ; $compress < 2 ; $compress++ ) {
        my $poolName = $compress ? "cpool4" : "pool4";
        for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) {
            #
            # Read the existing count
            #
            my $dirty       = 0;
            my $poolDir     = sprintf("%s/%02x", $compress ? $bpc->{CPoolDir} : $bpc->{PoolDir}, $refCntFile * 2);
            my $poolCntFile = "$poolDir/poolCnt";
            my $count       = BackupPC::XS::PoolRefCnt::new();
            #
            # Grab a lock to make sure BackupPC_dump won't unmark and use a pending
            # delete file.
            #
            my $lockFd = BackupPC::XS::DirOps::lockRangeFile("$poolDir/LOCK", 0, 1, 1);
            if ( -f $poolCntFile && $count->read($poolCntFile) ) {
                print("BackupPC_refCountUpdate: can't read pool count file $poolCntFile\n");
                $dirty = 1;
                $ErrorCnt++;
            }

            my($digest, $cnt);
            my $idx      = 0;
            my $minMtime = time() - 7 * 24 * 3600;
            while ( 1 ) {
                ($digest, $cnt, $idx) = $count->iterate($idx);
                last if ( !defined($digest) );
                next if ( $cnt > 0 );
                my $poolFile = $bpc->MD52Path($digest, $compress);
                my @s        = stat($poolFile);
                next if ( !@s || $s[7] == 0 || $s[9] >= $minMtime );
                my $mode  = $s[2];
                my $nBlks = $s[12];

                if ( $mode & S_IXOTH ) {
                    #
                    # figure out the next file in the sequence
                    #
                    my $ext = $bpc->digestExtGet($digest);
                    my($nextDigest, $nextPoolFile) = $bpc->digestConcat($digest, $ext + 1, $compress);
                    if ( !-f $nextPoolFile ) {
                        #
                        # last in the chain (or no chain) - just delete it
                        #
                        print("BackupPC_refCountUpdate: removing pool file $poolFile\n")
                          if ( $Conf{XferLogLevel} >= 4 );
                        if ( unlink($poolFile) != 1 ) {
                            print("BackupPC_refCountUpdate: can't remove $poolFile\n");
                            $ErrorCnt++;
                            next;
                        }
                    } else {
                        #
                        # in a chain of pool files we can't delete so
                        # we replace the file with an empty file.
                        # first remove S_IXOTH mode
                        #
                        print("BackupPC_refCountUpdate: zeroing pool file $poolFile (next $nextPoolFile exists)\n")
                          if ( $Conf{XferLogLevel} >= 4 );
                        if ( chmod(0644, $poolFile) != 1 ) {
                            print("BackupPC_refCountUpdate: can't chmod 0644 $poolFile\n");
                            $ErrorCnt++;
                        }
                        if ( open(my $fh, ">", $poolFile) ) {
                            close($fh);
                        } else {
                            print("BackupPC_refCountUpdate: can't truncate $poolFile\n");
                            $ErrorCnt++;
                            next;
                        }
                    }
                    $count->delete($digest);
                    $dirty = 1;
                    #
                    # update stats
                    #
                    $PoolStats->{$poolName}[$refCntFile]{fileCnt}--;
                    $PoolStats->{$poolName}[$refCntFile]{blkCnt} -= $nBlks;
                    $PoolStats->{$poolName}[$refCntFile]{fileCntRm}++;
                    $PoolStats->{$poolName}[$refCntFile]{blkCntRm} += $nBlks;
                } else {
                    #
                    # mark the pool file so no one links to it
                    #
                    print("BackupPC_refCountUpdate: marking pool file $poolFile\n") if ( $Conf{XferLogLevel} >= 4 );
                    if ( chmod(0445, $poolFile) != 1 ) {
                        print("BackupPC_refCountUpdate: can't chmod 0445 $poolFile\n");
                        $ErrorCnt++;
                    }
                }
            }
            if ( $dirty ) {
                #
                # rewrite the poolCnt file
                #
                if ( $count->write("$poolCntFile.$$") ) {
                    print("BackupPC_refCountUpdate: can't write new pool count file $poolCntFile.$$\n");
                    $ErrorCnt++;
                } elsif ( !rename("$poolCntFile.$$", $poolCntFile) ) {
                    print("BackupPC_refCountUpdate: can't rename $poolCntFile.$$ to $poolCntFile ($!)\n");
                    $ErrorCnt++;
                }
            }
            BackupPC::XS::DirOps::unlockRangeFile($lockFd);
            if ( $opts{s} ) {
                statsPrintSingle($poolName, $refCntFile);
            }
        }
    }
}

sub statsPrintSingle
{
    my($poolName, $refCntFile) = @_;

    my $s    = $PoolStats->{$poolName}[$refCntFile];
    my $kb   = $s->{blkCnt} >= 0 ? int($s->{blkCnt} / 2 + 0.5) : int($s->{blkCnt} / 2 - 0.5);
    my $kbRm = $s->{blkCntRm} >= 0 ? int($s->{blkCntRm} / 2 + 0.5) : int($s->{blkCntRm} / 2 - 0.5);
    printf(
        "BackupPC_stats4 %d = %s,%d,%d,%d,%d,%d,%d,%d,%d,%d\n",
        $refCntFile,      $poolName,         $s->{fileCnt},   $s->{dirCnt},
        $kb,              $kbRm,             $s->{fileCntRm}, $s->{fileCntRep},
        $s->{fileRepMax}, $s->{fileLinkMax}, $s->{fileLinkTotal}
    );
}

sub statsPrint
{
    foreach my $poolName ( qw(pool4 cpool4) ) {
        for ( my $refCntFile = $refCntStart ; $refCntFile <= $refCntEnd ; $refCntFile++ ) {
            statsPrintSingle($poolName, $refCntFile);
        }
    }
}