# Copyright (c) 2004, 2005 MySQL AB
# Use is subject to license terms
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; version 2
# of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA

#
# file : test/ndbnet/testError.run
# usage: perl testError.run
#
# you need to have $NDB_TOP/lib/perl5 on search path $PERL5LIB
# or else write perl -I$NDB_TOP/lib/perl5 test1.run
#
# The database is specified by the $NDB_DATABASE environment variable
#
# method names and argument style will change slightly.
#
#

use strict;
use NDB::Run;

my $env = NDB::Run->getenv;
my $log = $env->getlog;
$log->setpart(time => 1, line => 0);
$log->setprio("info");

my $database = $ENV{NDB_DATABASE};
my $api_cmd = $ENV{API_CMD};
my $api_cmd2 = $ENV{API_CMD2};
$log->put("start test database=$database");
$env->init or $log->push("init failed")->fatal;

my $db = $env->getdb($database) or $log->push->fatal;
my $mgm = $db->getnode(1) or $log->push->fatal;
my $api = $db->getnode(6) or $log->push->fatal;
my $api2 = $db->getnode(7) or $log->push->fatal;

my @dbnode = ();	# array of db nodes indexed 2..5
for my $i (2..5) {
    $dbnode[$i] = $db->getnode($i) or $log->push->fatal;
}

# list of db nodes and errors to insert
my @errors = (		# array of array refs
    [ 3, 4, 8030, 8031 ],
    [ 2, 5, 8027, 8030 ],
    [ 2, 4, 5000, 5000 ],
    [ 2, 5, 5000, 5000 ],
    [ 3, 4, 5000, 7008 ],
    [ 3, 5, 5000, 7008 ],
    [ 2, 4, 5000, 7007 ],
    [ 3, 4, 5000, 7007 ],
    [ 3, 5, 5000, 7006 ],
    [ 2, 5, 5000, 7006 ],
    [ 2, 4, 5000, 7005 ],
    [ 3, 5, 5000, 7005 ],
    [ 3, 4, 7005, 7005 ],
    [ 2, 5, 7005, 7005 ],
    [ 3, 4, 7005, 7007 ],
    [ 3, 5, 7005, 7007 ],
    [ 2, 5, 7005, 7008 ],
    [ 2, 4, 7005, 7008 ],
    [ 3, 5, 7006, 7006 ],
    [ 3, 4, 7006, 7006 ],
    [ 2, 4, 7006, 7007 ],
    [ 2, 5, 7006, 7007 ],
    [ 2, 5, 7006, 7008 ],
    [ 2, 4, 7006, 7008 ],
    [ 3, 4, 7007, 7007 ],
    [ 3, 5, 7007, 7007 ],
    [ 3, 5, 7007, 7008 ],
    [ 3, 4, 7007, 7008 ],
    [ 2, 4, 7008, 7008 ],
    [ 2, 5, 7008, 7008 ],
    [ 2, 5, 7008, 7008 ],
    [ 2, 5, 7008, 7008 ],
    [ 2, 5, 8000, 8000 ],
    [ 2, 4, 8000, 8000 ],
    [ 3, 5, 8000, 8001 ],
    [ 3, 4, 8000, 8001 ],
    [ 2, 5, 8000, 5001 ],
    [ 3, 5, 8000, 5001 ],
    [ 3, 4, 8001, 5001 ],
    [ 2, 4, 8001, 5001 ],
    [ 2, 5, 8002, 8029 ],
    [ 3, 4, 8030, 8031 ],
    [ 3, 5, 8020, 8021 ],
    [ 2, 4, 8022, 9999 ],
    [ 2, 4, 8023, 8025 ],
    [ 2, 5, 8027, 8030 ],
    [ 3, 4, 8002, 8002 ],
    [ 3, 5, 8029, 8030 ],
    [ 3, 5, 8031, 8031 ],
    [ 3, 4, 8020, 8020 ],
    [ 2, 4, 8021, 8021 ],
    [ 2, 5, 8022, 8022 ],
    [ 3, 4, 8023, 8023 ],
    [ 3, 5, 8025, 8025 ],
    [ 2, 4, 8027, 8027 ],
    [ 2, 5, 8027, 8027 ],
    [ 3, 4, 8023, 9999 ],
    [ 3, 5, 8025, 9998 ],
    [ 2, 4, 8027, 9999 ],
    [ 2, 5, 8027, 9998 ],
    [ 3, 4, 8000, 9999 ],
    [ 3, 5, 8001, 9998 ],
    [ 2, 4, 5001, 9999 ],
    [ 2, 5, 5000, 9999 ],
    [ 3, 4, 7005, 9999 ],
    [ 3, 5, 7006, 9999 ],
    [ 2, 4, 7007, 9999 ],
    [ 2, 5, 7008, 9999 ],
    [ 2, 4, 9998, 9998 ],
    [ 3, 5, 9998, 9998 ],
    [ 2, 4, 9998, 9998 ],
    [ 2, 4, 9998, 9998 ],
    [ 3, 5, 9998, 9998 ],
    [ 2, 4, 9999, 9999 ],
    [ 2, 4, 9998, 9998 ],
    [ 3, 5, 9999, 9999 ],
    [ 2, 4, 9999, 9999 ],
    [ 2, 4, 9999, 9999 ],
    [ 3, 5, 9999, 9999 ],
);

$db->kill;
$db->start({init_rm=>1}) or $log->push->fatal;
sleep 10;

# should be option (or default) in $db->start
sub wait_until_started {
    my $local_cnt = 60;
    while (--$local_cnt > 0) {
	sleep 5;
	my $ret = $mgm->write("all status", { wait => 3 });
	$ret or $log->fatal;
	my $output = $ret->{output};
        if ($output =~ /((.|\n)*\bstarted\b(.|\n)*){4}/i) {
	    $log->put("*** db is started ***")->info;
	    return;
	}
        if ($output =~ /((.|\n)*\bno.contact\b(.|\n)*){4}/i) {
	    print "NDBT_ProgramExit: 1 - test error\n";
	    $log->put("*** db is dead ***")->fatal;
	}
    }
    print "NDBT_ProgramExit: 1 - test error\n";
    $log->put("*** node recovery failed ***")->fatal;
}

sub getdbstatus {
    my $ret = $mgm->write("all status", { wait => 3 });
    $ret or return undef;
    my $output = $ret->{output};
    my @status = ();	# indexed 2..5
    for my $i (2..5) {
	my $s = "Unknown";
	if ($output =~ /Node\s*$i\s*:\s*(\w+)/i) {
	    $s = $1;
	}#if
	$status[$i] = lc $s;
    }#for
    return \@status;
}

# count elapsed time
my $lasttime;
sub settime { $lasttime = time };
sub gettime { return time - $lasttime };

wait_until_started();
$api->start({run=>$api_cmd});
sleep 15;
$api2->start({run=>$api_cmd2});
sleep 15;

# loop over error inserts
for my $x (0..1) {
for my $e (@errors) {
for my $z (0..3) {
  my $i1 = $e->[0];                     # db node number 2..5
  my $i2 = $e->[1];                     # db node number 2..5
  my $c1 = $e->[2];                     # error code
  my $c2 = $e->[3];                     # error code
  my $dbnode1 = $dbnode[$i1];
  my $dbnode2 = $dbnode[$i2];
  my $kill_no = 9998;
  my @survivor = ();
  my $survivor_count = 0;

  $log->put("insert error $c1 on node $i1")->push($dbnode1)->notice;
  $log->put("and insert error $c2 on node $i2")->push($dbnode2)->notice;

  for my $node (2..5) {
    if (($node ne $i1) && ($node ne $i2)) {
      $survivor[$survivor_count] = $node;
      $survivor_count++;
    }#if
  }#for
  # insert error
  if ($c1 eq $kill_no) {
    $dbnode1->kill
       or $log->put("Kill 1 failed")->fatal;
  } else {
    $mgm->write("$i1 error $c1")
       or $log->put("insert error 1 fault")->fatal;
  }#if
  if ($c2 eq $kill_no) {
    $dbnode2->kill
       or $log->put("Kill 2 failed")->fatal;
  } else {
    $mgm->write("$i2 error $c2")
       or $log->put("insert error 2 fault")->fatal;
  }#if

  # after a few seconds check that node is dead
  settime();
  loop: {
    gettime() <= 300
      or $log->put("db node $i1 or $i2 refuses to die")->fatal;
    my $status = getdbstatus()
       or $log->put("getdbstatus error")->fatal;

    if (($status->[$survivor[0]] eq 'no') ||
        ($status->[$survivor[1]] eq 'no')) {
      print "NDBT_ProgramExit: 1 - test error\n";
      $db->kill;
      $log->put("*** db is dead ***")->fatal;
    }#if
    if (($status->[$i1] ne 'no') || ($status->[$i2] ne 'no')) {   # ...contact
      sleep 2;
      redo loop;
    }#if
  }#loop

  # have to even check the process is gone
  sleep 5;
  if ($dbnode1->stat ne "down") {
    $log->put("ndb did not die, kill it")->push($dbnode1)->warn;
    $dbnode1->kill
       or $log->put("Kill 3 failed")->fatal;
  }#if
  if ($dbnode2->stat ne "down") {
    $log->put("ndb did not die, kill it")->push($dbnode2)->warn;
    $dbnode2->kill
       or $log->put("Kill 4 failed")->fatal;
  }#if

  $log->put("node $i1 and node $i2 is dead")->notice;

  # start the failed nodes
  $dbnode1->start
     or $log->put("Start ndb node 1 failed")->fatal;
  $dbnode2->start
     or $log->put("Start ndb node 2 failed")->fatal;
  wait_until_started();
  $log->put("node $i1 and node $i2 is up again")->notice;

  # check test pgm is running
  my $stat = $api->stat
    or $log->put("api->stat failed")->fatal;
  if ($stat ne "up") {
    print "NDBT_ProgramExit: 1 - test error\n";
    sleep 15;
    $db->kill;
    $log->put("flexBench has crashed")->fatal;
  }#if
  my $stat = $api2->stat
    or $log->put("api2->stat failed")->fatal;
  if ($stat ne "up") {
    print "NDBT_ProgramExit: 1 - test error\n";
    sleep 15;
    $db->kill;
    $log->put("flexBench2 has crashed")->fatal;
  }#if
}#for
}#for
}#for
print "NDBT_ProgramExit: 0 - test ready\n";
$db->kill;

# vim: set sw=4: