#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#***********************************************************************
# This file is part of OpenMolcas.                                     *
#                                                                      *
# OpenMolcas is free software; you can redistribute it and/or modify   *
# it under the terms of the GNU Lesser General Public License, v. 2.1. *
# OpenMolcas is distributed in the hope that it will be useful, but it *
# is provided "as is" and without any express or implied warranties.   *
# For more details see the full text of the license in the file        *
# LICENSE or in <http://www.gnu.org/licenses/>.                        *
#                                                                      *
# Copyright (C) 2000-2012, Valera Veryazov                             *
#               2015, Steven Vancoillie                                *
#               2017-2020,2023-2024, Ignacio Fdez. Galván              *
#***********************************************************************
#
# verify
#
# Automated test verification for Molcas.
# For usage documentation read the help subroutines.
#
# Author:  Valera Veryazov                                                    
#          Dept. of Theoretical Chemistry                                     
#          Lund, Sweden                                                       
# Written: 2000-2012                                                          
#
# Re-written:
# Steven Vancoillie, summer 2015
#
# Support for two test directories
# Ignacio Fdez. Galván, February-March 2017
#
# Further updates for better interaction with dailymerge
# Ignacio Fdez. Galván, June 2017
#
# Add --fromfile option
# Ignacio Fdez. Galván, May 2018
#
# Add --timest and --validate options
# Ignacio Fdez. Galván, August-September 2019
#
# Add --grep option
# Ignacio Fdez. Galván, September 2020
#
# Translate from Perl to Python, include updatetest and chkunprint
# Ignacio Fdez. Galván, November 2020
#
# Add --jobs option
# Ignacio Fdez. Galván, November 2023
#
# Rearrange for better multiprocessing compatibility
# Ignacio Fdez. Galván, June 2024

import sys
import os
import subprocess as sp
import multiprocessing
import concurrent.futures
import re

import locale
import signal
import datetime
import time
import argparse
import shutil
from collections import OrderedDict

#---------------
# output control
#---------------

def print_stdout(text):
  sys.stdout.write(text.encode())

def msg(text):
  if not opt['quiet']:
    print_stdout(text)

def msg_nl(lines):
  if not opt['quiet']:
    for line in lines:
      print_stdout(line)
      print_stdout('\n')

def msg_list(prefix, items):
  max_items = 10
  counter = 0
  padding = 4*' '
  msg_nl([prefix])
  for item in items:
    if counter == 0:
      msg(padding)
    msg(item)
    counter += 1
    if counter == max_items:
      msg('\n')
      counter = 0
  if counter:
    msg('\n')

def msg_overwrite(text, erase):
  l = len(erase)
  backup = l * '\b'
  spaces = l * ' '
  msg(backup + spaces + backup + text)

def warning(text):
  if not opt['quiet']:
    print(text, file=sys.stderr)

def error(text):
  warning(text)
  sys.exit(1)

#-------------------------------------------------------
# helper functions to deal with the group subdirectories
#-------------------------------------------------------

def list_groups():
  groups = {}
  vgroups = {}
  for TESTDIR in TESTDIRS:
    try:
      subdir_names = os.listdir(TESTDIR)
    except:
      pass
    else:
      for subdir_name in subdir_names:
        if subdir_name in ['log', 'failed', 'tmp']:
          continue
        subdir = os.path.join(TESTDIR, subdir_name)
        if not os.path.isdir(subdir):
          continue
        if subdir_name in groups:
          warning('duplicate directory "{}" at {} will be ignored'.format(subdir_name, TESTDIR))
          continue
        groups[subdir_name] = os.path.join(TESTDIR, subdir_name)
        try:
          dotfiles = os.listdir(subdir)
        except:
          pass
        else:
          for dotfile in dotfiles:
            path = os.path.join(subdir, dotfile)
            if not (os.path.isfile(path) and dotfile.startswith('.')):
              continue
            vgroups[dotfile] = True
  maxlength = 0
  # special groups
  vgroups['.none'] = 1
  vgroups['.everything'] = 1
  for key in groups:
    maxlength = max(maxlength, len(key))
  for key in vgroups:
    maxlength = max(maxlength, len(key))
  fmt = '  {{:{}}} = {{}}\n'.format(maxlength)
  msg('\nPhysical groups (directories)\n\n')
  for key in sorted(groups, key=by_type_first):
    group = vgroup_to_groups(key)
    msg(fmt.format(key, location[key]))
  msg('\nSpecial groups\n\n')
  for key in ['.none', '.everything']:
    group = vgroup_to_groups(key)
    group = [i for i in group if i in groups]
    msg(fmt.format(key, ' '.join(sorted(group, key=by_type_first))))
  msg('\nVirtual groups\n\n')
  for key in sorted(vgroups):
    if key in ['.none', '.everything']:
      continue
    group = vgroup_to_groups(key)
    group = [i for i in group if i in groups]
    msg(fmt.format(key, ' '.join(sorted(group, key=by_type_first))))

def vgroup_to_groups(vgroup):
  # convert virtual group name from task to array of real group names:
  #   standard  -> (standard)
  #   .critical -> (standard, additional)
  global location
  groups = []
  if vgroup.startswith('.'):
    for TESTDIR in TESTDIRS:
      try:
        subdir_names = os.listdir(TESTDIR)
      except:
        pass
      else:
        for subdir_name in subdir_names:
          subdir = os.path.join(TESTDIR, subdir_name)
          if not os.path.isdir(subdir):
            continue
          if location.get(subdir_name, subdir) != subdir:
            continue
          # special groups ".none" and ".everything" don't rely on dot files
          if vgroup == '.none':
            continue
          if vgroup == '.everything' or os.path.isfile(os.path.join(subdir, vgroup)):
            groups.append(subdir_name)
            location[subdir_name] = subdir
  else:
    for TESTDIR in TESTDIRS:
      if os.path.isdir(os.path.join(TESTDIR, vgroup)):
        groups.append(vgroup)
        location[vgroup] = os.path.join(TESTDIR, vgroup)
    if vgroup not in location:
      warning('non-existing group: {}'.format(vgroup))
  return groups

def add_names_from_group_to_dictionary(group, name_list, dictionary):
  for name in name_list:
    dictionary['{}:{}'.format(group, name)] = True

def flatten_numbers(numbers):
  # convert task list to flat list of numbers:
  #   002-004 -> (002, 003, 004)
  start, end = re.match(r'^(\d{3})(?:-(\d{3}))?$', numbers).groups()
  if end:
    flattened = ['{:03d}'.format(i) for i in range(int(start), int(end)+1)]
  else:
    flattened = [start]
  return flattened

def find_all_names_from_group(group):
  # convert a group name into a list of all test names in that group:
  #   standard -> (000, 001, ..., 099) + any input file basenames
  directory = location[group]
  try:
    inputs = os.listdir(directory)
  except:
    error('cannot open directory {} for group {}'.format(directory, group))
  names = []
  for inputfile in inputs:
    name, ext = os.path.splitext(inputfile)
    if ext == '.input':
      names.append(name)
  return names

def special_failure(filename):
  # function to check unprintable chars in a file (previously chkunprint)
  segfault = re.compile(b'Segmentation *Vio')
  controlchars = re.compile(b'[\x00-\x08\x0E-\x1F]')
  extendedchars = re.compile(b'[\xA0-\xFF].*[\xA0-\xFF].*[\xA0-\xFF]')
  failure = ''
  try:
    with open(filename, 'rb') as f:
      for line in f:
        if segfault.search(line):
          failure = 'segfault'
          break
        if controlchars.search(line) or extendedchars.search(line):
          failure = 'garbage'
          break
  except:
    error('failure checking for unprintable characters')
  return failure

def failed_module(filename):
  mod = ''
  with open(filename, 'rb') as f:
    for line in f:
      if not re.search(rb'(Start|Stop) Module', line):
        continue
      if re.search(rb'check|auto', line):
        continue
      mod = re.search(rb'(?:Start|Stop) Module: +([^ ]+)', line).group(1).decode()
  if not mod:
    mod = '[none]'
  return mod

def by_standard_first(item):
  # routine to sort keys starting with "standard"
  if item.startswith('standard'):
    return '0' + item
  else:
    return '1' + item

def by_type_first(item):
  # routine to sort keys according to the groups
  group = item.split(':')[0]
  if group in basic:
    return '1' + item
  elif group in critical:
    return '2' + item
  elif group in official:
    return '3' + item
  else:
    return '4' + item

#-----------------------------------------
# functions originally in separate modules
#-----------------------------------------

# Function that appends generated checkfile to a test input.
# First argument is the test input file and second is the
# checkfile
#
# A checkfile is imbedded inside a test file using the EMIL
# file command:
# >>FILE checkfile
# ... checkfile contents ...
# >>EOF
# We just replace the contents with the provided checkfile, or,
# if the ">>FILE checkfile" line is not found it will be added.
def updatetest(inputfile, checkfile):
  # read and store the first section of the input
  try:
    input_lines = []
    with open(inputfile, 'r', encoding='utf-8') as f:
      for line in f:
        if line.rstrip() == '>>FILE checkfile':
          break
        input_lines.append(line)
  except IOError as e:
    warning('cannot open test input: {}'.format(e))
    return 1
  # write the first section of the input and the contents of the
  # checkfile back to the input file.
  try:
    with open(inputfile, 'w+', encoding='utf-8') as fi, open(checkfile, 'r', encoding='utf-8') as fc:
      for line in input_lines:
        fi.write(line)
      fi.write('>>FILE checkfile\n')
      for line in fc:
        fi.write(line)
      fi.write('>>EOF\n')
  except IOError as e:
    warning('cannot open file: {}'.format(e))
    return 2
  return 0

# trap interrupt
def killexit(*args):
  # attempt to cancel parallel processes
  try:
    for pid in pool._processes:
      os.kill(pid, signal.SIGKILL)
    pool.shutdown(wait=True, cancel_futures=True)
  except AttributeError:
    pass
  # kill active processes
  for pid in shared['active_pids']:
    try:
      os.kill(-active_pid, signal.SIGKILL)
    except:
      pass
  error('\nSTOP: user has terminated {}'.format(thisfile))

# worker function to allow running simultaneous jobs
def run_test(filename, shared):
  endloop = False
  with shared['lock']:
    # after the loop is interrupted, new tasks will do nothing
    if shared['endloop']:
      return
    group = filegroup[filename]
    name = os.path.splitext(os.path.basename(filename))[0]
    project = '{}__{}'.format(group, name)
    workdir = os.path.join(tmpdir, project)
    infile = os.path.join(workdir, project + '.input')
    outfile = os.path.join(tmpdir, project + '.out')
    errfile = os.path.join(tmpdir, project + '.err')
    status = os.path.join(tmpdir, project + '.status')
    shared['index'] += 1
    pc = int(shared['index'] / n_tests * 100)
    if group != shared['prev_group'] and group != 'external':
      print('----------\ngroup {} from: {}'.format(group, location[group]), file=RESULT)
      shared['prev_group'] = group
  mark = '4'
  if group in official:
    mark = '3'
  if group in critical:
    mark = '2'
    if group in basic:
      mark = '1'
  #----------------------------
  # set up the work environment
  #----------------------------
  os.environ['Project'] = project
  os.environ['WorkDir'] = workdir
  if group in nolimit:
    os.environ.pop('MOLCAS_TIMELIM', None)
  else:
    os.environ['MOLCAS_TIMELIM'] = os.environ.get('MOLCAS_TIMELIM', '600')
  if opt['cover']:
    with shared['lock']:
      process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--prep'])
      shared['active_pids'].append(process.pid)
      process.wait()
      shared['active_pids'].remove(process.pid)
  # start repeated cycles (e.g. when tests fail randomly)
  cycles = opt['cycles']
  if opt['parallel']:
    cycles *= 2
  while cycles > 0:
    cycles -= 1
    sttime = time.time()
    if opt['parallel']:
      if cycles % 2 == 1:
        os.environ['MOLCAS_NPROCS'] = 1
      else:
        os.environ['MOLCAS_NPROCS'] = opt['parallel']
    # if the work directory exists, remove it unless existing mode is used
    if os.path.isdir(workdir):
      if opt['existing']:
        # we only need to delete the check counter, leave the rest
        check_counter = os.path.join(workdir, 'molcas_check_count')
        os.remove(check_counter)
      else:
        shutil.rmtree(workdir)
    if not os.path.isdir(workdir):
      os.mkdir(workdir)
    shutil.copyfile(filename, infile)
    with shared['lock']:
      text = 'Running test {}: {}... ({}%)'.format(group, name, pc)
      if interactive:
        time.sleep(0.1)
        msg_overwrite(text, shared['prompt'])
        shared['prompt'] = text
      elif opt['status']:
        if opt['jobs'] > 1:
          msg(text+'\n')
        else:
          msg(text)
    # run pymolcas and capture the return code.
    if opt['debug']:
      process = sp.Popen([MOLCAS_DRIVER] + cli_opts + [infile], stdout=sys.stdout, stderr=sys.stderr)
    else:
      with open(outfile, 'w+', encoding='utf-8') as fo, open(errfile, 'w+', encoding='utf-8') as fe:
        process = sp.Popen([MOLCAS_DRIVER] + cli_opts + [infile], stdout=fo, stderr=fe)
    with shared['lock']:
      shared['active_pids'].append(process.pid)
    rc = process.wait()
    with shared['lock']:
      shared['active_pids'].remove(process.pid)
    special = False
    if not opt['debug']:
      # check for special cases where the return code might be 0
      # but we should still cause a failure (segfault, garbage)
      special = special_failure(outfile)
      if special:
        rc = 30
    with shared['lock']:
      result = ''
      if rc:
        cycles = 0
        if not opt['debug']:
          shutil.copyfile(outfile, os.path.join(log_failed, project + '.out'))
          shutil.copyfile(errfile, os.path.join(log_failed, project + '.err'))
        # skip the test if a program is not available (RC_NOT_AVAILABLE)
        if rc == 36:
          result = ' S'
          # print results
          print('{}:{}:{} Skipped!'.format(mark, group, name), file=RESULT)
          text = '{}:{} Skipped!\n'.format(group, name)
          if interactive:
            msg_overwrite(text, shared['prompt'])
            shared['prompt'] = text
          elif opt['status']:
            if opt['jobs'] > 1:
              msg(text)
            else:
              msg(' Skipped!\n')
          # update counters
          shared['skipped_tests'] += 1
          # pretend to be fine for the remainder of this test
          rc = 0
        else:
          result = ' F'
          # gather extra info
          mod = ''
          if not opt['debug']:
            mod = failed_module(outfile)
          if special:
            mod = ', '.join([mod, special])
          # print results
          print('{}:{}:{} Failed! ({})'.format(mark, group, name, mod), file=RESULT)
          text = '{}:{} Failed! ({})\n'.format(group, name, mod)
          if interactive:
            msg_overwrite(text, shared['prompt'])
            shared['prompt'] = text
          elif opt['status']:
            if opt['jobs'] > 1:
              msg(text)
            else:
              msg(' Failed! ({})\n'.format(mod))
          # update counters
          shared['failed_tests'] += 1
          if group in critical:
            shared['failed_critical_tests'] += 1
          # update list of failed input files
          print(filename, file=FAILED_LIST)
      else:
        print('{}:{}:{} OK'.format(mark, group, name), file=RESULT)
        if interactive:
          pass
        elif opt['status']:
          if opt['jobs'] > 1:
            msg('{}:{} OK\n'.format(group, name))
          else:
            msg(' OK\n')
        if opt['generate']:
          rc = updatetest(filename, os.path.join(workdir, 'checkfile'))
          if rc:
            print_stdout('rc={}\n'.format(rc))
      # save timing info
      if opt['timest']:
        runtime = time.time() - sttime
        print('--- {}:{}{}'.format(group, name, result), file=TIMING)
        print(int(round(runtime)), file=TIMING)
      elif not opt['debug']:
        print('--- Run: {}'.format(project), file=TIMING)
        try:
          with open(errfile, 'r', encoding='utf-8') as f:
            for line in f:
              print(line, end='', file=TIMING)
        except:
          error('could not open {}'.format(errfile))
      if opt['postproc']:
        os.environ['project'] = project
        sp.call(opt['postproc'], shell=True)
    # clean up
    if not (rc or opt['keep']):
      if os.path.exists(infile):
        os.remove(infile)
      if os.path.exists(outfile):
        os.remove(outfile)
      if os.path.exists(errfile):
        os.remove(errfile)
      if os.path.exists(status):
        os.remove(status)
      shutil.rmtree(workdir)
    if rc and opt['trap']:
      endloop = True
      break
  with shared['lock']:
    if endloop:
        shared['endloop'] = True
    elif opt['cover']:
      if interactive:
        text = 'Capturing coverage data for test {}: {}... ({}%)'.format(group, name, pc)
        msg_overwrite('', shared['prompt'])
        shared['prompt'] = text
      process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--measure', '--name', project])
      shared['active_pids'].append(process.pid)
      process.wait()
      shared['active_pids'].remove(process.pid)

# wrapper for when mp_context didn't exist
def cfProcessPool(jobs):
  try:
    return concurrent.futures.ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('fork'))
  except TypeError:
    return concurrent.futures.ProcessPoolExecutor(max_workers=jobs)

# dummy context manager to replace locks with 1 job (compatible with older Python versions)
class NullContextManager(object):
  def __init__(self, dummy_resource=None):
    self.dummy_resource = dummy_resource
  def __enter__(self):
    return self.dummy_resource
  def __exit__(self, *args):
    pass


opt = {'quiet': False}

# use proper locale
locale.setlocale(locale.LC_ALL, 'C')

# disable stdout buffering
# (note that we should use print_stdout instead of print)
sys.stdout = os.fdopen(sys.stdout.fileno(), 'wb', 0)

# my name
thisfile = os.path.basename(__file__)

shared = {'active_pids': []}

signal.signal(signal.SIGINT, killexit)

starting_cwd = os.getcwd()

MOLCAS_DRIVER = os.environ.get('MOLCAS_DRIVER', 'pymolcas')
DRIVER_base = os.path.basename(MOLCAS_DRIVER)

short_help = '''
{} {} [--keep|-k] [--debug|-d] [--list|-l] [--generate] [task]

where: task is a group name (standard, additional, benchmark,
grayzone, failed) followed by colon and then a comma-separated
list of numbers and/or ranges, e.g.: standard:000,005-098,-014

use the long option --help for a complete description!
'''.format(DRIVER_base, thisfile)

epilog = '''
task:
  Tests are divided into different groups: e.g.
  standard, additional, benchmark, grayzone, ...
  These groups are subdirectories of the test/
  directory. If the group name begins with a dot,
  it is a virtual group which consists of all
  subdirectories which have a file of that name.

  To specify which tests to run, you need to specify
  a group, optionally followed by a colon with a list
  or range of test numbers. Numbers or ranges preceded
  by a '-' sign are excluded. When no numbers or ranges
  are given, all tests from the group are included. When
  only a '-' is give, all tests are excluded. When no
  group is specified, a default group is selected
  (i.e. the virtual '.default' group, which consists of
  'standard', 'additional', and 'grayzone').

  empty: run all tests from group '.default'
  a list of tests (group:nr1[,nr2,...]):
    000 (same as standard:000)
  a range of tests (two numbers separated by dash):
    005-134
  a combination of the above:
    003,005-009,054
  an additional exclude list:
    001-010,-004 (tests 1 to 10 except 4)
  a group (subdirectory), specified by name:
    standard
    additional
    performance
    benchmark
    grayzone
    ...
  a group (virtual), specified by name:
    .basic
    .default
    .critical
    .all
  a group followed by a colon and then any of the above
  lists of numbers/ranges:
    performance:000-100,-005-009,-043
  a file:
    path/to/file1.input

Individual tasks can be combined in any way as long as they
are separated by a space (you can mix e.g. number tasks and
files)

Examples:
  {driver} {script}                   - run default tests (standard, additional, grayzone)
  {driver} {script} .all              - run all tests (but not performance, benchmark)
  {driver} {script} .everything       - run _all_ tests (yes, ALL tests)
  {driver} {script} standard          - run all standard tests
  {driver} {script} performance       - run performance tests (ca. 30 min)
  {driver} {script} benchmark         - run benchmark tests (several hours)
  {driver} {script} -m caspt2         - run default tests that contain &caspt2 module
  {driver} {script} -w ksdft          - run default tests with 'ksdft' word
  {driver} {script} 001-005,-003      - run tests 001 to 005 but not 003
  {driver} {script} standard:050-070  - run standard tests from 050 to 070
  {driver} {script} .all standard:-   - run all tests except those from standard
  {driver} {script} --failed          - run (default) tests that failed the last time
  {driver} {script} -d 000            - run standard test 000 with output on the screen
'''.format(driver=DRIVER_base, script=thisfile)

# command-line options
try:
  parser = argparse.ArgumentParser(add_help=False, usage='{} {} [options] [task [task ...]]'.format(DRIVER_base, thisfile), epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('task', nargs='*', help=argparse.SUPPRESS)
  parser.add_argument('-h', action='store_true', help='print short help')
  parser.add_argument('--help', action='store_true', help='print long help (you\'re reading it)')
  parser.add_argument('--clean', action='store_true', help='clean up log directory before run')
  parser.add_argument('--cover', action='store_true', help='generate code coverage report (WARNING: can take a long time!')
  parser.add_argument('--cycles', type=int, help='cycle each test N times', metavar='N', default=1)
  parser.add_argument('--debug', '-d', action='store_true', help='print output to terminal')
  parser.add_argument('--existing', action='store_true', help='use existing scratch if available')
  parser.add_argument('--failed', action='store_true', help='rerun tests that failed the last time')
  parser.add_argument('--flatlist', action='store_true', help='only list matching tests (as a flat list of tasks)')
  parser.add_argument('--fromfile', help='read tasks from file FILE, in addition to command line', metavar='FILE')
  parser.add_argument('--fuzzy', action='store_true', help='do not fail if a label is not in the reference (old behavior)')
  parser.add_argument('--generate', action='store_true', help='generate checkfile and append to input')
  parser.add_argument('--grep', help='filter test files containing WORD (anywhere but comments)', metavar='WORD')
  parser.add_argument('--grouplist', action='store_true', help='list the groups (see below)')
  parser.add_argument('--keep', '-k', action='store_true', help='keep the work directory after running a test')
  parser.add_argument('--jobs', '-j', type=int, help='run N simultaneous jobs (0: auto, considering NPROCS and NTHREADS)', metavar='N', default=1)
  parser.add_argument('--list', '-l', action='store_true', help='only list matching tests')
  parser.add_argument('--module', '-m', help='filter test files containing MODULE', metavar='MODULE')
  parser.add_argument('--parallel', type=int, help='double cycles: one with 1 process, and one with N processes', metavar='N')
  parser.add_argument('--pass', action='store_true', help='ignore checkfile failures')
  parser.add_argument('--path', help='run with PATH as temporary directory', metavar='PATH')
  parser.add_argument('--postproc', help='postprocessing command to run before cleaning up the work directory', metavar='CMD')
  parser.add_argument('--quiet', '-q', action='store_true', help='do not print any messages to the screen')
  parser.add_argument('--rawlist', action='store_true', help='only list matching tests (as a list of input files)')
  parser.add_argument('--reset', action='store_true', help='clean up any results and tmp/log directories and exit')
  parser.add_argument('--status', action='store_true', help='print status of the verification (useful for redirecting the output)')
  parser.add_argument('--timest', action='store_true', help='write time estimates (for split_tests)')
  parser.add_argument('--tmp', help='run with TMP as parent scratch directory', metavar='TMP')
  parser.add_argument('--trap', action='store_true', help='stop immediately after a failed test')
  parser.add_argument('--validate', action='store_true', help='only validate the input')
  parser.add_argument('--word', '-w', help='filter test files containing keyword WORD', metavar='WORD')
  opt = vars(parser.parse_args(sys.argv[1:]))
except:
  msg(short_help)
  sys.exit(1)

# do we need help?
if opt['help']:
  print_stdout(parser.format_help())
  sys.exit(0)
if opt['h']:
  msg(short_help)
  sys.exit(0)

tasklist = []
if opt['fromfile']:
  try:
    with open(opt['fromfile'], 'r', encoding='utf-8') as f:
      tasklist = f.read().split()
  except:
    error('cannot open file {}'.format(file_list))

tasklist.extend(opt['task'])

if not tasklist:
  tasklist = ['.default']

# early parsing of options
if opt['debug']:
  opt['keep'] = True

# are we running interactively?
interactive = False
if sys.stdout.isatty() and not opt['debug'] and not opt['status']:
  interactive = True

# store environment info
try:
  MOLCAS = os.environ['MOLCAS']
except KeyError:
  MOLCAS = '/usr/share/openmolcas'
try:
  MOLCAS_ID = sp.check_output([MOLCAS_DRIVER, 'version', '-l'], cwd=MOLCAS).decode().strip()
except:
  MOLCAS_ID = None
MACHINE = sp.check_output(['uname', '-a']).decode().strip()
date = datetime.datetime.today()
DATE = date.strftime('%c')

header = '''{}
== verification run ==
machine: {}
date: {}
'''.format(MOLCAS_ID, MACHINE, DATE)

#-------------------------------------------------
# set up global Molcas settings used for each test
#-------------------------------------------------

# Get list of test directories, adding OPENMOLCAS at the beginning
# and MOLCAS at the end, and removing duplicates
TESTDIRS = []
if os.path.exists(os.path.join(MOLCAS, 'sbin', 'find_sources')):
  OPENMOLCAS = sp.check_output('. $MOLCAS/sbin/find_sources ; echo $OPENMOLCAS_SOURCE', shell=True).decode().strip()
  TESTDIRS.append(os.path.join(OPENMOLCAS, 'test'))
if os.path.exists(os.path.join(MOLCAS, 'test', 'testdirs')):
  with open(os.path.join(MOLCAS, 'test', 'testdirs'), 'r', encoding='utf-8') as f:
    TESTDIRS.extend(f.read().split())
TESTDIRS.append(os.path.join(MOLCAS, 'test'))
TESTDIRS = list(OrderedDict.fromkeys(TESTDIRS))

if opt['path']:
  testdir = os.path.abspath(opt['path'])
else:
  testdir = os.getcwd()

if opt['tmp']:
  tmpdir = os.path.abspath(opt['tmp'])
else:
  tmpdir = os.path.join(testdir, 'tmp')

result  = os.path.join(testdir, 'result')
timing  = os.path.join(testdir, 'timing.data')
logroot = os.path.join(testdir, 'log')
logdir  = os.path.join(logroot, date.strftime('%F_at_%H-%M-%S'))
failed  = os.path.join(testdir, 'failed')

# reset only cleans up and then quits
if opt['reset']:
  if os.path.islink(result):
    os.remove(result)
  if os.path.islink(timing):
    os.remove(timing)
  if os.path.isdir(tmpdir):
    shutil.rmtree(tmpdir)
  if os.path.isdir(logroot):
    shutil.rmtree(logroot)
  if os.path.islink(failed):
    os.remove(failed)
  sys.exit(0)

os.environ['MOLCAS_OUTPUT'] = 'WORKDIR'
os.environ['MOLCAS_TIME'] = 'YES'
# set test type: generating or checking?
if opt['generate']:
  os.environ['MOLCAS_TEST'] = 'GENE'
  os.environ['MOLCAS_NPROCS'] = '1'
else:
  os.environ['MOLCAS_TEST'] = 'CHECK'
# enable old behavior (do not fail with extra labels)
if opt['fuzzy']:
  os.environ['MOLCAS_CHECK_FUZZY'] = 'YES'
# to ignore failures, choose negative threshold
if opt['pass']:
  os.environ['MOLCAS_THR'] = '-1'
  os.environ['MOLCAS_PASSCHECK'] = '1'
os.environ['MOLCAS_VALIDATE'] = os.environ.get('MOLCAS_VALIDATE', 'YES')
# command-line options
cli_opts = '--ignore_environment'
if opt['validate']:
  cli_opts += ' --validate'
cli_opts = cli_opts.split()

#-------------------------------------------
# generate a list of files from the tasklist
#-------------------------------------------

location = {}
filelist = []
filegroup = {}
none = False

# (these must be done early in order to use it for sorting)
# groups belonging to basic
basic = {}
basic_list = vgroup_to_groups('.basic')
for group in basic_list:
  basic[group] = True
# groups belonging to critical
critical = {}
critical_list = vgroup_to_groups('.critical')
for group in critical_list:
  critical[group] = True
# groups belonging to official
official = {}
official_list = vgroup_to_groups('.official')
for group in official_list:
  official[group] = True
# groups with no time limit
nolimit = {'performance': True, 'benchmark': True}

# First convert the tasks to actual included/skipped tests.
# These are kept in two dictionaries to allow different tasks to
# influence each other, e.g.:
#   $ molcas verify .all standard:-
# would run all the tests from virtual group '.all', but
# at the same time exclude all tests from standard.
included = {} # keeps a dictionary of included test names (group:name)
excluded = {} # keeps a dictionary of excluded test names (group:name)
for task in tasklist:
  if task == '.none':
    # the special task .none will generate a result file
    # even if there are no tests to run
    none = True
  elif os.path.isfile(task):
    # the task is a file name, add immediately to the file list
    filename = os.path.abspath(task)
    filelist.append(filename)
    filegroup[filename] = 'external'
  else:
    # A single task is composed of:
    # - an optional group name (if omitted, defaults to '.default')
    # - an optional, comma-separated lists of the following items:
    #   * 3 digits: a test number to be included
    #   * a minus sign and 3 digits: a test number to be excluded
    #   * two groups of 3 digits separated by a dash:
    #     a range of test numbers to be included
    #   * a minus sign and two groups of 3 digits separated by a dash:
    #     a range of test numbers to be excluded
    #   * empty: includes all possible test numbers
    #   * a minus sign: excludes all possible test numbers
    # All of the above is condensed into a single regex:
    vgroup, task_string = re.match(r'^([^\d:][^:]*)?:?([^:]*)$', task).groups()
    if not vgroup:
      vgroup = '.default'
    # if no subtasks, set empty string as only element (this means all tests)
    subtasks = task_string.split(',')
    # translate virtual group to list of real group names
    for group in vgroup_to_groups(vgroup):
      for subtask in subtasks:
        # decide on exclusion
        exclude_flag = False
        if subtask.startswith('-'):
          subtask = subtask[1:]
          exclude_flag = True
        # create a list of file basenames
        basenames = []
        if not subtask:
          # empty task, match every possible input file
          basenames = find_all_names_from_group(group)
        else:
          number_string = re.match(r'\d{3}(?:-\d{3})?$', subtask)
          if number_string:
            # extract name from number(range)
            basenames = flatten_numbers(number_string.group(0))
          else:
            # no number, assume subtask is a basename
            basenames.append(subtask)
        # add basenames to proper dictionary
        if exclude_flag:
          add_names_from_group_to_dictionary(group, basenames, excluded)
        else:
          add_names_from_group_to_dictionary(group, basenames, included)

# add generated test names to file list
for key in sorted(included, key=by_type_first):
  if key in excluded:
    continue
  group, number = key.split(':')
  filename = os.path.join(location[group], number + '.input')
  if os.path.isfile(filename):
    if opt['generate']:
      shutil.copyfile(filename, filename + '.bak')
    filelist.append(filename)
  filegroup[filename] = group

#-------------------------------------------------------------------------
# now that we have the filelist, we can apply filters to the file contents
#-------------------------------------------------------------------------

# build up the regex pattern to use for filtering
# this is a very simple filter: match _any_ pattern
if opt['module'] or opt['word'] or opt['grep']:
  pattern_list = []
  if opt['module']:
    for mod in opt['module'].split(','):
      pattern_list.append('&' + mod + r'\b')
  if opt['word']:
    for key in opt['word'].split(','):
      pattern_list.append(key)
  if opt['grep']:
    for key in opt['grep'].split(','):
      pattern_list.append(r'[^*]*' + key)
  filter_pattern = r'^\s*(' + '|'.join(pattern_list) + ')'
  filtered_filelist = []
  for filename in filelist:
    with open(filename, 'rb') as f:
      in_file = False
      for line in f:
        # ignore lines between >FILE and >EOF
        if (not in_file) and re.match(r'>* *FILE '.encode(), line, re.IGNORECASE):
          in_file = True
        if in_file and re.match(r'>* *EOF'.encode(), line, re.IGNORECASE):
          in_file = False
        if (not in_file) and re.match(filter_pattern.encode(), line, re.IGNORECASE):
          filtered_filelist.append(filename)
          break
  filelist = filtered_filelist

#--------------------------------------------------------------
# if we only rerun failed tests, then filter the filenames here
#--------------------------------------------------------------

if opt['failed']:
  failed_previously = {}
  failed_list = os.path.join(failed, 'list')
  if os.path.isfile(failed_list):
    try:
      with open(failed_list, 'r', encoding='utf-8') as f:
        for line in f:
          failed_previously[line.strip()] = True
    except IOError:
      sys.exit('cannot open file {}'.failed(failed_list))
  filtered_filelist = []
  for filename in filelist:
    if filename in failed_previously:
      filtered_filelist.append(filename)
  filelist = filtered_filelist

#------------------------------------------------------
# print the final, filtered list if requested, and exit
#------------------------------------------------------

if opt['rawlist']:
  msg_nl(filelist)
  sys.exit(0)
elif opt['flatlist']:
  test_list = []
  for filename in filelist:
    name = os.path.splitext(os.path.basename(filename))[0]
    test_list.append('{}:{}'.format(filegroup[filename], name))
  msg_nl(test_list)
  sys.exit(0)
elif opt['grouplist']:
  list_groups()
  sys.exit(0)
elif opt['list']:
  msg('matching tests:\n')
  test_list = {}
  for filename in filelist:
    name = os.path.splitext(os.path.basename(filename))[0]
    if filegroup[filename] not in test_list:
      test_list[filegroup[filename]] = []
    test_list[filegroup[filename]].append(' ' + name)
  for key in sorted(test_list, key=by_type_first):
    prefix = '  {}:'.format(key)
    msg_list(prefix, sorted(test_list[key]))
  sys.exit(0)

# final check: if empty filelist, just quit nicely
if not (filelist or none):
  msg('no tests requested, bye!\n')
  sys.exit(0)

dirty_error = '''\
************************************************************
Dirty [Open]Molcas installation, cannot generate check files
************************************************************
  The compiled version of molcas does not correspond
to a clean source tree (does not match a git commit).
For reproducibility's sake generation of check files
is disabled.
  Please stash your changes and recompile.'''

autogen = '''\
# Automatically-generated file. Do not modify!
#
# To generate this file run {} {} {}
#
# Note that what matters is relative timings, so do not
# mix runs with different settings or environments.
'''

typelist = '''\
1: Basic tests that must pass
2: Additional tests that must pass
3: Other tests that may fail, but should be fixed
4: Personal development tests that may fail or not
'''

summary = '''\
************************************************************************
A total of {} test(s) failed, with {} critical failure(s).
************************************************************************
Please check the directory:
  {}
for the .out/.err files of the failed tests,
and check the submit directory:
  {}
for the working directories of the last run.
'''

if __name__ == '__main__':

  #--------------------------
  # set up the infrastructure
  #--------------------------

  if MOLCAS_ID is None:
    error('Could not find [Open]Molcas version')

  # the base directory for running tests
  if not os.path.isdir(testdir):
    try:
      os.mkdir(testdir)
    except:
      error('could not create {}'.format(testdir))
  if not os.access(testdir, os.W_OK | os.X_OK):
    error('cannot write to {}'.format(testdir))
  if os.path.isdir(tmpdir):
    shutil.rmtree(tmpdir)
  if opt['clean'] and os.path.isdir(logroot):
    shutil.rmtree(logroot)
  if not os.path.isdir(tmpdir):
    os.mkdir(tmpdir)
  if not os.path.isdir(logroot):
    os.mkdir(logroot)
  # remove log directories not ending in .bak and older than a day
  try:
    subdir_names = os.listdir(logroot)
  except:
    error('cannot open directory {}'.format(logroot))
  for subdir_name in subdir_names:
    if subdir_name.endswith('.bak'):
      continue
    subdir = os.path.join(logroot, subdir_name)
    if not os.path.isdir(subdir):
      continue
    age = date - datetime.datetime.fromtimestamp(os.path.getmtime(subdir))
    if age.days > 1:
      shutil.rmtree(subdir)
  # finally, create the new log directory we are about to use
  if os.path.isdir(logdir):
    error('existing log: {}'.format(logdir))
  else:
    os.mkdir(logdir)
  
  try:
    os.chdir(tmpdir)
  except:
    error('could not switch to {}'.format(tmpdir))

  # Re-generate the version information from the build that will
  # actually be used to run the tests
  MOLCAS_ID = sp.check_output([MOLCAS_DRIVER, 'version', '-l'], cwd=MOLCAS).decode().strip()
  if opt['generate']:
    if 'dirty' in MOLCAS_ID:
      error(dirty_error)
    else:
      os.environ['MOLCAS_INFO'] = '{}!{}!{}!'.format(MOLCAS_ID, MACHINE, DATE)

  log_result = os.path.join(logdir, 'result')
  log_timing = os.path.join(logdir, 'result.timing')
  log_failed = os.path.join(logdir, 'failed')
  log_failed_list = os.path.join(log_failed, 'list')

  try:
    os.mkdir(log_failed)
  except:
    error('could not create {}'.format(log_failed))

  # set up links to the actual result files
  if os.path.islink(result):
    os.remove(result)
  if os.path.islink(timing):
    os.remove(timing)
  if os.path.islink(failed):
    os.remove(failed)
  os.symlink(os.path.relpath(log_result, testdir), result)
  os.symlink(os.path.relpath(log_timing, testdir), timing)
  os.symlink(os.path.relpath(log_failed, testdir), failed)

  # open the information files
  try:
    RESULT = open(log_result, 'w+', encoding='utf-8', buffering=1)
  except:
    error('cannot open file {}'.format(log_result))
  try:
    TIMING = open(log_timing, 'w+', encoding='utf-8', buffering=1)
  except:
    error('cannot open file {}'.format(log_timing))
  try:
    FAILED_LIST = open(log_failed_list, 'w+', encoding='utf-8', buffering=1)
  except:
    error('cannot open file {}'.format(log_failed_list))

  # start by printing headers
  print(header, file=RESULT)
  if opt['timest']:
    print(autogen.format(DRIVER_base, thisfile, ' '.join(sys.argv[1:])), file=TIMING)
  else:
    print(header, file=TIMING)

  print(typelist, file=RESULT)

  #-------------------------------------
  # loop over tests and run verification
  #-------------------------------------

  # get automatic number of jobs
  if opt['jobs'] < 1:
    try:
      n_cpu = len(os.sched_getaffinity(0))
      n_procs = int(os.environ.get('MOLCAS_NPROCS','1'))
      n_threads = int(os.environ.get('MOLCAS_THREADS','1'))
      opt['jobs'] = max(1, n_cpu//(n_procs*n_threads))
    except:
      opt['jobs'] = 1

  n_tests = len(filelist)

  if opt['cover']:
    msg('WARNING: you are running tests with code coverage,\n'
        '         this can add up to 1 min of time per test\n')
    msg('running code coverage startup... ')
    process = sp.Popen([MOLCAS_DRIVER, 'codecov', '-q', '--start'])
    shared['active_pids'].append(process.pid)
    process.wait()
    shared['active_pids'].remove(process.pid)
    msg('done\n')

  with multiprocessing.Manager() as manager:
    # set up shared data for the simultaneous jobs
    if opt['jobs'] > 1:
      shared = manager.dict()
      shared['lock'] = manager.Lock()
      shared['active_pids'] = manager.list()
    else:
      shared = {}
      shared['lock'] = NullContextManager()
      shared['active_pids'] = []
    shared['index'] = 0
    shared['prompt'] = ''
    shared['prev_group'] = '.none'
    shared['skipped_tests'] = 0
    shared['failed_tests'] = 0
    shared['failed_critical_tests'] = 0
    shared['endloop'] = False
    if opt['jobs'] > 1:
      msg_nl(['Running {} simultaneous jobs, output may appear in wrong order'.format(opt['jobs'])])
      with cfProcessPool(opt['jobs']) as pool:
        tasks = [pool.submit(run_test, filename, shared) for filename in filelist]
        for task in concurrent.futures.as_completed(tasks):
          # this doesn't work as intended, though
          if shared['endloop']:
            pool.shutdown(wait=True, cancel_futures=True)
            break
    else:
      for filename in filelist:
        run_test(filename, shared)
        if shared['endloop']:
          break
    if interactive:
      msg_overwrite('', shared['prompt'])
    if shared['prev_group'] != '.none':
      print('----------', file=RESULT)
    print('\n*Failed critical tests* {}'.format(shared['failed_critical_tests']), file=RESULT)
    # fetch necessary info before the manager is destroyed
    active_pids = shared['active_pids'][:]
    skipped_tests = shared['skipped_tests']
    failed_tests = shared['failed_tests']
    failed_critical_tests = shared['failed_critical_tests']

  shared = {'active_pids': active_pids}

  if opt['cover']:
    process = sp.Popen([MOLCAS_DRIVER, 'codecov', '--html'])
    shared['active_pids'].append(process.pid)
    process.wait()
    shared['active_pids'].remove(process.pid)

  RESULT.close()
  TIMING.close()
  FAILED_LIST.close()

  if failed_tests:
    # report directory with failed out/err relative to directory where verify was run
    log_rel = os.path.relpath(failed, starting_cwd)
    tmp_rel = os.path.relpath(tmpdir, starting_cwd)
    info = summary.format(failed_tests, failed_critical_tests, log_rel, tmp_rel)
    if failed_critical_tests:
      error(info)
    else:
      msg(info)

if opt['generate']:
  msg('Generation of check files has been completed\n')
else:
  msg('Verification has been completed\n')