#!/usr/bin/env python3
#===============================================================================
# Copyright 2013 NetApp, Inc. All Rights Reserved,
# contribution by Jorge Mora <mora@netapp.com>
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#===============================================================================
import os
import time
import errno
import signal
import struct
import traceback
from formatstr import *
import nfstest_config as c
from baseobj import BaseObj
from nfstest.test_util import TestUtil
import packet.nfs.nfs3_const as nfs3_const
import packet.nfs.nfs4_const as nfs4_const
import packet.nfs.nlm4_const as nlm4_const
from fcntl import fcntl,F_RDLCK,F_WRLCK,F_UNLCK,F_SETLK,F_SETLKW,F_GETLK

# Module constants
__author__    = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL
__copyright__ = "Copyright (C) 2013 NetApp, Inc."
__license__   = "GPL v2"
__version__   = "1.3"

USAGE = """%prog --server <server> [--client <client>] [options]

Locking tests
=============
Basic locking tests verify that a lock is granted using various arguments
to fcntl. These include blocking and non-blocking locks, read or write locks,
where the file is opened either for reading, writing or both. It also checks
different ranges including limit conditions.

Non-overlapping tests verity that locks are granted on both the client under
test and a second process or a remote client when locking the same file.

Overlapping tests verity that a lock is granted on the client under test
and a second process or a remote client trying to lock the same file will
be denied if a non-blocking lock is issue or will be blocked if a blocking
lock is issue on the second process or remote client.

Examples:
    Run the tests which use only the main client (no client option):
        %prog --server 192.168.0.2 --export /exports

    Use short options instead:
        %prog -s 192.168.0.2 -e /exports

    Use positional arguments with nfsversion=3 for extra client:
        %prog -s 192.168.0.2 -e /exports --client 192.168.0.10:::3

    Use named arguments instead:
        %prog -s 192.168.0.2 -e /exports --client 192.168.0.10:nfsversion=3

    Use positional arguments with nfsversion=3 for second process:
        %prog -s 192.168.0.2 -e /exports --nfsopts :::3

    Use named arguments instead:
        %prog -s 192.168.0.2 -e /exports --nfsopts nfsversion=3

Notes:
    The user id in the local host and the host specified by --client must
    have access to run commands as root using the 'sudo' command without
    the need for a password.

    The user id must be able to 'ssh' to remote host without the need for
    a password."""

# Test script ID
SCRIPT_ID = "LOCK"

# Basic tests
BTESTS = ['btest01']

# Non-overlapping lock tests using a second process
NPTESTS = [
    'nptest01',
    'nptest02',
    'nptest03',
    'nptest04',
]
# Non-overlapping lock tests using a second client
NCTESTS = [
    'nctest01',
    'nctest02',
    'nctest03',
    'nctest04',
]
# Overlapping lock tests using a second process
OPTESTS = [
    'optest01',
    'optest02',
    'optest03',
    'optest04',
    'optest05',
    'optest06',
    'optest07',
    'optest08',
    'optest09',
]
# Overlapping lock tests using a second client
OCTESTS = [
    'octest01',
    'octest02',
    'octest03',
    'octest04',
    'octest05',
    'octest06',
    'octest07',
    'octest08',
    'octest09',
]

# Dictionary having the number of processes required by each test
TEST_PROCESS_DICT = {x:1 for x in NPTESTS + OPTESTS}
TEST_PROCESS_DICT.update({x:2 for x in ["optest09"]})

# Dictionary having the number of clients required by each test
TEST_CLIENT_DICT = {x:1 for x in NCTESTS + OCTESTS}
TEST_CLIENT_DICT.update({x:2 for x in ["octest09"]})

# All tests, include the test groups in the list of test names
# so they are displayed in the help
TESTNAMES = BTESTS + ["noverlap", "nptest"] + NPTESTS + ["nctest"] + \
            NCTESTS + ["overlap", "optest"] + OPTESTS + ["octest"] + OCTESTS

TESTGROUPS = {
    "noverlap": {
         "tests": NPTESTS + NCTESTS,
         "desc": "Run all non-overlapping locking tests: ",
    },
    "nptest": {
         "tests": NPTESTS,
         "desc": "Run all non-overlapping locking tests using a second process: ",
    },
    "nctest": {
         "tests": NCTESTS,
         "desc": "Run all non-overlapping locking tests using a second client: ",
    },
    "overlap": {
         "tests": OPTESTS + OCTESTS,
         "desc": "Run all overlapping locking tests: ",
    },
    "optest": {
         "tests": OPTESTS,
         "desc": "Run all overlapping locking tests using a second process: ",
    },
    "octest": {
         "tests": OCTESTS,
         "desc": "Run all overlapping locking tests using a second client: ",
    },
}

# Mapping dictionaries
LOCKMAP    = {F_RDLCK:'F_RDLCK', F_WRLCK:'F_WRLCK', F_UNLCK:'F_UNLCK'}
LOCKMAP_R  = {'read':F_RDLCK, 'write':F_WRLCK, 'unlock':F_UNLCK}
SLOCKMAP   = {F_SETLK:'F_SETLK', F_SETLKW:'F_SETLKW'}
SLOCKMAP_R = {'immediate':F_SETLK, 'block':F_SETLKW}
OPENMAP    = {os.O_RDONLY:'O_RDONLY', os.O_WRONLY:'O_WRONLY', os.O_RDWR:'O_RDWR'}
OPENMAP_R  = {'read':os.O_RDONLY, 'write':os.O_WRONLY, 'rdwr':os.O_RDWR}

# Locking and helper functions
def getlock(fd, lock_type, offset=0, length=0, stype=F_SETLK, timeout=30):
    """Get byte range lock on file given by file descriptor"""
    lockdata = struct.pack('hhllhh', lock_type, 0, offset, length, 0, 0)
    if stype == F_SETLK:
        out = fcntl(fd, stype, lockdata)
    else:
        # Set alarm so the blocking lock could be interrupted
        signal.alarm(timeout)
        try:
            out = fcntl(fd, stype, lockdata)
        finally:
            # Reset alarm
            signal.alarm(0)
    return struct.unpack('hhllhh', out)

def testlock(fd, lock_type, offset=0, length=0):
    """Test byte range lock on file given by file descriptor"""
    lockdata = struct.pack('hhllhh', lock_type, 0, offset, length, 0, 0)
    out = fcntl(fd, F_GETLK, lockdata)
    return struct.unpack('hhllhh', out)

def get_ioerror(ioerrno, err):
    """Return fail message when expecting an error"""
    fmsg = ""
    # Test expression to return
    expr = ioerrno == err
    if not ioerrno:
        fmsg = ": no error was returned"
    elif ioerrno != err:
        # Got the wrong error
        expected = errno.errorcode[err]
        error = errno.errorcode[ioerrno]
        fmsg =  ": expecting %s, got %s" % (expected, error)
    return (expr, fmsg)

def get_range(offset, length):
    """Return byte range (start, end) given by the offset and length"""
    if length == 0:
        # Lock until the end of file
        end = 0xffffffffffffffff
    else:
        end = offset + length - 1
    return (offset, end)

class ProcInfo(BaseObj):
    """ProcInfo object"""
    # Class attributes:
    # ProcInfo object could be used as a direct replacement for Rexec object
    _fattrs = ("execobj",)
    # ProcInfo local and remote ordinal number
    _procidx = [2, 2]

    def __init__(self, clientobj, execobj):
        self.clientobj = clientobj  # Host object associated with this process
        self.execobj = execobj      # Rexec object for this process
        self.offset = 0  # Lock offset for this process
        self.length = 0  # Lock length for this process
        self.fd = None   # File descriptor associated with this process
        self.result = None  # Output of locking operation
        self.need_unlock = None # Lock was granted and it needs to be unlocked
        self.isoverlap = None   # Lock overlaps with main lock
        # Ordinal number for this object
        self.proc_ordnum = ordinal_number(self._procidx[self.remote])
        # Increment local or remote ordinal number
        self._procidx[self.remote] += 1

    def close_fd(self):
        """Close opened file"""
        if self.fd is not None:
            self.run(os.close, self.fd)
            self.fd = None

    def reset(self):
        """Reset lock info"""
        self.result = None
        self.need_unlock = None

    def lock_range(self, offset, length):
        """Set lock range and return this object"""
        self.offset = offset
        self.length = length
        self.isoverlap = None
        self.reset()
        return self

# Main test object definition
class LockTest(TestUtil):
    """LockTest object

       LockTest() -> New test object

       Usage:
           x = LockTest(testnames=['test1', ...])

           # Run all the tests
           x.run_tests()
           x.exit()
    """
    def __init__(self, **kwargs):
        """Constructor

           Initialize object's private data.
        """
        TestUtil.__init__(self, **kwargs)
        self.opts.version = "%prog " + __version__

        # Set default script options
        # Display all test messages
        self.opts.set_defaults(tverbose='2')

        # Options specific for this test script
        hmsg = "Remote NFS client and options used for conflicting lock tests. " \
               "Clients are separated by a ',' and each client definition is " \
               "a list of arguments separated by a ':' given in the following " \
               "order if positional arguments is used (see examples): " \
               "clientname:server:export:nfsversion:port:proto:sec:mtpoint"
        self.test_opgroup.add_option("--client", default=None, help=hmsg)
        hmsg = "Local process NFS options used for conflicting lock tests. " \
               "Processes are separated by a ',' and each process definition " \
               "is a list of arguments separated by a ':' given in the " \
               "following order if positional arguments is used (see examples): " \
               ":server:export:nfsversion:port:proto:sec:mtpoint"
        self.test_opgroup.add_option("--nfsopts", default=None, help=hmsg)
        hmsg = "Offset of first lock granted [default: %default]"
        self.test_opgroup.add_option("--offset", default="4k", help=hmsg)
        hmsg = "Length of first lock granted [default: %default]"
        self.test_opgroup.add_option("--length", default="4k", help=hmsg)
        # Object attribute: self.unlock_delay
        hmsg = "Time in seconds to unlock first lock [default: %default]"
        self.test_opgroup.add_option("--unlock-delay", type="float", default=2.0, help=hmsg)
        # Object attribute: self.lockw_timeout
        hmsg = "Time in seconds to wait for blocked lock after " \
               "conflicting lock has been released [default: %default]"
        self.test_opgroup.add_option("--lockw-timeout", type="int", default=30, help=hmsg)
        hmsg = "List of open types to test [default: %default]"
        self.test_opgroup.add_option("--opentype", default="read,write,rdwr", help=hmsg)
        hmsg = "List of lock types to test [default: %default]"
        self.test_opgroup.add_option("--locktype", default="read,write", help=hmsg)
        hmsg = "List of open types to test on remote client [default: %default]"
        self.test_opgroup.add_option("--opentype2", default="read,write,rdwr", help=hmsg)
        hmsg = "List of lock types to test on remote client [default: %default]"
        self.test_opgroup.add_option("--locktype2", default="read,write", help=hmsg)
        hmsg = "List of set lock types to test [default: %default]"
        self.test_opgroup.add_option("--setlock", default="immediate,block", help=hmsg)
        hmsg = "Create a packet trace for each sub-test. Use it with " + \
               "caution since it will create a lot of packet traces. " + \
               "Use --createtraces instead unless trying to get a packet " + \
               "trace for a specific sub-test. Best if it is used in " + \
               "combination with the --runtest option."
        self.cap_opgroup.add_option("--subtraces", action="store_true", default=False, help=hmsg)
        self.scan_options()
        self.st_time = time.time()

        # Convert units
        self.offset = int_units(self.offset)
        self.length = int_units(self.length)

        if self.subtraces:
            # Disable createtraces option since a packet trace will be created
            # here for every sub-test when the --subtraces option is set and
            # a packet trace should not be started by test_util.py
            self.createtraces = False

        # Sanity checks
        if self.offset < 2:
            self.opts.error("invalid value given in --offset [%d], must be > 1" % self.offset)
        if self.length < 2:
            self.opts.error("invalid value given in --length [%d], must be > 1" % self.length)

        if float(self.lockw_timeout) < (1.2*self.unlock_delay):
            self.opts.error("invalid value given in --lockw-timeout, must be greater than 1.2(--unlock-delay)")

        # Process options
        self.open_list  = self.get_list(self.opentype,  OPENMAP_R)
        self.lock_list  = self.get_list(self.locktype,  LOCKMAP_R)
        self.open2_list = self.get_list(self.opentype2, OPENMAP_R)
        self.lock2_list = self.get_list(self.locktype2, LOCKMAP_R)
        self.setl_list  = self.get_list(self.setlock,   SLOCKMAP_R)
        if self.open_list is None:
            self.opts.error("invalid type given in --opentype [%s]" % self.opentype)
        if self.lock_list is None:
            self.opts.error("invalid type given in --locktype [%s]" % self.locktype)
        if self.open2_list is None:
            self.opts.error("invalid type given in --opentype2 [%s]" % self.opentype2)
        if self.lock2_list is None:
            self.opts.error("invalid type given in --locktype2 [%s]" % self.locktype2)
        if self.setl_list is None:
            self.opts.error("invalid type given in --setlock [%s]" % self.setlock)

    def lock_setup(self, **kwargs):
        """Setup for locking tests:
             - start different processes if needed
             - start the remote procedure server on other clients if needed
             - mount the exported file system on remote clients
             - mount the exported file system locally for each extra
               process if different NFS options are specified
             - mount the exported file system locally for the main process
             - call setup()

           Arguments are passed to the main setup() method
        """
        # List of local(index=0) and remote(index=1) ProcInfo items
        self.proc_info_list = ([], [])

        # Find how many extra processes and clients should be started
        nclients = 0
        nprocesses = 0
        for tname in self.testlist:
            ncount = TEST_CLIENT_DICT.get(tname, 0)
            nclients = max(nclients, ncount)
            ncount = TEST_PROCESS_DICT.get(tname, 0)
            nprocesses = max(nprocesses, ncount)

        client_list = self.process_client_option(count=nclients)
        self.verify_client_option(TEST_CLIENT_DICT)

        nfsopts_list = self.process_client_option("nfsopts", remote=False, count=nprocesses)

        # Flush log file before starting child processes
        self.flush_log()

        # Start remote procedure server(s) locally
        for nfsopt_item in nfsopts_list:
            self.create_proc_info(nfsopt_item)

        # Start remote procedure server(s) remotely
        for client_item in client_list:
            self.create_proc_info(client_item)

        # Unmount server on local host
        self.umount()
        # Mount server on local host
        self.mount()

        # Call base object setup method
        self.setup(**kwargs)

    def start_rexec(self, clientobj):
        """Start remote procedure server locally or on the host given by
           the client object.
           Set up the remote server with helper functions to lock and
           unlock a file.

           clientobj:
               Client object where the remote procedure server will be started
        """
        # Start remote procedure server on given client
        execobj = self.create_rexec(clientobj.host)

        # Setup function to lock and unlock a file
        execobj.rimport("fcntl", ["fcntl", "F_SETLK"])
        execobj.rimport("struct")
        execobj.rimport("signal")
        execobj.rcode(getlock)
        # Set SIGALRM handler to do nothing but not ignoring the signal
        # just to interrupt a blocked lock
        execobj.reval("signal.signal(signal.SIGALRM, lambda signum,frame:None)")
        return ProcInfo(clientobj, execobj)

    def create_proc_info(self, proc_item):
        """Create a ProcInfo object and mount server if necessary"""
        # Create a copy of the process item
        client_args = dict(proc_item)
        # Create a Host object for the given client
        client_name = client_args.pop("client", "")
        clientobj = self.create_host(client_name, **client_args)
        if proc_item.get("mount"):
            # Mount only if necessary
            clientobj.umount()
            clientobj.mount()
        # Start a remote procedure server locally or remotely
        pinfo = self.start_rexec(clientobj)
        self.proc_info_list[pinfo.remote].append(pinfo)
        return pinfo

    def get_info_list(self, lock_list, remote=0):
        """Return a list of ProcInfo objects representing the locks in the
           given list.

           lock_list:
               List of lock definitions where each definition is given by a
               tuple (offset, length)
           remote:
               Using a remote client or a local process [default: 0]
        """
        idx = 0
        info_list = []
        x_info_list = self.proc_info_list[remote]
        for offset, length in lock_list:
            # Set the lock range in the ProcInfo object and add it to the list
            info_list.append(x_info_list[idx].lock_range(offset, length))
            idx += 1
        return info_list

    def get_time(self):
        """Return the number of seconds since the object was instantiated"""
        return time.time() - self.st_time

    def get_opts_list(self, sflag=True, oflag=True, lflag=True, coflag=True, clflag=True):
        """Return a list of all the permutations given by all option lists

           sflag:
               If true, use blocking and non-blocking locks
           oflag:
               If true, use open type of read, write and rdwr
           lflag:
               If true, use lock type of read, write and unlock
           coflag:
               If true, use open type of read, write and rdwr on second
               process or client
           clflag:
               If true, use lock type of read, write and unlock on second
               process or client
        """
        ret = []
        setl_list  = self.setl_list  if sflag  else [None]
        open_list  = self.open_list  if oflag  else [None]
        lock_list  = self.lock_list  if lflag  else [None]
        open2_list = self.open2_list if coflag else [None]
        lock2_list = self.lock2_list if clflag else [None]
        for stype in setl_list:
            for oltype in open_list:
                for ltype in lock_list:
                    for octype in open2_list:
                        for ctype in lock2_list:
                            ret.append({
                                'stype'  : stype,
                                'oltype' : oltype,
                                'ltype'  : ltype,
                                'octype' : octype,
                                'ctype'  : ctype,
                            })
        return ret

    def open_file(self, otype, pinfo=None):
        """Open file with given open type on either local or remote client

           otype:
               Open type, either O_RDONLY, O_WRONLY or O_RDWR
           pinfo:
               ProcInfo object to open file on a different process or client
               [default: None]
        """
        filename = self.files[0]
        # Use the correct mount point in the full path of file
        if pinfo is None:
            absfile = self.abspath(filename)
        else:
            absfile = pinfo.clientobj.abspath(filename)

        if otype & os.O_WRONLY == os.O_WRONLY:
            ostr = "writing"
        elif otype & os.O_RDWR == os.O_RDWR:
            ostr = "reading and writing"
        else:
            ostr = "reading"

        if pinfo:
            # OPEN on different process or remote client
            pstr = "client" if pinfo.remote else "process"
            self.dprint('DBG2', "Open file for %s [%s] on %s %s" % (ostr, filename, pinfo.proc_ordnum, pstr))
            fd = pinfo.run(os.open, absfile, otype)
        else:
            # OPEN locally
            self.dprint('DBG2', "Open file for %s [%s]" % (ostr, filename))
            fd = os.open(absfile, otype)
        return fd

    def do_lock_test(self, fd, ltype, offset=None, length=None, msg="", submsg=None, stype=F_SETLK, block=False, error=0, pinfo=None):
        """Do the actual lock on a file given by the file descriptor

           fd:
               File descriptor of file to lock
           ltype:
               Lock type: F_RDLCK, F_WRLCK or F_UNLCK
           offset:
               Starting offset of byte range lock [default: None]
               If offset is None then use the offset from the ProcInfo
               object given by pinfo
           length:
               Length of byte range lock [default: None]
               If length is None then use the length from the ProcInfo
               object given by pinfo
           msg:
               Test message to display [default: ""]
           submsg:
               Subtest message to display [default: None]
           stype:
               Blocking lock type: F_SETLK or F_SETLKW [default: F_SETLK]
           block:
               Expect lock to block [default: False]
           error:
               Expected locking error [default: 0]
           pinfo:
               ProcInfo object to lock file on a different process or client
               [default: None]
        """
        try:
            fmsg = ""
            pexpr = False
            ioerrno = 0
            if pinfo:
                # Use the lock range given in the ProcInfo object
                offset = pinfo.offset if offset is None else offset
                length = pinfo.length if length is None else length

            # Set up debugging message options
            lmsg = "Unlock"  if ltype == F_UNLCK else "Lock"
            smsg = "F_SETLK" if stype == F_SETLK else "F_SETLKW"
            (start, end) = get_range(offset, length)
            info = "%s file (%s, %s) off=%d len=%d range(%d, %d)" % (lmsg, LOCKMAP[ltype], smsg, offset, length, start, end)
            if pinfo:
                # Lock file in different process or remote client
                pstr = "client" if pinfo.remote else "process"
                self.dprint('DBG1', info + " on %s %s @%.2f" % (pinfo.proc_ordnum, pstr, self.get_time()))
                nowait = True if stype == F_SETLKW else False
                out = pinfo.run("getlock", fd, ltype, offset, length, stype, self.lockw_timeout, NOWAIT=nowait)
                if nowait:
                    # It is a blocking lock, it could block if overlapping range
                    # Poll Rexec object to make sure the lock is blocked if it
                    # is expected to block
                    pexpr = pinfo.poll(0.1)
                    if pexpr:
                        if block:
                            # Expected to block, but did not
                            fmsg = ": lock did not block"
                        elif error != errno.EAGAIN:
                            # Get lock results
                            out = pinfo.results()
            else:
                # Lock file locally
                self.dprint('DBG1', info)
                getlock(fd, ltype, offset, length, stype, self.lockw_timeout)
        except IOError as ioerr:
            # Set up fail message if expecting no errors
            ioerrno = ioerr.errno
            errstr = errno.errorcode[ioerr.errno]
            self.dprint("DBG7", "Got error %s" % errstr)
            fmsg = ": got error %s" % errstr
        if error:
            # Expecting an error
            (expr, fmsg) = get_ioerror(ioerrno, error)
        else:
            # Not expecting an error
            expr = not ioerrno and (not block or not pexpr)
        self.test(expr, msg, subtest=submsg, failmsg=fmsg)
        return ioerrno is None

    def wait_for_lock(self, fd, info_list, lockmsg, submsg_list):
        """Wait for blocked lock

           fd:
               File descriptor of file holding the current lock, so it will
               be unlocked to let the blocked lock be granted
           info_list:
               List of ProcInfo objects with info for each blocked lock
               on a different process or client
           lockmsg:
               Description of current lock
           submsg_list:
               List of subtest messages to display for each lock given in
               info_list
        """
        fmsg = ""
        out = None
        expr = False
        # Flag used for keeping track of the time since the first client
        # unlocked the file and timed out if blocked lock is not granted
        sl_time = 0
        # Flag used for waiting to unlock file on the first client
        stime = time.time()
        # Flag used to verify if the unlock of the first client was done
        need_unlock = True
        # Polling granularity
        delta = self.unlock_delay/5.0
        self.dprint('DBG3', "Wait %.2f secs to unlock conflicting lock @%.2f" % (self.unlock_delay, self.get_time()))
        dbgmsg = "Check if blocked lock is still waiting"
        while True:
            self.dprint('DBG3', "%s @%.2f" % (dbgmsg, self.get_time()))
            if need_unlock and (expr or time.time() - stime >= self.unlock_delay):
                # Unlock current file lock so the blocked lock could be granted
                msg = "Unlocking full file after delay should be granted"
                self.do_lock_test(fd, F_UNLCK, 0, 0, msg, lockmsg)
                if expr:
                    # Blocked lock has already been granted
                    break
                need_unlock = False
                sl_time = time.time()
                self.dprint('DBG3', "Wait up to %d secs to check if blocked lock has been granted @%.2f" % (self.lockw_timeout, self.get_time()))
                dbgmsg = "Check if blocked lock has been granted"
                delta = self.lockw_timeout/30.0
                if delta < 0.2:
                    delta = 0.2
            delta_time = delta
            idx = 0
            ordnum = ""
            for pinfo in info_list:
                submsg = submsg_list[idx]
                idx += 1
                if len(info_list) > 1:
                    ordnum = "%s " % ordinal_number(idx)
                if pinfo.need_unlock is not None:
                    # This lock has already been granted
                    continue
                if pinfo.poll(delta_time):
                    out = None
                    try:
                        # Blocking lock just returned
                        self.dprint('DBG3', "Getting results from %sblocked lock @%.2f" % (ordnum, self.get_time()))
                        out = pinfo.results()
                        expr = True
                    except Exception as e:
                        # Unable to get results from blocked lock
                        if getattr(e, "errno", None) == errno.EINTR:
                            self.test(False, "Timeout waiting for %sblocked lock to be granted" % ordnum, subtest=submsg)
                        else:
                            self.test(False, "Error while getting results from %sblocked lock" % ordnum, subtest=submsg, failmsg=": %s" % e)
                    pinfo.result = out
                    pinfo.need_unlock = need_unlock
                delta_time = min(delta/10.0, 0.01)
            if not need_unlock:
                if [x.need_unlock for x in info_list].count(None) == 0:
                    # All locks have been granted or timed out
                    break
        idx = 0
        for pinfo in info_list:
            submsg = submsg_list[idx]
            idx += 1
            if pinfo.need_unlock:
                self.test(not pinfo.result, "Blocked lock is granted before conflicting lock was unlocked", subtest=submsg)
            else:
                self.test(pinfo.result, "Blocked lock is granted after conflicting lock is released", subtest=submsg, failmsg=fmsg)

    def basic_lock(self, info_list, oltype, ltype, octype, ctype, stype):
        """This is the main locking method for overlapping and non-overlapping
           tests. This method does the following:
               1. Open file locally
               2. Lock file locally -- this is the conflicting lock
               3. Open file on a different process or remote client
               4. Lock file on a different process or remote client
               5. If locks do not overlap, verify both locks are granted
               6. If using a blocking lock on an overlapping range, then wait
                  for the number of seconds given by option unlock-delay and
                  verify the blocking lock is not granted until the conflicting
                  lock has been unlocked at the end of the wait period. Once the
                  conflicting lock has been unlocked verify the blocked lock
                  is granted
               7. If using a non-blocking lock on an overlapping range, verify
                  the correct error is returned
               8. Unlock both the local and remote locks

           info_list:
               List of ProcInfo objects with info for each extra lock to take
               on a different process or client
           oltype:
               Open type to use on local file
           ltype:
               Lock type to use on local file
           octype:
               Open type to use on second file (other process or remote)
           ctype:
               Lock type to use on second file (other process or remote)
           stype:
               Either a blocking or non-blocking lock
        """
        try:
            err = 0
            fdl = None

            if self.subtraces:
                self.trace_start()

            if self.createtraces or self.subtraces:
                # Have a marker on the packet trace for the running test,
                # this will make the client send a LOOKUP with the test
                # info as the file name
                self.insert_trace_marker(self.sub_testname)

            # Find if ranges overlap
            isoverlap = info_list[0].isoverlap

            pstr = "client" if info_list[0].remote else "process"
            smsg1 = ", lock1(%s, %s, %s)" % (OPENMAP[oltype], LOCKMAP[ltype], SLOCKMAP[stype])
            smsg_list = []
            for i in range(len(info_list)):
                smsgx = ", lock%d(%s, %s, %s)" % (i+2, OPENMAP[octype], LOCKMAP[ctype], SLOCKMAP[stype])
                smsg_list.append(smsgx)

            # Set up main test message
            error = 0
            blocking = False
            ostr = "" if isoverlap else "non-"
            if ctype == F_RDLCK and octype == os.O_WRONLY or \
               ctype == F_WRLCK and octype == os.O_RDONLY:
                error = errno.EBADF
                imsg  = "return %s" % errno.errorcode[error]
            elif not isoverlap or (ltype == F_RDLCK and ctype == F_RDLCK):
                error = 0
                imsg  = "be granted"
                if isoverlap:
                    imsg += " since both locks are %s" % LOCKMAP[ltype]
            elif stype == F_SETLKW:
                error = 0
                imsg  = "block"
                blocking = True
            elif ltype != ctype or (ltype == F_WRLCK and ctype == ltype):
                error = errno.EAGAIN
                imsg  = "return %s" % errno.errorcode[error]
            else:
                error = 0
                imsg  = "be granted"

            lmsg = "be granted"
            if ltype == F_RDLCK and oltype == os.O_WRONLY or \
               ltype == F_WRLCK and oltype == os.O_RDONLY:
                err  = errno.EBADF
                lmsg = "return %s" % errno.errorcode[err]

            # Open file on main process and lock it, this will become the
            # conflicting lock
            fdl = self.open_file(oltype)
            submsg = " should %s%s" % (lmsg, smsg1)
            msg = "Locking byte range"
            self.do_lock_test(fdl, ltype, self.offset, self.length, msg, submsg, stype=stype, error=err)

            if not err:
                idx = 0
                for pinfo in info_list:
                    pinfo.fd = self.open_file(octype, pinfo)
                    submsg = " should %s%s" % (imsg, smsg_list[idx])
                    msg = "Locking with %soverlapping range on %s %s" % (ostr, pinfo.proc_ordnum, pstr)
                    locked = self.do_lock_test(pinfo.fd, ctype, msg=msg, submsg=submsg, stype=stype, error=error, pinfo=pinfo, block=blocking)
                    idx += 1

                if blocking:
                    # Wait for blocked lock to be granted by unlocking the
                    # conflicting lock
                    self.wait_for_lock(fdl, info_list, smsg1, smsg_list)
                else:
                    # No locking conflict so unlock local file
                    msg = "Unlocking full file should be granted"
                    self.do_lock_test(fdl, F_UNLCK, 0, 0, msg, smsg1)

                    if not locked and error != errno.EBADF:
                        xmsg = "be granted"
                        if ctype == F_RDLCK and octype == os.O_WRONLY or \
                           ctype == F_WRLCK and octype == os.O_RDONLY:
                            err  = errno.EBADF
                            xmsg = "return %s" % errno.errorcode[err]
                        idx = 0
                        for pinfo in info_list:
                            submsg = " should %s%s" % (xmsg, smsg_list[idx])
                            msg = "Locking byte range on %s %s" % (pinfo.proc_ordnum, pstr)
                            self.do_lock_test(pinfo.fd, ctype, msg=msg, submsg=submsg, stype=stype, error=err, pinfo=pinfo)
                            idx += 1
                if error != errno.EBADF:
                    idx = 0
                    for pinfo in info_list:
                        msg = "Unlocking full file on %s %s should be granted" % (pinfo.proc_ordnum, pstr)
                        self.do_lock_test(pinfo.fd, F_UNLCK, 0, 0, msg, smsg_list[idx], pinfo=pinfo)
                        idx += 1
        except:
            self.test(False, traceback.format_exc())
        finally:
            # Close open files
            if fdl is not None:
                os.close(fdl)
            for pinfo in info_list:
                pinfo.close_fd()
            if self.subtraces:
                self.trace_stop()
                self.trace_open()
                self.pktt.close()

    def do_basic_lock(self, info_list, overlap=False):
        """This is the main locking method for testing all different
           permutations of the same test by varying the open type of both
           files, the locking type and for blocking and non-blocking locks.

           info_list:
               List of ProcInfo objects with info for each extra lock to take
               on a different process or client
           overlap:
               True if range is expected to overlap
        """
        # Find if ranges overlap
        (start1, end1) = get_range(self.offset, self.length)

        ridx = 2
        for pinfo in info_list:
            (start2, end2) = get_range(pinfo.offset, pinfo.length)
            isoverlap = (start1 <= end2 and start2 <= end1)
            pinfo.isoverlap = isoverlap

            # Check if ranges overlap when expected
            fmsg = ": range1(%d, %d), range%d(%d, %d)" % (start1, end1, ridx, start2, end2)
            if overlap and not isoverlap:
                self.test(False, "Range does not overlap", failmsg=fmsg)
                return
            elif not overlap and isoverlap:
                self.test(False, "Range overlaps", failmsg=fmsg)
                return
            ridx += 1

        testidx = 1
        for item in self.get_opts_list():
            stype  = item['stype']
            oltype = item['oltype']
            ltype  = item['ltype']
            octype = item['octype']
            ctype  = item['ctype']

            self.sub_testname = "%s_%03d" % (self.testname, testidx)

            if self.tverbose == 2:
                self.dprint("INFO", "Running %s" % self.sub_testname)

            # Reset ProcInfo objects for next test
            expr = self.nfs_version < 4
            for pinfo in info_list:
                expr = expr or pinfo.clientobj.nfs_version < 4
                pinfo.reset()

            if expr or info_list[0].remote:
                # Expect NFS errors only if NFS version < 4 or a remote client
                self.set_nfserr_list(
                    nfs3list=[nfs3_const.NFS3ERR_NOENT, nfs3_const.NFS3ERR_NOTEMPTY],
                    nfs4list=[nfs4_const.NFS4ERR_NOENT, nfs4_const.NFS4ERR_DENIED],
                    nlm4list=[nlm4_const.NLM4_BLOCKED, nlm4_const.NLM4_DENIED],
                )

            # Do actual test
            self.basic_lock(info_list, oltype, ltype, octype, ctype, stype)
            testidx += 1

    def btest01_test(self):
        """Basic locking tests
           These tests verify that a lock is granted using various arguments
           to fcntl. These include blocking and non-blocking locks, read or
           write locks, where the file is opened either for reading, writing
           or both. It also checks different ranges including limit conditions.
        """
        self.test_group("Basic locking tests")
        nmax = 0x7fffffff if self.nfs_version == 2 else 0x7fffffffffffffff
        tlist = [ (0, 0), (0, 1), (1, 0), (0, self.length),
                  (self.offset, self.length), (self.offset, 0),
                  (self.offset, 1), (0, nmax), (1, nmax), (nmax, 1),
                  (nmax, 0), (0, -1), (-1, 0) ]

        testidx = 1
        for offset, length in tlist:
            offstr = "NMAX" if offset == nmax else str(offset)
            lenstr = "NMAX" if length == nmax else str(length)

            for item in self.get_opts_list(coflag=False, clflag=False):
                try:
                    fd = None
                    self.sub_testname = "%s_%03d" % (self.testname, testidx)

                    if self.tverbose > 1:
                        self.dprint("INFO", "Running %s" % self.sub_testname)

                    if self.subtraces:
                        self.trace_start()

                    if self.createtraces or self.subtraces:
                        # Have a marker on the packet trace for the running
                        # test, this will make the client send a LOOKUP with
                        # the test info as the file name
                        self.insert_trace_marker(self.sub_testname)
                    testidx += 1

                    stype  = item['stype']
                    oltype = item['oltype']
                    ltype  = item['ltype']
                    lerr = 0
                    uerr = 0
                    if offset < 0 or length < 0:
                        lerr  = errno.EINVAL
                        uerr  = errno.EINVAL
                    elif ltype == F_RDLCK and oltype == os.O_WRONLY or \
                       ltype == F_WRLCK and oltype == os.O_RDONLY:
                        lerr  = errno.EBADF
                    lmsg = "return %s" % errno.errorcode[lerr] if lerr else "be granted"
                    umsg = "return %s" % errno.errorcode[uerr] if uerr else "be granted"
                    submsg = "open(%s) lock(%s, %s)" % (OPENMAP[oltype], LOCKMAP[ltype], SLOCKMAP[stype])
                    lsubmsg = " should %s, %s" % (lmsg, submsg)
                    usubmsg = " should %s, %s" % (umsg, submsg)

                    # Open file
                    fd = self.open_file(oltype)

                    msg = "Unlocking byte range (off:%s, len:%s) while file is not locked" % (offstr, lenstr)
                    self.do_lock_test(fd, F_UNLCK, offset, length, msg, usubmsg, stype=stype, error=uerr)

                    msg = "Locking byte range (off:%s, len:%s)" % (offstr, lenstr)
                    locked = self.do_lock_test(fd, ltype, offset, length, msg, lsubmsg, stype=stype, error=lerr)
                    if locked:
                        msg = "Unlocking byte range (off:%s, len:%s)" % (offstr, lenstr)
                        self.do_lock_test(fd, F_UNLCK, offset, length, msg, usubmsg, stype=stype, error=uerr)
                except:
                    self.test(False, traceback.format_exc())
                finally:
                    # Close open file
                    if fd is not None:
                        os.close(fd)
                    if self.subtraces:
                        self.trace_stop()
                        self.trace_open()
                        self.pktt.close()

    def ntest01(self, remote=0):
        """Locking non-overlapping range from a second process where end2 < start1
             process1:                     |------------------|
             process2: |--------|
        """
        pstr = "client" if remote else "process"
        self.test_group("Locking non-overlapping range from a second %s where end2 < start1" % pstr)
        self.do_basic_lock(self.get_info_list([(0, int(self.offset/2))], remote))

    def ntest02(self, remote=0):
        """Locking non-overlapping range from a second process where end2 == start1 - 1
             process1:                     |------------------|
             process2: |------------------|
        """
        pstr = "client" if remote else "process"
        self.test_group("Locking non-overlapping range from a second %s where end2 == start1 - 1" % pstr)
        self.do_basic_lock(self.get_info_list([(0, self.offset)], remote))

    def ntest03(self, remote=0):
        """Locking non-overlapping range from a second process where start2 > end1
             process1: |------------------|
             process2:                               |--------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset + self.length + int(self.length/2)
        length2 = int(self.length/2)
        self.test_group("Locking non-overlapping range from a second %s where start2 > end1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, length2)], remote))

        self.test_group("Locking non-overlapping range from a second %s where start2 > end1 and end2 == EOF" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, 0)], remote))

    def ntest04(self, remote=0):
        """Locking non-overlapping range from a second process where start2 == end1 + 1
             process1: |------------------|
             process2:                     |------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset + self.length
        self.test_group("Locking non-overlapping range from a second %s where start2 == end1 + 1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, self.length)], remote))

        self.test_group("Locking non-overlapping range from a second %s where start2 == end1 + 1 and end2 == EOF" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, 0)], remote))

    def otest01(self, remote=0):
        """Locking same range from a second process
             process1:                     |------------------|
             process2:                     |------------------|
        """
        pstr = "client" if remote else "process"
        self.test_group("Locking same range from a second %s" % pstr)
        self.do_basic_lock(self.get_info_list([(self.offset, self.length)], remote), overlap=True)

    def otest02(self, remote=0):
        """Locking overlapping range from a second process where start2 < start1
             process1:                     |------------------|
             process2:           |------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset - int(self.length/2)
        if offset2 < 0:
            offset2 = 0
        length2 = self.offset + int(self.length/2) - offset2
        self.test_group("Locking overlapping range from a second %s where start2 < start1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, length2)], remote), overlap=True)

        if offset2 > 0:
            length2 = self.offset + int(self.length/2)
            self.test_group("Locking overlapping range from a second %s where start2 < start1 and start2 == 0" % pstr)
            self.do_basic_lock(self.get_info_list([(0, length2)], remote), overlap=True)

    def otest03(self, remote=0):
        """Locking overlapping range from a second process where end2 > end1
             process1:                     |------------------|
             process2:                               |------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset + int(self.length/2)
        self.test_group("Locking overlapping range from a second %s where end2 > end1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, self.length)], remote), overlap=True)

        self.test_group("Locking overlapping range from a second %s where end2 > end1 and end2 == EOF" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, 0)], remote), overlap=True)

    def otest04(self, remote=0):
        """Locking overlapping range from a second process where range2 is entirely within range1
             process1:                     |------------------|
             process2:                          |--------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset + int(self.length/4)
        length2 = int(self.length/2)
        self.test_group("Locking overlapping range from a second %s where range2 is entirely within range1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, length2)], remote), overlap=True)

    def otest05(self, remote=0):
        """Locking overlapping range from a second process where range1 is entirely within range2
             process1:                     |------------------|
             process2:                |----------------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset - int(self.length/4)
        if offset2 < 0:
            offset2 = 0
        length2 = self.length + int(self.length/2)
        self.test_group("Locking overlapping range from a second %s where range1 is entirely within range2" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, length2)], remote), overlap=True)

        if offset2 > 0:
            length2 = self.offset + self.length + int(self.length/4)
            self.test_group("Locking overlapping range from a second %s where range1 is entirely within range2 and start2 == 0" % pstr)
            self.do_basic_lock(self.get_info_list([(0, length2)], remote), overlap=True)

        self.test_group("Locking overlapping range from a second %s where range1 is entirely within range2 and end2 == EOF" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, 0)], remote), overlap=True)

    def otest06(self, remote=0):
        """Locking full file range from a second process"""
        pstr = "client" if remote else "process"
        self.test_group("Locking full file range from a second %s" % pstr)
        self.do_basic_lock(self.get_info_list([(0, 0)], remote), overlap=True)

    def otest07(self, remote=0):
        """Locking overlapping range from a second process where end2 == start1
             process1:                     |------------------|
             process2:  |------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset - self.length + 1
        if offset2 < 0:
            offset2 = 0
        length2 = self.offset - offset2 + 1
        self.test_group("Locking overlapping range from a second %s where end2 == start1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, length2)], remote), overlap=True)

        if offset2 > 0:
            length2 = self.offset + 1
            self.test_group("Locking overlapping range from a second %s where end2 == start1 and start2 == 0" % pstr)
            self.do_basic_lock(self.get_info_list([(0, length2)], remote), overlap=True)

    def otest08(self, remote=0):
        """Locking overlapping range from a second process where start2 == end1
             process1:  |------------------|
             process2:                     |------------------|
        """
        pstr = "client" if remote else "process"
        offset2 = self.offset + self.length - 1
        self.test_group("Locking overlapping range from a second %s where start2 == end1" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, self.length)], remote), overlap=True)

        self.test_group("Locking overlapping range from a second %s where start2 == end1 and end2 == EOF" % pstr)
        self.do_basic_lock(self.get_info_list([(offset2, 0)], remote), overlap=True)

    def otest09(self, remote=0):
        """Locking overlapping range from multiple processes where range2 and
           range3 are entirely within range1
             process1:                     |-----------------------------|
             process2:                          |--------|
             process3:                                    |--------|
        """
        pstr = "clients" if remote else "processes"
        offset2 = self.offset + int(self.length/4)
        length2 = int(self.length/4)
        lock_list = [
            (offset2,         length2),
            (offset2+length2, length2),
        ]
        self.test_group("Locking overlapping range from multiple %s where " \
                        "range2 and range3 are entirely within range1" % pstr)
        info_list = self.get_info_list(lock_list, remote)
        self.do_basic_lock(info_list, overlap=True)

    def nptest01_test(self):
        """Locking non-overlapping range from a second process where end2 < start1
             process1:                     |------------------|
             process2: |--------|
        """
        self.ntest01(remote=0)

    def nptest02_test(self):
        """Locking non-overlapping range from a second process where end2 == start1 - 1
             process1:                     |------------------|
             process2: |------------------|
        """
        self.ntest02(remote=0)

    def nptest03_test(self):
        """Locking non-overlapping range from a second process where start2 > end1
             process1: |------------------|
             process2:                               |--------|
        """
        self.ntest03(remote=0)

    def nptest04_test(self):
        """Locking non-overlapping range from a second process where start2 == end1 + 1
             process1: |------------------|
             process2:                     |------------------|
        """
        self.ntest04(remote=0)

    def nctest01_test(self):
        """Locking non-overlapping range from a second client where end2 < start1
             client1:                      |------------------|
             client2:  |--------|
        """
        self.ntest01(remote=1)

    def nctest02_test(self):
        """Locking non-overlapping range from a second client where end2 == start1 - 1
             client1:                      |------------------|
             client2:  |------------------|
        """
        self.ntest02(remote=1)

    def nctest03_test(self):
        """Locking non-overlapping range from a second client where start2 > end1
             client1:  |------------------|
             client2:                                |--------|
        """
        self.ntest03(remote=1)

    def nctest04_test(self):
        """Locking non-overlapping range from a second client where start2 == end1 + 1
             client1:  |------------------|
             client2:                      |------------------|
        """
        self.ntest04(remote=1)

    def optest01_test(self):
        """Locking same range from a second process
             process1:                     |------------------|
             process2:                     |------------------|
        """
        self.otest01(remote=0)

    def optest02_test(self):
        """Locking overlapping range from a second process where start2 < start1
             process1:                     |------------------|
             process2:           |------------------|
        """
        self.otest02(remote=0)

    def optest03_test(self):
        """Locking overlapping range from a second process where end2 > end1
             process1:                     |------------------|
             process2:                               |------------------|
        """
        self.otest03(remote=0)

    def optest04_test(self):
        """Locking overlapping range from a second process where range2 is entirely within range1
             process1:                     |------------------|
             process2:                          |--------|
        """
        self.otest04(remote=0)

    def optest05_test(self):
        """Locking overlapping range from a second process where range1 is entirely within range2
             process1:                     |------------------|
             process2:                |----------------------------|
        """
        self.otest05(remote=0)

    def optest06_test(self):
        """Locking full file range from a second process"""
        self.otest06(remote=0)

    def optest07_test(self):
        """Locking overlapping range from a second process where end2 == start1
             process1:                     |------------------|
             process2:  |------------------|
        """
        self.otest07(remote=0)

    def optest08_test(self):
        """Locking overlapping range from a second process where start2 == end1
             process1:  |------------------|
             process2:                     |------------------|
        """
        self.otest08(remote=0)

    def optest09_test(self):
        """Locking overlapping range from multiple processes where range2 and
           range3 are entirely within range1
             process1:                     |-----------------------------|
             process2:                          |--------|
             process3:                                    |--------|
        """
        self.otest09(remote=0)

    def octest01_test(self):
        """Locking same range from a second client
             client1:                      |------------------|
             client2:                      |------------------|
        """
        self.otest01(remote=1)

    def octest02_test(self):
        """Locking overlapping range from a second client where start2 < start1
             client1:                      |------------------|
             client2:            |------------------|
        """
        self.otest02(remote=1)

    def octest03_test(self):
        """Locking overlapping range from a second client where end2 > end1
             client1:                      |------------------|
             client2:                                |------------------|
        """
        self.otest03(remote=1)

    def octest04_test(self):
        """Locking overlapping range from a second client where range2 is entirely within range1
             client1:                      |------------------|
             client2:                           |--------|
        """
        self.otest04(remote=1)

    def octest05_test(self):
        """Locking overlapping range from a second client where range1 is entirely within range2
             client1:                      |------------------|
             client2:                 |----------------------------|
        """
        self.otest05(remote=1)

    def octest06_test(self):
        """Locking full file range from a second client"""
        self.otest06(remote=1)

    def octest07_test(self):
        """Locking overlapping range from a second client where end2 == start1
             client1:                      |------------------|
             client2:   |------------------|
        """
        self.otest07(remote=1)

    def octest08_test(self):
        """Locking overlapping range from a second client where start2 == end1
             client1:   |------------------|
             client2:                      |------------------|
        """
        self.otest08(remote=1)

    def octest09_test(self):
        """Locking overlapping range from multiple clients where range2 and
           range3 are entirely within range1
             client1:                     |-----------------------------|
             client2:                          |--------|
             client3:                                    |--------|
        """
        self.otest09(remote=1)

################################################################################
# Entry point
x = LockTest(usage=USAGE, testnames=TESTNAMES, testgroups=TESTGROUPS, sid=SCRIPT_ID)

try:
    # Call setup
    x.lock_setup(nfiles=1)

    # Run all the tests
    x.run_tests()
except Exception:
    x.test(False, traceback.format_exc())
finally:
    x.cleanup()
    x.exit()