load ../common
tag=CDI

setup () {
    [[ $ch_libc = glibc ]] || skip 'glibc only'
    nvidia_json=$BATS_TMPDIR/nvidia.json
    warn_tag='warning'
    if command -v nvidia-smi > /dev/null; then
        have_nvidia_smi=yes
    fi
    if command -v nvidia-ctk > /dev/null; then
        have_nvidia_ctk=yes
    fi
    if [[ -f /proc/driver/nvidia/version ]]; then
        have_nvidia_driver=yes
    fi
    if [[ $have_nvidia_smi && $have_nvidia_ctk && $have_nvidia_driver ]]; then
        have_nvidia=yes
    fi
}

glibc_version_ok () {
    # glibc 2.34 and later are incompatible with older versions, i.e.,
    # binaries built on a host with glibc ≥ 2.34 cannot run in a container
    # with < 2.34. See #1430 & https://news.ycombinator.com/item?id=32479545.
    host=$(ldd --version | grep -oE '[0-9.]+[^.]$')
    guest=$(ch-run "$1" -- ldd --version | grep -oE '[0-9.]+[^.]$')
    # Check version compatibility. If host glibc ≥ 2.34 and guest glibc < 2.34,
    # skip the test.
    if [[    $(printf "%s\n2.34" "$host"  | sort -V | head -1)  = 2.34 \
          && $(printf "%s\n2.34" "$guest" | sort -V | head -1) != 2.34 ]]; then
        pedantic_fail "glibc: host $host ≥ 2.34 > guest $guest"
    fi
}

img_use () {
    export LC_ALL=C  # target image has no locales
    name=$1
    img="${CH_TEST_IMGDIR}/${name}"
    echo "img: $name ($img)"
    if [[ ! -e $img ]]; then
        skip "image not found: $img"
    fi
    prerequisites_ok "$img"
    glibc_version_ok "$img"
    bb=/ch/cdim/$(echo "$PWD" | tr '/' '_')_sotest  # varies by install path
    echo "bind base: ${bb}"
}

nvidia_ok () {
    [[ $have_nvidia ]] || skip 'nVidia driver & user-space required'
}


@test "$tag: generate nvidia.json" {
    scope standard
    nvidia_ok

    nvidia-ctk cdi generate --output "$nvidia_json"
    printf 'wrote %s\n' "$nvidia_json"
    printf 'found CDI version: '
    sed -En 's/^.+"cdiVersion"\s*:\s*"([0-9.]+).+$/\1/p' < "$nvidia_json"
}


@test "$tag: Debian: file locations" {
    scope standard
    img_use debian_12ch

    # nothing on the host image
    if [[ $CH_TEST_PACK_FMT == *-unpack ]]; then
        test -e "${img}/ch/cdim" && exit 1
        [[ 0 = $(find "$img" -xdev -name '*sotest*' -printf . | wc -c) ]]
    fi

    printf '\n*** guest files\n'
    run ch-run --cdi=./sotest/cdi.json "$img" -- \
          find / -xdev -name "*sotest*" -printf "%y: %p -> %l\n"
    echo "$output"
    [[ $status -eq 0 ]]
    files=$(echo "$output" | grep -Fv "$warn_tag" | sed -E 's/ -> $//' | sort)
    #echo '-- actual file list --'
    #echo "$files"
    files_blessed=$(cat <<EOF
d: ${bb}_bin
d: ${bb}_lib
d: ${bb}_lib_libfabric
l: /usr/bin/sotest -> ${bb}_bin/sotest
l: /usr/lib/libfabric/libsotest.so.1.0 -> ${bb}_lib_libfabric/libsotest-fi.so
l: /usr/lib/libsotest.so.1 -> libsotest.so.1.0
l: /usr/lib/libsotest.so.1.0 -> ${bb}_lib/libsotest.so.1.0
EOF
)
    #echo '-- diff vs. expected --'
    diff -u <(echo "$files_blessed") <(echo "$files")
}


@test "$tag: Debian: sotest: file path" {
    scope standard
    img_use debian_12ch

    run ch-run --cdi=./sotest/cdi.json "$img" -- sotest
    echo "$output"
    [[ $status -eq 0 ]]
    out_blessed=$(cat <<'EOF'
libsotest says 8675308 incremented is 8675309
EOF
)
    diff -u <(echo "$out_blessed") <(echo "$output" | grep -Fv "$warn_tag")
}


@test "$tag: Debian: sotest: all resources" {
    scope standard
    img_use debian_12ch

    # command line option
    ch-run -d -v --cdi-dirs=./sotest "$img" -- sotest

    # environment variable
    export CH_RUN_CDI_DIRS=$PWD/sotest
    ch-run -d "$img" -- sotest
}


@test "$tag: Debian: sotest: by kind" {
    scope standard
    img_use debian_12ch
    export CH_RUN_CDI_DIRS=$PWD/sotest:/doesnotexist  # see #2065

    for kind in weird/al weird/al=all; do
        run ch-run -v --cdi="$kind" "$img" -- sotest
        echo "$output"
        [[ $status -eq 0 ]]
        [[ $output = *"CDI: looking for resource files in: $PWD/sotest"* ]]
        [[ $output = *"CDI: spec read OK: weird/al: $PWD/sotest/cdi.json"* ]]
        [[ $output = *"CDI: directory not found, skipping: /doesnotexist"* ]]
    done
}


@test "$tag: Debian: sotest: no specs found" {
    scope standard
    img_use debian_12ch

    printf '\n*** file: not found\n'
    run ch-run -v --cdi=/doesnotexist.json "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *"error: can't stat CDI resource file: /doesnotexist.json"* ]]

    printf '\n*** kind: resource files found but not what we wanted\n'
    run ch-run -v --cdi-dirs="$PWD/sotest" --cdi=doesnot/exist "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *'error: CDI: no matching resources found'* ]]

    printf '\n*** kind: CDI directories don’t exist\n'
    run ch-run -v --cdi-dirs=/nope1:/nope2 --cdi=doesnot/exist "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *'error: CDI: no directories found: /nope1:/nope2'* ]]

    printf '\n*** kind: CDI directories have no resource files\n'
    run ch-run -v --cdi-dirs=/usr/bin --cdi=doesnot/exist "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *'error: CDI: no resource files found: /usr/bin'* ]]

    printf '\n*** all: CDI directories don’t exist\n'
    run ch-run -v --cdi --cdi-dirs=/nope1:/nope2 "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *'error: CDI: no directories found: /nope1:/nope2'* ]]

    printf '\n*** kind: CDI directories have no resource files\n'
    run ch-run -v --cdi --cdi-dirs=/usr/bin "$img" -- sotest
    echo "$output"
    [[ $status -eq $CH_ERR_MISC ]]
    [[ $output = *'error: CDI: no resource files found: /usr/bin'* ]]
}


@test "$tag: warnings" {
    scope standard
    img_use debian_12ch

    run ch-run --cdi=./sotest/cdi.json "$img" -- true
    echo "$output"
    [[ $status -eq 0 ]]
    out_blessed=$(cat <<'EOF'
ch-run[XXX]: warning: CDI: ignoring unknown hook: createContainer: unknown-hook (json.c:YYY)
ch-run[XXX]: warning: CDI: ignoring non-bind mount: /dev/sda -> /mnt/sda (json.c:YYY)
EOF
)
    diff -u <(echo "$out_blessed") \
            <(  echo "$output" \
              | grep -F "$warn_tag" \
              | sed -E -e 's/\[[0-9]+\]/[XXX]/g' \
                       -e 's/:[0-9]+\)$/:YYY)/g' \
                       -e '/reprinting first/,$d' )
}


@test "$tag: with --seccomp" {
    scope standard
    img_use debian_12ch
    ch-run --seccomp --cdi=./sotest/cdi.json "$img" -- sotest
}


@test "$tag: nvidia.json: no warnings" {
    scope standard
    img_use debian_12ch
    nvidia_ok

    run ch-run -v --cdi="$nvidia_json" "$img" -- true
    echo "$output"
    [[ $output == *'spec read OK'* ]]
    [[ $output != *'warning'* ]]
}


@test "$tag: nvidia-smi" {
    scope standard
    img_use debian_12ch
    nvidia_ok

    host_output=$(nvidia-smi -L)
    echo "host nvidia-smi(1):"
    echo "$host_output"

    run ch-run --cdi="$nvidia_json" "$img" -- nvidia-smi -L
    echo "guest nvidia-smi(1):"
    echo "$output"
    [[ $status -eq 0 ]]
    echo "diff:"
    diff -u <(echo "$host_output") <(echo "$output")
}


@test "$tag: matrixMulCUBLAS" {
    scope standard
    img_use nvidia
    nvidia_ok
    export CH_RUN_CDI_DIRS=$BATS_TMPDIR

    # Without CDI: failure.
    run ch-run "$img" -- matrixMulCUBLAS
    echo "$output"
    [[ $status -ne 0 ]]
    [[ $output = *'CUDA error at'*'cudaGetDeviceCount'* ]]

    # With CDI: runs OK and output looks OK.
    run ch-run -d "$img" -- matrixMulCUBLAS
    echo "$output"
    [[ $status -eq 0 ]]
    [[ $output = *'GPU Device 0'* ]]
    [[ $output = *'Comparing CUBLAS Matrix Multiply with CPU results: PASS'* ]]
}


# FIXME: errors
# FIXME: AlmaLinux
# FIXME: multi-GPU