#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Run toolchain torture tests and llvm testsuite tests.
# For now, run on linux64, build and run unsandboxed newlib tests
# for all 3 architectures.
# Note: This script builds the toolchain from scratch but does
#       not build the translators and hence the translators
#       are from an older revision, see comment below.

set -o xtrace
set -o nounset
set -o errexit

readonly PNACL_BUILD="pnacl/build.sh"
readonly TOOLCHAIN_BUILD="toolchain_build/toolchain_build_pnacl.py"
readonly PACKAGES_SCRIPT="buildbot/packages.py"
readonly TEMP_PACKAGES="toolchain_build/out/packages.txt"
readonly UP_DOWN_LOAD="buildbot/file_up_down_load.sh"
readonly TORTURE_TEST="tools/toolchain_tester/torture_test.py"
readonly LLVM_TEST="pnacl/scripts/llvm-test.py"
readonly CLANG_UPDATE="../tools/clang/scripts/update.py"
readonly INSTALL_SUBDIR="toolchain/linux_x86/pnacl_newlib"
readonly INSTALL_ABSPATH=$(pwd)/${INSTALL_SUBDIR}
readonly LIBRARY_ABSPATH=${INSTALL_ABSPATH}/lib

# build.sh, llvm test suite and torture tests all use this value
export PNACL_CONCURRENCY=${PNACL_CONCURRENCY:-4}

# Change the  toolchain build script (PNACL_BUILD) behavior slightly
# wrt to error logging and mecurial retry delays.
# TODO(robertm): if this special casing is still needed,
#                make this into separate vars
export PNACL_BUILDBOT=true
# Make the toolchain build script (PNACL_BUILD) more verbose.
# This will also prevent bot timeouts which otherwise gets triggered
# by long periods without console output.
export PNACL_VERBOSE=true

EXIT_STATUS=0

clobber() {
  echo @@@BUILD_STEP clobber@@@
  rm -rf scons-out
  # Don't clobber pnacl_translator; these bots currently don't build
  # it, but they use the DEPSed-in version.
  rm -rf toolchain/linux_x86/pnacl_newlib* \
      toolchain/mac_x86/pnacl_newlib* \
      toolchain/win_x86/pnacl_newlib*
}

handle-error() {
  echo "@@@STEP_FAILURE@@@"
  EXIT_STATUS=1
}

ignore-error() {
  echo "@==  IGNORING AN ERROR  ==@"
}

#### Support for running arm sbtc tests on this bot, since we have
# less coverage on the main waterfall now:
# http://code.google.com/p/nativeclient/issues/detail?id=2581
readonly SCONS_COMMON="./scons --verbose bitcode=1 -j${PNACL_CONCURRENCY}"
readonly SCONS_COMMON_SLOW="./scons --verbose bitcode=1 -j2"

build-run-prerequisites() {
  local platform=$1
  ${SCONS_COMMON} platform=${platform} sel_ldr irt_core elf_loader
}


scons-tests-translator() {
  local platform=$1
  local flags="--mode=opt-host,nacl use_sandboxed_translator=1 \
               platform=${platform} -k skip_trusted_tests=1"
  local targets="small_tests medium_tests large_tests"

  # ROUND 1: regular builds
  # generate pexes with full parallelism
  echo "@@@BUILD_STEP scons-sb-trans-pexe [${platform}] [${targets}]@@@"
  ${SCONS_COMMON} ${flags} ${targets} \
      translate_in_build_step=0 do_not_run_tests=1 || handle-error

  # translate pexes
  echo "@@@BUILD_STEP scons-sb-trans-trans [${platform}] [${targets}]@@@"
  if [[ ${platform} = arm ]] ; then
      # For ARM we use less parallelism to avoid mysterious QEMU crashes.
      # We also force a timeout for translation only.
      export QEMU_PREFIX_HOOK="timeout 120"
      # Run sb translation twice in case we failed to translate some of the
      # pexes.  If there was an error in the first run this shouldn't
      # trigger a buildbot error.  Only the second run can make the bot red.
      ${SCONS_COMMON_SLOW} ${flags} ${targets} \
          do_not_run_tests=1 || ignore-error
      ${SCONS_COMMON_SLOW} ${flags} ${targets} \
          do_not_run_tests=1 || handle-error
      # Do not use the prefix hook for running actual tests as
      # it will break some of them due to exit code sign inversion.
      unset QEMU_PREFIX_HOOK
  else
      ${SCONS_COMMON} ${flags} ${targets} \
          do_not_run_tests=1 || handle-error
  fi
  # finally run the tests
  echo "@@@BUILD_STEP scons-sb-trans-run [${platform}] [${targets}]@@@"
  ${SCONS_COMMON_SLOW} ${flags} ${targets} || handle-error

  # ROUND 2: builds with "fast translation"
  flags="${flags} translate_fast=1"
  echo "@@@BUILD_STEP scons-sb-trans-pexe [fast] [${platform}] [${targets}]@@@"
  ${SCONS_COMMON} ${flags} ${targets} \
      translate_in_build_step=0 do_not_run_tests=1 || handle-error

  echo "@@@BUILD_STEP scons-sb-trans-trans [fast] [${platform}] [${targets}]@@@"
  if [[ ${platform} = arm ]] ; then
      # For ARM we use less parallelism to avoid mysterious QEMU crashes.
      # We also force a timeout for translation only.
      export QEMU_PREFIX_HOOK="timeout 120"
      # Run sb translation twice in case we failed to translate some of the
      # pexes.  If there was an error in the first run this shouldn't
      # trigger a buildbot error.  Only the second run can make the bot red.
      ${SCONS_COMMON_SLOW} ${flags} ${targets} \
          do_not_run_tests=1 || ignore-error
      ${SCONS_COMMON_SLOW} ${flags} ${targets} \
          do_not_run_tests=1 || handle-error
      # Do not use the prefix hook for running actual tests as
      # it will break some of them due to exit code sign inversion.
      unset QEMU_PREFIX_HOOK
  else
      ${SCONS_COMMON} ${flags} ${targets} \
          do_not_run_tests=1 || handle-error
  fi
  echo "@@@BUILD_STEP scons-sb-trans-run [fast] [${platform}] [${targets}]@@@"
  ${SCONS_COMMON_SLOW} ${flags} ${targets} || handle-error
}

scons-tests-x86-64-zero-based-sandbox() {
  echo "@@@BUILD_STEP hello_world (x86-64 zero-based sandbox)@@@"
  local flags="--mode=opt-host,nacl platform=x86-64 \
               x86_64_zero_based_sandbox=1"
  ${SCONS_COMMON} ${flags} "run_hello_world_test"
}


tc-test-bot() {
  local archset="$1"
  clobber

  # Update Clang
  python ${CLANG_UPDATE}

  # Only build MIPS stuff on mips bots
  if [[ ${archset} == "mips" ]]; then
    export PNACL_BUILD_MIPS=true
    # Don't run any of the tests yet
    echo "MIPS bot: Only running build, and not tests"
    archset=
  fi

  # Build the un-sandboxed toolchain. The build script outputs its own buildbot
  # annotations.
  ${TOOLCHAIN_BUILD} --verbose --sync --clobber --testsuite-sync \
                     --packages-file ${TEMP_PACKAGES}

  # Extract the built packages using the packages script.
  python ${PACKAGES_SCRIPT} extract --skip-missing --packages ${TEMP_PACKAGES}

  # Linking the tests require additional sdk libraries like libnacl.
  # Do this once and for all early instead of attempting to do it within
  # each test step and having some late test steps rely on early test
  # steps building the prerequisites -- sometimes the early test steps
  # get skipped.
  echo "@@@BUILD_STEP install sdk libraries @@@"
  ${PNACL_BUILD} sdk
  for arch in ${archset}; do
    # Similarly, build the run prerequisites (sel_ldr and the irt) early.
    echo "@@@BUILD_STEP build run prerequisites [${arch}]@@@"
    build-run-prerequisites ${arch}
  done


  # Run the torture tests and compiler_rt tests.
  for arch in ${archset}; do
    echo "@@@BUILD_STEP pnacl bitcode compiler_rt tests@@@"
    export PNACL_RUN_ARCH=${arch}
    make -C toolchain_build/src/compiler-rt \
      -f lib/builtins/Makefile-pnacl-bitcode \
      TCROOT=${INSTALL_ABSPATH} nacltest-pnacl || handle-error

    # The CC arg is just a dummy to keep the make scripts from complaining
    # if clang is not found in PATH
    echo "@@@BUILD_STEP clang compiler_rt tests $arch @@@"
    make -C toolchain_build/src/compiler-rt TCROOT=${INSTALL_ABSPATH} \
      CC=gcc nacltest-${arch} || handle-error

    echo "@@@BUILD_STEP torture_tests_clang $arch @@@"
    ${TORTURE_TEST} clang ${arch} --verbose \
      --concurrency=${PNACL_CONCURRENCY} || handle-error

    if [[ "${arch}" == "x86-32" ]]; then
      # Torture tests on x86-32 are covered by tc-tests-all in
      # buildbot_pnacl.sh.
      continue
    fi
    echo "@@@BUILD_STEP torture_tests_pnacl $arch @@@"
    ${TORTURE_TEST} pnacl ${arch} --verbose \
      --concurrency=${PNACL_CONCURRENCY} || handle-error
  done


  local optset
  optset[1]="--opt O3f --opt O2b"
  for arch in ${archset}; do
    # Run all appropriate frontend/backend optimization combinations.  For now,
    # this means running 2 combinations for x86 (plus one more for Subzero on
    # x86-32) since each takes about 20 minutes on the bots, and making a single
    # run elsewhere since e.g. arm takes about 75 minutes.  In a perfect world,
    # all 4 combinations would be run, plus more for Subzero.
    if [[ ${archset} =~ x86 ]]; then
      optset[2]="--opt O3f --opt O0b"
      if [[ ${archset} == arm || ${archset} == x86-32 || \
            ${archset} == x86-64 ]]; then
        # Run a Subzero -O2 test set on x86-32.
        optset[3]="--opt O3f --opt O2b_sz"
      fi
    fi
    for opt in "${optset[@]}"; do
      echo "@@@BUILD_STEP llvm-test-suite ${arch} ${opt} @@@"
      python ${LLVM_TEST} --testsuite-clean
      LD_LIBRARY_PATH=${LIBRARY_ABSPATH} python ${LLVM_TEST} \
        --testsuite-configure --testsuite-run --testsuite-report \
        --arch ${arch} ${opt} -v -c || handle-error
    done

    echo "@@@BUILD_STEP libcxx-test ${arch} @@@"
    LD_LIBRARY_PATH=${LIBRARY_ABSPATH} python ${LLVM_TEST} \
      --libcxx-test --arch ${arch} -c || handle-error

    # Note: we do not build the sandboxed translator on this bot
    # because this would add another 20min to the build time.
    # The upshot of this is that we are using the sandboxed
    # toolchain which is currently deps'ed in.
    # There is a small upside here: we will notice that bitcode has
    # changed in a way that is incompatible with older translators.
    # TODO(pnacl-team): rethink this.
    # Note: the tests which use sandboxed translation are at the end,
    # because they can sometimes hang on arm, causing buildbot to kill the
    # script without running any more tests.
    scons-tests-translator ${arch}

    if [[ ${arch} = x86-64 ]] ; then
      scons-tests-x86-64-zero-based-sandbox
    fi

  done
  exit $EXIT_STATUS
}


if [ $# = 0 ]; then
  # NOTE: this is used for manual testing only
  tc-test-bot "x86-64 x86-32 arm"
else
  "$@"
fi