/*
 * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2024, Rivos Inc. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 *
 * This code is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * version 2 for more details (a copy is included in the LICENSE file that
 * accompanied this code).
 *
 * You should have received a copy of the GNU General Public License version
 * 2 along with this work; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 * or visit www.oracle.com if you need additional information or have any
 * questions.
 */

#if (defined(RISCV) || defined(RISCV64)) && !defined(ZERO)

#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.hpp"
#include "memory/resourceArea.hpp"
#include "metaprogramming/enableIf.hpp"
#include "runtime/orderAccess.hpp"
#include "threadHelper.inline.hpp"
#include "unittest.hpp"

#include <limits>

typedef int64_t (*zicond_func)(int64_t cmp1, int64_t cmp2, int64_t dst, int64_t src);
typedef void (MacroAssembler::*cmov_func)(Register cmp1, Register cmp2, Register dst, Register src);

class CmovTester {
 public:
  static void test(cmov_func func, int64_t a0, int64_t a1, int64_t a2, int64_t a3, int64_t result) {
    BufferBlob* bb = BufferBlob::create("riscvTest", 128);
    CodeBuffer code(bb);
    MacroAssembler _masm(&code);
    address entry = _masm.pc();
    {
      ((&_masm)->*func)(c_rarg0, c_rarg1, c_rarg2, c_rarg3);
      _masm.mv(c_rarg0, c_rarg2);
      _masm.ret();
    }
    _masm.flush(); // icache invalidate
    int64_t ret = ((zicond_func)entry)(a0, a1, a2, a3);
    ASSERT_EQ(ret, result);
    BufferBlob::free(bb);
  }
};

static void run_cmov_tests() {
  // If 42(a0) eq 42(a1): assign dest(a2/66) the src(a3/77), expect result: 77
  CmovTester::test(&MacroAssembler::cmov_eq, 42, 42, 66, 77, 77);
  // If 41(a0) eq 42(a1): assign dest(a2/66) the src(a3/77), expect result: 66
  CmovTester::test(&MacroAssembler::cmov_eq, 41, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_ne, 41, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_ne, 42, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_le, 41, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_le, 42, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_le, 42, -1, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_leu, 41, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_leu, 42, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_leu, -1, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_ge, 43, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_ge, 42, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_ge, -1, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_geu, 43, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_geu, 42, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_geu, 42, -1, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_lt, 41, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_lt, 42, 42, 66, 77, 66);
  CmovTester::test(&MacroAssembler::cmov_lt, 42, -1, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_ltu, 41, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_ltu, 42, 42, 66, 77, 66);
  CmovTester::test(&MacroAssembler::cmov_ltu, -1, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_gt, 43, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_gt, 42, 42, 66, 77, 66);
  CmovTester::test(&MacroAssembler::cmov_gt, -1, 42, 66, 77, 66);

  CmovTester::test(&MacroAssembler::cmov_gtu, 43, 42, 66, 77, 77);
  CmovTester::test(&MacroAssembler::cmov_gtu, 42, 42, 66, 77, 66);
  CmovTester::test(&MacroAssembler::cmov_gtu, 42, -1, 66, 77, 66);
}

TEST_VM(RiscV, cmov) {
  run_cmov_tests();
  if (UseZicond) {
    UseZicond = false;
    run_cmov_tests();
    UseZicond = true;
  }
}

template <Assembler::operand_size ASMSIZE>
bool using_narrow() {
  if (ASMSIZE == Assembler::int8 || ASMSIZE == Assembler::int16) {
    return !(UseZacas && UseZabha);
  }
  return false;
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
class CmpxchgTester {
  // The functions expect arguments to be type represented, not C-ABI argument representation.
  // Hence an unsigned should be zero-extended, and the same goes for the return value.
  typedef int64_t (*cmpxchg_func)(intptr_t addr, int64_t expected, int64_t new_value, int64_t result);

  typedef int64_t (*cmpxchg_narrow_func)(intptr_t addr, int64_t expected, int64_t new_value, int64_t result,
                                          int64_t scratch0, int64_t scratch1, int64_t scratch2);

  BufferBlob*  _bb;
  cmpxchg_func _func;
  cmpxchg_narrow_func _narrow;

 public:
  CmpxchgTester(int variant, bool boolean_result) {
    _bb = BufferBlob::create("riscvTest", 128);
    CodeBuffer code(_bb);
    MacroAssembler _masm(&code);
    address entry = _masm.pc();
    if (using_narrow<ASMSIZE>()) {
        address entry = _masm.pc();
       _masm.cmpxchg_narrow_value(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/c_rarg2,
                        ASMSIZE, Assembler::relaxed, Assembler::relaxed,
                        /*result*/ c_rarg3, boolean_result, c_rarg4, c_rarg5, c_rarg6); /* Uses also t0-t1, caller saved */
      _masm.mv(c_rarg0, c_rarg3);
      _masm.ret();
      _narrow = ((cmpxchg_narrow_func)entry);
    } else {
      switch(variant) {
        default:
          _masm.cmpxchg(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/c_rarg2,
                        ASMSIZE, Assembler::aq, Assembler::rl,
                        /*result*/ c_rarg3, boolean_result);
          _masm.mv(c_rarg0, c_rarg3);
          break;
        case 1:
          // expected == result
          _masm.cmpxchg(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/c_rarg2,
                        ASMSIZE, Assembler::aq, Assembler::rl,
                        /*result*/ c_rarg1, boolean_result);
          _masm.mv(c_rarg0, c_rarg1);
          break;
        case 2:
          // new_value == result
          _masm.cmpxchg(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/c_rarg2,
                        ASMSIZE, Assembler::aq, Assembler::rl,
                        /*result*/ c_rarg2, boolean_result);
          _masm.mv(c_rarg0, c_rarg2);
          break;
        case 3:
          // expected == new_value
          _masm.cmpxchg(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/ c_rarg1,
                        ASMSIZE, Assembler::aq, Assembler::rl,
                        /*result*/ c_rarg2, boolean_result);
          _masm.mv(c_rarg0, c_rarg2);
          break;

      }
      _masm.ret();
      _func = ((cmpxchg_func)entry);
    }
    _masm.flush(); // icache invalidate
  }

  ~CmpxchgTester() {
    BufferBlob::free(_bb);
  }

  TESTSIZE cmpxchg(intptr_t addr, TESTSIZE expected, TESTSIZE new_value) {
    if (using_narrow<ASMSIZE>()) {
      return _narrow(addr, expected, new_value, /* dummy result */ 67, -1, -1, -1);
    } else {
      return _func(addr, expected, new_value, /* dummy result */ 67);
    }
  }
};

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void plain_cmpxchg_test(int variant, TESTSIZE dv, TESTSIZE ex, TESTSIZE nv, TESTSIZE eret, TESTSIZE edata, bool bv) {
  CmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg(variant, bv);
  TESTSIZE data = dv;
  TESTSIZE ret = cmpxchg.cmpxchg((intptr_t)&data, ex, nv);
  ASSERT_EQ(ret,  eret);
  ASSERT_EQ(data, edata);
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_plain_cmpxchg_tests() {
  TESTSIZE max = std::numeric_limits<TESTSIZE>::max();
  TESTSIZE min = std::numeric_limits<TESTSIZE>::min();
  TESTSIZE val[] = {37, min, max};
  for (int i = 0; i < 3; i++) {
    // Normal
    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     0 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                          val[i] /* return */  ,     42 /* end value*/, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     0 /* variant */ , val[i] /* start value */,
                                              36 /* expected */,     42 /* new value */,
                                          val[i] /* return */  , val[i] /* end value */, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     0 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                               1 /* return */  ,     42 /* end value*/, true /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     0 /* variant */ , val[i] /* start value */,
                                              36 /* expected */,     42 /* new value */,
                                               0 /* return */  , val[i] /* end value */, true /* boolean ret*/);

    // result == expected register
    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     1 /* variant */ ,  val[i] /* start value */,
                                          val[i] /* expected */,      42 /* new value */,
                                          val[i] /* return */  ,      42 /* end value*/, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     1 /* variant */ ,  val[i] /* start value */,
                                              36 /* expected */,      42 /* new value */,
                                          val[i] /* return */  ,  val[i] /* end value */, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     1 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                               1 /* return */  ,     42 /* end value*/, true /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     1 /* variant */ , val[i] /* start value */,
                                              36 /* expected */,     42 /* new value */,
                                               0 /* return */  , val[i] /* end value */, true /* boolean ret*/);

    // new_value == result register
    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     2 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                          val[i] /* return */  ,     42 /* end value*/, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     2 /* variant */ , val[i] /* start value */,
                                              36 /* expected */,     42 /* new value */,
                                          val[i] /* return */  , val[i] /* end value */, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     2 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                               1 /* return */  ,     42 /* end value*/, true /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(    2 /* variant */ , val[i] /* start value */,
                                             36 /* expected */,     42 /* new value */,
                                              0 /* return */  , val[i] /* end value */, true /* boolean ret*/);

    // expected == new_value register
    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     3 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,    42 /* new value */,
                                          val[i] /* return */  , val[i] /* end value */, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     3 /* variant */ , val[i] /* start value */,
                                              36 /* expected */,     42 /* new value */,
                                          val[i] /* return */  , val[i] /* end value */, false /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(     3 /* variant */ , val[i] /* start value */,
                                          val[i] /* expected */,     42 /* new value */,
                                               1 /* return */  , val[i] /* end value */, true /* boolean ret*/);

    plain_cmpxchg_test<TESTSIZE, ASMSIZE>(    3 /* variant */ , val[i] /* start value */,
                                             36 /* expected */,     42 /* new value */,
                                              0 /* return */  , val[i] /* end value */, true /* boolean ret*/);
  }
}

TEST_VM(RiscV, cmpxchg_int64_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_plain_cmpxchg_tests<int64_t, Assembler::int64>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_int64_maybe_zacas) {
  if (UseZacas) {
    run_plain_cmpxchg_tests<int64_t, Assembler::int64>();
  }
}

TEST_VM(RiscV, cmpxchg_int32_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_plain_cmpxchg_tests<int32_t, Assembler::int32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_int32_maybe_zacas) {
  if (UseZacas) {
    run_plain_cmpxchg_tests<int32_t, Assembler::int32>();
  }
}

TEST_VM(RiscV, cmpxchg_uint32_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_plain_cmpxchg_tests<uint32_t, Assembler::uint32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_uint32_maybe_zacas) {
  if (UseZacas) {
    run_plain_cmpxchg_tests<uint32_t, Assembler::uint32>();
  }
}

TEST_VM(RiscV, cmpxchg_int16_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_plain_cmpxchg_tests<int16_t, Assembler::int16>();
  }
}

TEST_VM(RiscV, cmpxchg_int8_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_plain_cmpxchg_tests<int8_t, Assembler::int8>();
  }
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_narrow_cmpxchg_tests() {
  CmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg(0, false);
  CmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg_bool(0, true);
  // Assume natural aligned
  TESTSIZE data[8];
  TESTSIZE ret;
  TESTSIZE max = std::numeric_limits<TESTSIZE>::max();
  TESTSIZE min = std::numeric_limits<TESTSIZE>::min();
  TESTSIZE val[] = {121, min, max};
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 7; j++) {
      memset(data, -1, sizeof(data));
      data[i] = val[i];
      ret = cmpxchg.cmpxchg((intptr_t)&data[i], val[i], 42);
      ASSERT_EQ(ret, val[i]);
      ASSERT_EQ(data[i], 42);

      data[i] = val[i];
      ret = cmpxchg.cmpxchg((intptr_t)&data[i], 120, 42);
      ASSERT_EQ(ret, val[i]);
      ASSERT_EQ(data[i], val[i]);

      data[i] = val[i];
      ret = cmpxchg_bool.cmpxchg((intptr_t)&data[i], val[i], 42);
      ASSERT_EQ(ret, 1);
      ASSERT_EQ(data[i], 42);

      data[i] = val[i];
      ret = cmpxchg_bool.cmpxchg((intptr_t)&data[i], 120, 42);
      ASSERT_EQ(ret, 0);
      ASSERT_EQ(data[i], val[i]);
    }
  }
}

TEST_VM(RiscV, cmpxchg_narrow_int16_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_narrow_cmpxchg_tests<int16_t, Assembler::int16>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_narrow_int16_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_narrow_cmpxchg_tests<int16_t, Assembler::int16>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, cmpxchg_narrow_int8_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_narrow_cmpxchg_tests<int8_t, Assembler::int8>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_narrow_int8_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_narrow_cmpxchg_tests<int8_t, Assembler::int8>();
    UseZabha = zabha;
  }
}

template <typename TESTSIZE>
TESTSIZE next_count(TESTSIZE now, TESTSIZE add) {
  if ((std::numeric_limits<TESTSIZE>::max() - add) >= now) {
    return now + add;
  }
  TESTSIZE diff = std::numeric_limits<TESTSIZE>::max() - now;
  add -= diff + 1; // add one to the diff for the wrap around.
  return std::numeric_limits<TESTSIZE>::min() + add;
}

constexpr int64_t PAR_IT_END       = 10000;
constexpr int64_t NUMBER_THREADS   = 4;
constexpr int64_t TOTAL_ITERATIONS = NUMBER_THREADS * PAR_IT_END;

template <typename TESTSIZE, ENABLE_IF(std::numeric_limits<TESTSIZE>::max() <= (std::numeric_limits<TESTSIZE>::min() + TOTAL_ITERATIONS))>
constexpr TESTSIZE result_count() {
  int64_t range = std::numeric_limits<TESTSIZE>::max() - std::numeric_limits<TESTSIZE>::min() + 1;
  int64_t rest = TOTAL_ITERATIONS % range;
  return std::numeric_limits<TESTSIZE>::min() + rest;
}

template <typename TESTSIZE, ENABLE_IF(std::numeric_limits<TESTSIZE>::max() > (std::numeric_limits<TESTSIZE>::min() + TOTAL_ITERATIONS))>
constexpr TESTSIZE result_count() {
  return std::numeric_limits<TESTSIZE>::min() + TOTAL_ITERATIONS;
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_concurrent_cmpxchg_tests() {
  volatile TESTSIZE data = std::numeric_limits<TESTSIZE>::min();
  int num_threads = NUMBER_THREADS;
  CmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg(0, false); // variant 0, not bool ret
  auto incThread = [&](Thread* _current, int _id) {   // _id starts from 0..(CTHREAD-1)
    TESTSIZE my_oldvalue = std::numeric_limits<TESTSIZE>::min() + _id;
    for (int64_t i = 0; i < PAR_IT_END ; i++) {
      TESTSIZE newvalue = next_count<TESTSIZE>(my_oldvalue,  1);
      TESTSIZE ret;
      do {
        ret = cmpxchg.cmpxchg((intptr_t)&data, my_oldvalue, newvalue);
      } while (ret != my_oldvalue);
      my_oldvalue = next_count<TESTSIZE>(my_oldvalue, num_threads);
    }
  };
  TestThreadGroup<decltype(incThread)> ttg(incThread, num_threads);
  ttg.doit();
  ttg.join();
  ASSERT_EQ(data, result_count<TESTSIZE>());
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_concurrent_alt_cmpxchg_tests() {
  volatile TESTSIZE data = std::numeric_limits<TESTSIZE>::min();
  int num_threads = NUMBER_THREADS;
  CmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg(0, false); // variant 0, not bool ret
  auto incThread = [&](Thread* _current, int _id) {   // _id starts from 0..(CTHREAD-1)
    for (int i = 0; i < PAR_IT_END; i++) {
      TESTSIZE oldvalue;
      TESTSIZE ret = 0;
      do {
        oldvalue = ret;
        TESTSIZE newvalue = next_count<TESTSIZE>(oldvalue, 1);
        ret = cmpxchg.cmpxchg((intptr_t)&data, oldvalue, newvalue);
      } while (ret != oldvalue);
    }
  };
  TestThreadGroup<decltype(incThread)> ttg(incThread, num_threads);
  ttg.doit();
  ttg.join();
  ASSERT_EQ(data, result_count<TESTSIZE>());
}

TEST_VM(RiscV, cmpxchg_int64_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_cmpxchg_tests<int64_t, Assembler::int64>();
  run_concurrent_alt_cmpxchg_tests<int64_t, Assembler::int64>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_int64_concurrent_maybe_zacas) {
  if (UseZacas) {
    run_concurrent_cmpxchg_tests<int64_t, Assembler::int64>();
    run_concurrent_alt_cmpxchg_tests<int64_t, Assembler::int64>();
  }
}

TEST_VM(RiscV, cmpxchg_int32_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_cmpxchg_tests<int32_t, Assembler::int32>();
  run_concurrent_alt_cmpxchg_tests<int32_t, Assembler::int32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_int32_concurrent_maybe_zacas) {
  if (UseZacas) {
    run_concurrent_cmpxchg_tests<int32_t, Assembler::int32>();
    run_concurrent_alt_cmpxchg_tests<int32_t, Assembler::int32>();
  }
}

TEST_VM(RiscV, cmpxchg_uint32_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_cmpxchg_tests<uint32_t, Assembler::uint32>();
  run_concurrent_alt_cmpxchg_tests<uint32_t, Assembler::uint32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_uint32_concurrent_maybe_zacas) {
  if (UseZacas) {
    run_concurrent_cmpxchg_tests<uint32_t, Assembler::uint32>();
    run_concurrent_alt_cmpxchg_tests<uint32_t, Assembler::uint32>();
  }
}

TEST_VM(RiscV, cmpxchg_narrow_int16_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_cmpxchg_tests<int16_t, Assembler::int16>();
  run_concurrent_alt_cmpxchg_tests<int16_t, Assembler::int16>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_narrow_int16_concurrent_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_concurrent_cmpxchg_tests<int16_t, Assembler::int16>();
    run_concurrent_alt_cmpxchg_tests<int16_t, Assembler::int16>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, cmpxchg_narrow_int8_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_cmpxchg_tests<int8_t, Assembler::int8>();
  run_concurrent_alt_cmpxchg_tests<int8_t, Assembler::int8>();
  UseZacas = zacas;
}

TEST_VM(RiscV, cmpxchg_narrow_int8_concurrent_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_concurrent_cmpxchg_tests<int8_t, Assembler::int8>();
    run_concurrent_alt_cmpxchg_tests<int8_t, Assembler::int8>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, cmpxchg_int16_concurrent_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_concurrent_cmpxchg_tests<int16_t, Assembler::int16>();
    run_concurrent_alt_cmpxchg_tests<int16_t, Assembler::int16>();
  }
}

TEST_VM(RiscV, cmpxchg_int8_concurrent_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_concurrent_cmpxchg_tests<int8_t, Assembler::int8>();
    run_concurrent_alt_cmpxchg_tests<int8_t, Assembler::int8>();
  }
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
class WeakCmpxchgTester {
  // The functions expect arguments to be type represented, not C-ABI argument representation.
  // Hence an unsigned should be zero-extended, and the same goes for the return value.
  typedef int64_t (*weak_cmpxchg_narrow_func)(intptr_t addr, int64_t expected, int64_t new_value, int64_t result,
                                   int64_t scratch0, int64_t scratch1, int64_t scratch2);

  typedef int64_t (*weak_cmpxchg_func)(intptr_t addr, int64_t expected, int64_t new_value, int64_t result);

  BufferBlob*  _bb;
  weak_cmpxchg_narrow_func _narrow_weak;
  weak_cmpxchg_func _weak;

 public:
  WeakCmpxchgTester() : _bb(nullptr), _narrow_weak(nullptr), _weak(nullptr) {
    _bb = BufferBlob::create("riscvTest", 128);
    CodeBuffer code(_bb);
    MacroAssembler _masm(&code);
    if (using_narrow<ASMSIZE>()) {
        address entry = _masm.pc();
       _masm.weak_cmpxchg_narrow_value(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/ c_rarg2,
                                      ASMSIZE, Assembler::relaxed, Assembler::relaxed,
                                      /*result*/ c_rarg3, c_rarg4, c_rarg5, c_rarg6); /* Uses also t0-t1, caller saved */
      _masm.mv(c_rarg0, c_rarg3);
      _masm.ret();
      _narrow_weak = ((weak_cmpxchg_narrow_func)entry);
    } else {
        address entry = _masm.pc();
       _masm.weak_cmpxchg(/*addr*/ c_rarg0, /*expected*/ c_rarg1, /*new_value*/ c_rarg2,
                          ASMSIZE, Assembler::relaxed, Assembler::relaxed, /*result*/ c_rarg3);
      _masm.mv(c_rarg0, c_rarg3);
      _masm.ret();
      _weak = ((weak_cmpxchg_func)entry);
    }
    _masm.flush(); // icache invalidate
  }

  TESTSIZE weak_cmpxchg(intptr_t addr, TESTSIZE expected, TESTSIZE new_value) {
    if (using_narrow<ASMSIZE>()) {
      return _narrow_weak(addr, expected, new_value, /* dummy result */ 67, -1, -1, -1);
    } else {
      return _weak(addr, expected, new_value, /* dummy result */ 67);
    }
  }

  ~WeakCmpxchgTester() {
    BufferBlob::free(_bb);
  }
};

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
void run_weak_cmpxchg_tests() {
  TESTSIZE max = std::numeric_limits<TESTSIZE>::max();
  TESTSIZE min = std::numeric_limits<TESTSIZE>::min();
  TESTSIZE val[] = {121, min, max};
  for (int i = 0; i < 3; i++) {
    WeakCmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg;
    TESTSIZE data = val[i];
    TESTSIZE ret = cmpxchg.weak_cmpxchg((intptr_t)&data, val[i], 42);
    ASSERT_EQ(ret, (TESTSIZE)1);
    ASSERT_EQ(data, (TESTSIZE)42);

    data = val[i];
    ret = cmpxchg.weak_cmpxchg((intptr_t)&data, 120, 42);
    ASSERT_EQ(ret, (TESTSIZE)0);
    ASSERT_EQ(data, (TESTSIZE)val[i]);
  }
}

TEST_VM(RiscV, weak_cmpxchg_int64_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_weak_cmpxchg_tests<int64_t, Assembler::int64>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_int64_maybe_zacas) {
  if (UseZacas) {
    run_weak_cmpxchg_tests<int64_t, Assembler::int64>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_int32_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_weak_cmpxchg_tests<int32_t, Assembler::int32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_int32_maybe_zacas) {
  if (UseZacas) {
    run_weak_cmpxchg_tests<int32_t, Assembler::int32>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_uint32_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_weak_cmpxchg_tests<uint32_t, Assembler::uint32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_uint32_maybe_zacas) {
  if (UseZacas) {
    run_weak_cmpxchg_tests<uint32_t, Assembler::uint32>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int16_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_weak_cmpxchg_tests<int16_t, Assembler::int16>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int8_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_weak_cmpxchg_tests<int8_t, Assembler::int8>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int16_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_weak_cmpxchg_tests<int16_t, Assembler::int16>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int8_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_weak_cmpxchg_tests<int8_t, Assembler::int8>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, weak_cmpxchg_int16_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_weak_cmpxchg_tests<int16_t, Assembler::int16>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_int8_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_weak_cmpxchg_tests<int8_t, Assembler::int8>();
  }
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_concurrent_weak_cmpxchg_tests() {
  volatile TESTSIZE data = std::numeric_limits<TESTSIZE>::min();
  int num_threads = NUMBER_THREADS;
  WeakCmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg; // not bool ret
  auto incThread = [&](Thread* _current, int _id) { // _id starts from 0..(CTHREAD-1)
    TESTSIZE my_oldvalue = std::numeric_limits<TESTSIZE>::min() + _id;
    for (int64_t i = 0; i < PAR_IT_END; i++) {
      TESTSIZE newvalue = next_count<TESTSIZE>(my_oldvalue, 1);
      TESTSIZE ret;
      do {
        ret = cmpxchg.weak_cmpxchg((intptr_t)&data, my_oldvalue, newvalue);
      } while (ret != 1);
      my_oldvalue = next_count<TESTSIZE>(my_oldvalue, num_threads);
    }
  };
  TestThreadGroup<decltype(incThread)> ttg(incThread, num_threads);
  ttg.doit();
  ttg.join();
  ASSERT_EQ(data, result_count<TESTSIZE>());
}

template <typename TESTSIZE, Assembler::operand_size ASMSIZE>
static void run_concurrent_alt_weak_cmpxchg_tests() {
  volatile TESTSIZE data = std::numeric_limits<TESTSIZE>::min();
  int num_threads = NUMBER_THREADS;
  WeakCmpxchgTester<TESTSIZE, ASMSIZE> cmpxchg; // not bool ret
  auto incThread = [&](Thread* _current, int _id) { // _id starts from 0..(CTHREAD-1)
    for (int i = 0; i < PAR_IT_END; i++) {
      TESTSIZE oldvalue;
      TESTSIZE ret = 0;
      do {
        oldvalue = data;
        TESTSIZE newvalue = next_count<TESTSIZE>(oldvalue, 1);
        ret = cmpxchg.weak_cmpxchg((intptr_t)&data, oldvalue, newvalue);
      } while (ret != 1);
    }
  };
  TestThreadGroup<decltype(incThread)> ttg(incThread, num_threads);
  ttg.doit();
  ttg.join();
  ASSERT_EQ(data, result_count<TESTSIZE>());
}

TEST_VM(RiscV, weak_cmpxchg_int64_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_weak_cmpxchg_tests<int64_t, Assembler::int64>();
  run_concurrent_alt_weak_cmpxchg_tests<int64_t, Assembler::int64>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_int64_concurrent_maybe_zacas) {
  if (UseZacas) {
    run_concurrent_weak_cmpxchg_tests<int64_t, Assembler::int64>();
    run_concurrent_alt_weak_cmpxchg_tests<int64_t, Assembler::int64>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_int32_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_weak_cmpxchg_tests<int32_t, Assembler::int32>();
  run_concurrent_alt_weak_cmpxchg_tests<int32_t, Assembler::int32>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_int32_concurrent_maybe_zacas) {
  if (UseZacas) {
    run_concurrent_weak_cmpxchg_tests<int32_t, Assembler::int32>();
    run_concurrent_alt_weak_cmpxchg_tests<int32_t, Assembler::int32>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int16_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_weak_cmpxchg_tests<int16_t, Assembler::int16>();
  run_concurrent_alt_weak_cmpxchg_tests<int16_t, Assembler::int16>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int16_concurrent_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_concurrent_weak_cmpxchg_tests<int16_t, Assembler::int16>();
    run_concurrent_alt_weak_cmpxchg_tests<int16_t, Assembler::int16>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int8_concurrent_lr_sc) {
  bool zacas = UseZacas;
  UseZacas = false;
  run_concurrent_weak_cmpxchg_tests<int8_t, Assembler::int8>();
  run_concurrent_alt_weak_cmpxchg_tests<int8_t, Assembler::int8>();
  UseZacas = zacas;
}

TEST_VM(RiscV, weak_cmpxchg_narrow_int8_concurrent_maybe_zacas) {
  if (UseZacas) {
    bool zabha = UseZabha;
    UseZabha = false;
    run_concurrent_weak_cmpxchg_tests<int8_t, Assembler::int8>();
    run_concurrent_alt_weak_cmpxchg_tests<int8_t, Assembler::int8>();
    UseZabha = zabha;
  }
}

TEST_VM(RiscV, weak_cmpxchg_int16_concurrent_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_concurrent_weak_cmpxchg_tests<int16_t, Assembler::int16>();
    run_concurrent_alt_weak_cmpxchg_tests<int16_t, Assembler::int16>();
  }
}

TEST_VM(RiscV, weak_cmpxchg_int8_concurrent_maybe_zacas_zabha) {
  if (UseZacas && UseZabha) {
    run_concurrent_weak_cmpxchg_tests<int8_t, Assembler::int8>();
    run_concurrent_alt_weak_cmpxchg_tests<int8_t, Assembler::int8>();
  }
}

#endif  // RISCV