!
! Copyright (C) 2021, MaX CoE
! Distributed under the MIT License 
! (license terms are at http://opensource.org/licenses/MIT).
!
!--
!
! Testing: dev_memcpy_async, devxlib_map, dev_auxfunc
!
!==================================================================
!==================================================================
! *DO NOT EDIT*: automatically generated from test_memcpy_async.jf90
!==================================================================
!==================================================================
!
#include<devxlib_macros.h>
#include<devxlib_defs.h>
!
program test_memcpy_async
  !
  ! This program tests the routines and interfaces related to
  ! device_memcpy_async and some device_auxfunc.
  !
  use iso_fortran_env, only : int32, int64, real32, real64
  use devxlib_async
  use devxlib_memcpy
  use devxlib_mapping
  use devxlib_auxfunc
  !
#if defined __DXL_CUDAF
  use cudafor
#elif defined __DXL_OPENACC
  use openacc
#endif
  implicit none

  integer(int64) :: cnt, count_max
  real(real64), parameter :: thr=1.0d-6
  integer,      parameter :: nranks={{dimensions}}
  integer      :: ndim(nranks)
  integer      :: vrange(2,nranks)
  integer      :: vlbound(nranks)
  real(real64) :: t0, t1, count_rate
  integer      :: npass,nfail
  integer      :: nfail_malloc
  logical      :: is_alloc, lfail_malloc
  integer(kind=dxl_async_kind) :: async_id, default_async_id


{%- for t in types %}
{%- for p in kinds[t] %}
{%- for d in range(1,dimensions+1) %}
  {{t}}({{p.val}}), allocatable DEV_PINN :: A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {{t}}({{p.val}}), allocatable DEV_PINN :: A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {{t}}({{p.val}}), allocatable DEV_PINN :: A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {{t}}({{p.val}}), allocatable DEV_ATTR :: A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {{t}}({{p.val}}), allocatable DEV_ATTR :: A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {% if t=="complex" or t=="integer" or t=="logical" %}
  real({{p.val}}),  allocatable DEV_PINN :: A_rtmp__{{p.name}}_{{d}}d({% for dd in range(d) %}:{% if not loop.last %}, {%- endif %}{% endfor %})
  {% endif %}
{% endfor %}
{% endfor %}
{% endfor %}
  
  integer :: i,ierr
  integer :: {% for dd in range(1,dimensions+1) %}ndim{{dd}}{% if not loop.last %}, {%- endif %} {% endfor %}
  integer :: {% for dd in range(1,dimensions+1) %}lbound{{dd}}{% if not loop.last %}, {%- endif %} {% endfor %}
  integer :: {% for dd in range(1,dimensions+1) %}range{{dd}}(2){% if not loop.last %}, {%- endif %} {% endfor %}
  integer :: {% for dd in range(1,dimensions+1) %}bound{{dd}}(2){% if not loop.last %}, {%- endif %} {% endfor %}
  logical :: is_pinned(3)
  character(256) :: arg, str

!
!============================
! get dims
!============================
!
  ! defaults
  ndim(:)=100
  vrange(1,:)=1
  vrange(2,:)=ndim
  vlbound(:)=1

  i=0
  do
    call get_command_argument(i, arg)
    if (len_trim(arg) == 0) exit
    !
    select case (trim(arg))
    case("-h","--help")
      write(6,"(a)") "Usage: "
      write(6,"(a)") "   ./test_memcpy_async.x [--dims <vals>] [--range <vals>] [--lbound <vals>]"
      stop
    end select
    !
    i = i+1
    call get_command_argument(i, str)
    if (len_trim(str) == 0) exit
    !
    select case (trim(arg))
    case("-dims","--dims")
      read(str,*,iostat=ierr) ndim(:)
      if (ierr/=0) STOP "reading cmd-line args: dims"
    case("-range","--range")
      read(str,*,iostat=ierr) vrange(:,:)
      if (ierr/=0) STOP "reading cmd-line args: range"
    case("-lbound","--lbound")
      read(str,*,iostat=ierr) vlbound(:)
      if (ierr/=0) STOP "reading cmd-line args: lbound"
    end select
  enddo
  !
  write(6,"(/,a,/)") "Running test with params: "
  write(6,"(3x,a,10i5)") "  ndim: ", ndim(:)
  write(6,"(3x,a,10i5)") "lbound: ", vlbound(:)
  do i = 1, nranks
     write(6,"(3x,a,i2,3x,10i5)") " range: ", i, vrange(:,i)
  enddo
  write(6,"()")
  !
  npass=0
  nfail=0
  nfail_malloc=0

  !
  ! init async_id (streams, queues)
  !
  call devxlib_async_create(default_async_id,default_async=.true.)
  !
  async_id=0
#ifdef __DXL_CUDAF
  call devxlib_async_create(async_id)
#elif defined __DXL_OPENACC
  call devxlib_async_create(async_id,val=int(10,KIND=dxl_async_kind))
#endif

  !
  ! main loop
  !
{% for d in range(1,dimensions+1) %}
  ndim{{d}}=ndim({{d}})
  lbound{{d}}=vlbound({{d}})
  range{{d}}=vrange(:,{{d}})
  bound{{d}}(1)=lbound{{d}}
  bound{{d}}(2)=lbound{{d}}+ndim{{d}}-1
{%- endfor %}

!
!============================
! check memcpy
!============================
!

{%- for t in types %}
{%- for p in kinds[t] %}
{%- for d in range(1,dimensions+1) %}
  !
  !=====================
  !
  write(6,"(/,3x,a)") "checking {{p.name}}_{{t[0]|lower}}{{d}}d ..." 
  call system_clock(cnt, count_rate, count_max)
  t0=cnt/count_rate
  lfail_malloc=.false.
  !
  is_alloc = devxlib_mapped( A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d ) .or. &
             devxlib_mapped( A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d )
  if (is_alloc) then
     nfail_malloc=nfail_malloc+1
     lfail_malloc=.true.
     write(6,"(3x,a)") "mem allocation check-pre FAILED" 
  endif
  !
  ! allocations
#ifdef __DXL_CUDAF
  allocate( A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}), PINNED=is_pinned(1) )
  allocate( A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}), PINNED=is_pinned(2) )
  allocate( A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}), PINNED=is_pinned(3) )
#else
  is_pinned=.false.
  allocate( A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
  allocate( A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
  allocate( A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
#endif
  allocate( A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
  allocate( A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_map(A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d)
  call devxlib_map(A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d)
#endif
  {%- if t=="complex" or t=="integer" or t=="logical" %}
  allocate( A_rtmp__{{p.name}}_{{d}}d({% for dd in range(d)%}bound{{dd+1}}(1):bound{{dd+1}}(2){% if not loop.last %},{%- endif %}{%endfor%}) )
  {%endif%}
  !write(6,"(3x,a,2l)") "host memory is pinned: ", is_pinned(1:3)
  !
  is_alloc = devxlib_mapped( A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d )
  if (.not.is_alloc) then
    nfail_malloc=nfail_malloc+1
    lfail_malloc=.true.
    write(6,"(3x,a)") "mem allocation check-post FAILED"
  endif
  !
  ! init
  {%- if t=="complex" %}
  call random_number( A_rtmp__{{p.name}}_{{d}}d )
  A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d=A_rtmp__{{p.name}}_{{d}}d
  call random_number( A_rtmp__{{p.name}}_{{d}}d )
  A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d=A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d+cmplx(0.0,1.0)*A_rtmp__{{p.name}}_{{d}}d
  {%- elif t=="integer" %}
  call random_number( A_rtmp__{{p.name}}_{{d}}d )
  A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d=int( 1000* A_rtmp__{{p.name}}_{{d}}d )
  {%- elif t=="logical" %}
  call random_number( A_rtmp__{{p.name}}_{{d}}d )
  A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d=( nint( A_rtmp__{{p.name}}_{{d}}d) == 1 ) 
  {%else%}
  call random_number( A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d )
  {%endif%}
  !
  call devxlib_async_synchronize(default_async_id)
  call devxlib_async_synchronize(async_id)
  !
  ! mem copy h2d
  call devxlib_memcpy_h2d(A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d, A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d, async_id)
  !
  ! mem copy d2d
  call devxlib_memcpy_d2d(A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d, A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d, async_id)
  !
  {%- if t=="complex" %}
  ! make cmplx conjg
  call devxlib_conjg_d( A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d, async_id)
  {%endif%}
  !
  ! mem copy d2h
  call devxlib_memcpy_d2h(A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d, A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d,async_id)
  call devxlib_async_synchronize(async_id)
  !
  {%- if t=="complex" %}
  ! retrieve conjg data
  A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d = conjg( A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d )
  {%endif%}
  !
  ! mem copy h2h
  call devxlib_memcpy_h2h(A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d, A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d)
  !
  ! check
  if (lfail_malloc) write(6,"(3x,a)") "Malloc FAILED"
   !
  {%- if t=="logical" %}
  if ( any( A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d .neqv. A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d ) ) then
  {%- else %}
  if ( any(abs(A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d -A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d )> thr) ) then
  {%- endif %}
     !
     write(6,"(3x,a)") "FAILED" 
     nfail=nfail+1
  else
     write(6,"(3x,a)") "passed" 
     npass=npass+1
  endif
  !
  deallocate(A_hst1__{{p.name}}_{{t[0]|lower}}{{d}}d)
  deallocate(A_hst2__{{p.name}}_{{t[0]|lower}}{{d}}d)
  deallocate(A_hst3__{{p.name}}_{{t[0]|lower}}{{d}}d)
  !
#if defined __DXL_OPENACC || defined __DXL_OPENMP_GPU
  call devxlib_unmap(A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d)
  call devxlib_unmap(A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d)
  !
  is_alloc = devxlib_mapped( A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d ) .or. &
             devxlib_mapped( A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d )
  if (is_alloc) then
    nfail_malloc=nfail_malloc+1
    lfail_malloc=.true.
    write(6,"(3x,a)") "mem allocation check-post2 FAILED"
  endif
  !
#endif
  !
  deallocate(A_dev1__{{p.name}}_{{t[0]|lower}}{{d}}d)
  deallocate(A_dev2__{{p.name}}_{{t[0]|lower}}{{d}}d)
  !
  call system_clock(cnt, count_rate, count_max)
  t1=cnt/count_rate
  write(6,"(3x,a,f12.6,' sec')") "Timing: ", t1-t0
  !
{% endfor %}
{% endfor %}
{% endfor %}

  !
  ! summary
  !
  write(6,"(/,a)") "# Test SUMMARY:"
  write(6,"(3x,a,i5)") "# passed: ", npass
  write(6,"(3x,a,i5)") "# failed: ", nfail
  write(6,"(3x,a,i5)") "# failed malloc: ", nfail_malloc
  write(6,"()")

end program test_memcpy_async