1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
# Note: thredds-test is currently down and the URLs need to be replaced
# Test Urls
if test "x$FEATURE_THREDDSTEST" = x1 ; then
URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes"
URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes"
fi
if test "x$FEATURE_S3TESTS" != xno ; then
URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
URL4c="https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3"
# Test alternate URL with no specified region and using s3 protocol
URL4e="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes"
fi
if test "x$FEATURE_S3TESTS" = xyes ; then
# Requires auth
URL3b="s3://${S3TESTBUCKET}/byterangefiles/upload3.nc#bytes"
# Requires auth
URL4d="s3://${S3TESTBUCKET}/byterangefiles/upload4.nc#bytes&aws.profile=unidata"
fi
URL4f="https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes"
if test "x$FEATURE_S3TESTS" = xyes ; then
# Do not use unless we know it has some permanence (note the segment 'testing' in the URL);
URL4x="https://s3.us-west-2.amazonaws.com/coawst-public/testing/HadCRUT.4.6.0.0.median.nc#mode=bytes,&aws.profile=none"
fi
echo ""
testsetup() {
U=$1
# Create and upload test files
rm -f upload4.nc upload3.nc
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear
${NCGEN} -lb -3 ${srcdir}/nc_enddef.cdl
mv nc_enddef.nc upload3.nc
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles/upload3.nc -f upload3.nc upload
if test "x$FEATURE_HDF5" = xyes ; then
${NCGEN} -lb -4 ${srcdir}/nc_enddef.cdl
mv nc_enddef.nc upload4.nc
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles/upload4.nc -f upload4.nc upload
fi
rm -f tst_http_nc3.cdl tst_http_nc4?.cdl
}
testcleanup() {
U=$1
rm -f upload4.nc upload3.nc
${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear
}
testbytes() {
TAG="$1"
EXPECTED="$2"
U="$3"
K=`${NCDUMP} -k "$U" | tr -d '\r\n'`
if test "x$K" != "x$EXPECTED" ; then
echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K"
exit 1
fi
rm -f tst_http_$TAG.cdl
# Now test the reading of at least the metadata
${NCDUMP} -h "$U" >tst_http_$TAG.cdl
# compare
diff -wb tst_http_$TAG.cdl ${srcdir}/ref_tst_http_$TAG.cdl
}
tests3auth() {
TAG="$1"
EXPECTED="$2"
U="$3"
K=`${NCDUMP} -k "$U" | tr -d '\r\n'`
if test "x$K" != "x$EXPECTED" ; then
echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K"
exit 1
fi
rm -f tmp_${TAG}.cdl
# Now test the reading of at least the metadata
${NCDUMP} -n nc_enddef "$U" >tmp_${TAG}.cdl
# compare
diff -wb tmp_$TAG.cdl ${srcdir}/nc_enddef.cdl
}
if test "x$FEATURE_S3TESTS" = xyes ; then
testsetup https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}
fi
echo "*** Testing reading NetCDF-3 file with http"
if test "x$FEATURE_THREDDSTEST" = x1 ; then
echo "***Test remote classic file"
testbytes nc3 classic "$URL3"
fi
if test "x$FEATURE_HDF5" = xyes ; then
echo "***Test remote netcdf-4 files: non-s3"
if test "x$FEATURE_THREDDSTEST" = x1 ; then
testbytes nc4b netCDF-4 "$URL4b"
fi
testbytes nc4f netCDF-4 "$URL4f"
fi
if test "x$URL3B" != x ; then
echo "***Test remote netcdf-3 file: s3 auth"
tests3auth nc3b classic "$URL3b"
fi
if test "x$URL4a" != x ; then
echo "***Test remote netdf-4 file: s3"
testbytes nc4a netCDF-4 "$URL4a"
fi
if test "x$URL4c" != x ; then
echo "***Test remote netcdf-4 file: s3"
testbytes nc4c netCDF-4 "$URL4c"
fi
if test "x$URL4d" != x ; then
echo "***Test remote netcdf-4 file: s3 auth"
tests3auth nc4d netCDF-4 "$URL4d"
fi
if test "x$URL4e" != x ; then
echo "***Test remote netcdf-4 file: s3 noauth"
testbytes nc4e netCDF-4 "$URL4e"
fi
# Cleanup
if test "x$FEATURE_S3TESTS" = xyes ; then
testcleanup https://s3.us-east-1.amazonaws.com/${S3TESTBUCKET}
fi
exit
|
