1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
|
#!/bin/bash
# Test the implementations of specific filters
# Also test nc_inq_filter_avail
# WARNING: This file is also used to build nczarr_test/run_specific_filters.sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
if test "x$TESTNCZARR" = x1 ; then
. "$srcdir/test_nczarr.sh"
s3isolate "testdir_specific_filters"
THISDIR=`pwd`
cd $ISOPATH
fi
BLOSCARGS="32001,0,0,0,256,5,1,1"
BLOSCARGSALT="32001,0,0,4,256,5,1,1"
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
# Locate the plugin path and the library names; argument order is critical
# Find bzip2 and capture
# Assume all test filters are in same plugin dir
findplugin h5bzip2
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| sed -e '/_Format/d' \
| sed -e '/global attributes:/d' \
| cat > $2
}
# Function to extract _Filter attribute from a file
# These attributes might be platform dependent
getfilterattr() {
V="$1"
sed -e '/${V}.*:_Filter/p' -ed <$2 >$3
}
# Function to extract _Codecs attribute from a file
# These attributes might be platform dependent
getcodecsattr() {
V="$1"
sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3
}
trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
setfilter() {
FF="$1"
FSRC="$2"
FDST="$3"
FIH5="$4"
FICX="$5"
FFH5="$6"
FFCX="$7"
if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi
if test "x$FFCX" = x ; then FFCX="$FICX" ; fi
rm -f $FDST
cat ${srcdir}/$FSRC \
| sed -e "s/ref_any/${FF}/" \
| sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \
| sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \
| sed -e 's/"/\\"/g' -e 's/@/"/g' \
| cat > $FDST
}
# Execute the specified tests
runfilter() {
zext=$1
zfilt="$2"
zparams="$3"
zcodec="$4"
echo "*** Testing processing of filter $zfilt for map $zext"
if test "x$TESTNCZARR" = x1 ; then
fileargs "tmp_filt_${zfilt}"
deletemap $zext $file
else
file="tmp_filt_${zfilt}.nc"
rm -f $file
fi
setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec"
if test "x$TESTNCZARR" = x1 ; then
${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp"
else
${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl"
${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp"
fi
sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump"
}
testfletcher32() {
zext=$1
runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]'
# need to do fixup
sed -e '/_Fletcher32 = "true"/d' -e '/_Filter = 3'/d -e '/_Codecs = \"[{\"id\": \"fletcher32\"}]\"/d' \
< tmp_filt_fletcher32.cdl > tmp_filt_fletcher32x.dump
diff -b -w "tmp_filt_fletcher32.cdl" "tmp_filt_fletcher32x.dump"
}
testshuffle() {
zext=$1
runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]'
# need to replace _Filter
sed -e '/_Shuffle = "true"/d' -e '/_Filter = 2'/d -e '/_Codecs = \"[{\"id\": \"shuffle\"}]\"/d' \
< tmp_filt_shuffle.cdl > tmp_filt_shufflex.dump
diff -b -w "tmp_filt_shuffle.cdl" "tmp_filt_shufflex.dump"
}
testdeflate() {
zext=$1
if ! avail deflate; then return 0; fi
runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]'
# need to replace _DeflateLevel
# sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_filt_deflate.dump > tmp_filt_deflatex.dump
diff -b -w "tmp_filt_deflate.cdl" "tmp_filt_deflate.dump"
}
testbzip2() {
zext=$1
if ! avail bzip2; then return 0; fi
runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]'
diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump"
}
testszip() {
zext=$1
if ! avail szip; then return 0; fi
# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32
runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]'
diff -b -w "tmp_filt_szip.cdl" "tmp_filt_szip.dump"
}
testblosc() {
zext=$1
if ! avail blosc; then return 0; fi
runfilter $zext blosc $BLOSCARGS "$BLOSCCODEC"
# Need to ignore the first three parameters by setting them to 0
rm -f tmp.dump
sed -e "s|${BLOSCARGSALT}|${BLOSCARGS}|" < tmp_filt_blosc.dump > tmp.dump
mv -f tmp.dump tmp_filt_blosc.dump
diff -b -w "tmp_filt_blosc.cdl" "tmp_filt_blosc.dump"
}
testzstd() {
zext=$1
if ! avail zstd; then return 0; fi
runfilter $zext zstd '32015,1' '[{\"id\": \"zstd\",\"level\": \"1\"}]'
diff -b -w "tmp_filt_zstd.cdl" "tmp_filt_zstd.dump"
}
testset() {
# Which test cases to exercise
if test "x$TESTNCZARR" = x1 ; then
testfletcher32 $1
testshuffle $1
fi
testdeflate $1
testszip $1
testbzip2 $1
testblosc $1
testzstd $1
}
if test "x$TESTNCZARR" = x1 ; then
testset file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi
if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup
else
testset nc
fi
|
