1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
// Purpose: CDL to generate large netCDF file for testing Large File Support (LFS)
// Usage:
// Create big.nc from big.cdl on a beefy 64-bit computer
// The file creation command fails on systems with insufficient RAM :)
// Transfer big.nc to a 32-bit system as an LFS testfile
// ncgen -b -o big.nc big.cdl
// ncgen -b -o ~/nco/data/big.nc ~/nco/data/big.cdl
// scp ~/nco/data/big.cdl esmf.ess.uci.edu:nco/data
// scp ~/nco/data/big.cdl dust.ess.uci.edu:nco/data
// Test 64-bit netCDF/NCO capabilities
// 32-bit machines do not work with files exceeding ~2 GB unless LFS enabled
// One billion floats are nco_typ_lng(NC_FLOAT)*10^9 = sizeof(float)*10^9 = 4*10^9 B = 4 GB
// Create file with variable at least 4 GB in size
// Expect failure on "small RAM" (< ~4 GB) machines:
// ncap2 -5 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0f" -s "wvl_1e1[wvl_1e1]=1.0f" -s "wvl_1e2[wvl_1e2]=1.0f" -s "wvl_1e3[wvl_1e3]=1.0f" -s "wvl_1e4[wvl_1e4]=1.0f" -s "wvl_1e5[wvl_1e5]=1.0f" -s "wvl_1e6[wvl_1e6]=1.0f" -s "wvl_1e7[wvl_1e7]=1.0f" -s "wvl_1e8[wvl_1e8]=1.0f" -s "wvl_1e9[wvl_1e9]=1.0f" ~/nco/data/big.nc ~/foo_big.nc
// ncap2 -5 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0d" -s "wvl_1e1[wvl_1e1]=1.0d" -s "wvl_1e2[wvl_1e2]=1.0d" -s "wvl_1e3[wvl_1e3]=1.0d" -s "wvl_1e4[wvl_1e4]=1.0d" -s "wvl_1e5[wvl_1e5]=1.0d" -s "wvl_1e6[wvl_1e6]=1.0d" -s "wvl_1e7[wvl_1e7]=1.0d" -s "wvl_1e8[wvl_1e8]=1.0d" -s "wvl_1e9[wvl_1e9]=1.0d" ~/nco/data/big.nc ~/foo_big.nc
// ncap2 -4 --cnk_dmn=wvl_1e9,100000 -D 3 -O -s "wvl_1e0[wvl_1e0]=1.0d" -s "wvl_1e1[wvl_1e1]=1.0d" -s "wvl_1e2[wvl_1e2]=1.0d" -s "wvl_1e3[wvl_1e3]=1.0d" -s "wvl_1e4[wvl_1e4]=1.0d" -s "wvl_1e5[wvl_1e5]=1.0d" -s "wvl_1e6[wvl_1e6]=1.0d" -s "wvl_1e7[wvl_1e7]=1.0d" -s "wvl_1e8[wvl_1e8]=1.0d" -s "wvl_1e9[wvl_1e9]=1.0d" ~/nco/data/big.nc ~/foo_big.nc
// ls -l ~/foo_big.nc
// On small-RAM machines, create 4 GB file with multiple variables instead
// This reduces peak memory usage to ~400 MB, and still tests LFS capability
// ncap2 -5 -D 3 -O -s "wvl_1e8[wvl_1e8]=1.0f" ~/nco/data/big.nc ${HOME}/big.nc # Data buffer size: 400 MB
// ncrename -D 3 -d wvl_1e8,wvl ${HOME}/big.nc # Rename because ncecat does not concatenate coordinate variables
// Aggregate ten files together
// ncecat -u time -D 3 -O -p ${HOME} big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc ${HOME}/big.nc
// ls -l ${HOME}/big.nc
// Test data access:
// This is where LFS is useful
// These commands should work on 32-bit systems with LFS support even though files exceed 2 GB
// ncks -m -M -H ${HOME}/big.nc | m
// ncks -H -d wvl_1e9,999999999 ${HOME}/big.nc | m
// Valid CDF/netCDF files need not have any defined variable or data
// Use ncap LHS-casting to define variables with big dimensions
netcdf big {
dimensions:
// CDL file only used to supply dimension size = one billion to ncap
// Create actual variable with ncap LHS-casting
wvl_1e9=1000000000,
wvl_1e8=100000000,
wvl_1e7=10000000,
wvl_1e6=1000000,
wvl_1e5=100000,
wvl_1e4=10000,
wvl_1e3=1000,
wvl_1e2=100,
wvl_1e1=10,
wvl_1e0=1,
lev=32,
lat=128,
lon=256,
time=unlimited;
variables:
// float wvl_1e9(wvl_1e9);
// float wvl_1e8(wvl_1e8);
// float wvl_1e7(wvl_1e7);
// float wvl_1e6(wvl_1e6);
// float wvl_1e5(wvl_1e5);
// float wvl_1e4(wvl_1e4);
// float wvl_1e3(wvl_1e3);
// float wvl_1e2(wvl_1e2);
// float wvl_1e1(wvl_1e1);
// float wvl_1e0(wvl_1e0);
double time(time);
time:long_name = "Time coordinate record variable";
data:
// wvl_1e9=1;
// wvl_1e8=1;
// wvl_1e7=1;
// wvl_1e6=1;
// wvl_1e5=1;
// wvl_1e4=1;
// wvl_1e3=1;
// wvl_1e2=1;
// wvl_1e1=1;
// wvl_1e0=1;
time=1.,2.,3.,4.,5.,6.,7.,8.,9.,10.;
}
|
