1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
// Purpose: CDL file to generate large netCDF test file for Large File Support (LFS) testing
// Usage:
// Create big.nc from big.cdl on a beefy 64-bit computer
// The file creation command will fail on all systems with insufficient RAM
// Transfer big.nc to a 32-bit system as an LFS testfile
// ncgen -b -o big.nc big.cdl
// ncgen -b -o ~/nco/data/big.nc ~/nco/data/big.cdl
// scp ~/nco/data/big.cdl esmf.ess.uci.edu:nco/data
// scp ~/nco/data/big.cdl sand.ess.uci.edu:nco/data
// Test 64-bit netCDF/NCO capabilities
// 32-bit machines do not work with files exceeding ~2 GB unless LFS enabled
// One billion floats are nco_typ_lng(NC_FLOAT)*10^9 = sizeof(float)*10^9 = 4*10^9 B = 4 GB
// Create 4 GB file with one variable
// Expect this to fail on "small RAM" (< ~4 GB) machines:
// ncap -D 3 -O -s "wvl_1e9[wvl_1e9]=1.0f" ~/nco/data/big.nc ${DATA}/tmp/big.nc
// ls -l ${DATA}/tmp/big.nc
// On small-RAM machines, create 4 GB file with multiple variables instead
// This reduces peak memory usage to ~400 MB, and still tests LFS capability
// ncap -D 3 -O -s "wvl_1e8[wvl_1e8]=1.0f" ~/nco/data/big.nc ${DATA}/tmp/big.nc
// ncrename -D -d wvl_1e8,wvl big.nc # Rename because ncecat does not do coordinate variables
// Aggregate ten files together
// ncecat -D 3 -O -p ${DATA}/tmp big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc big.nc ${DATA}/tmp/big.nc
// ls -l ${DATA}/tmp/big.nc
// Test data access:
// This is where LFS is useful
// These commands should work on 32-bit systems with LFS support even though files exceed 2 GB
// ncks -m -M -H ${DATA}/tmp/big.nc | m
// ncks -H -d wvl_1e9,999999999 ${DATA}/tmp/big.nc | m
// Valid CDF/netCDF files need not have any defined variable or data
// Use ncap LHS-casting to define variables with big dimensions
netcdf big {
dimensions:
// CDL file only used to supply dimension size = one billion to ncap
// Create actual variable with ncap LHS-casting
wvl_1e9=1000000000;
wvl_1e8=100000000;
wvl_1e7=10000000;
wvl_1e6=1000000;
wvl_1e5=100000;
wvl_1e4=10000;
wvl_1e3=1000;
wvl_1e2=100;
wvl_1e1=10;
wvl_1e0=1;
//variables:
// float wvl_1e9(wvl_1e9);
// float wvl_1e8(wvl_1e8);
// float wvl_1e7(wvl_1e7);
// float wvl_1e6(wvl_1e6);
// float wvl_1e5(wvl_1e5);
// float wvl_1e4(wvl_1e4);
// float wvl_1e3(wvl_1e3);
// float wvl_1e2(wvl_1e2);
// float wvl_1e1(wvl_1e1);
// float wvl_1e0(wvl_1e0);
//data:
// wvl_1e9=1;
// wvl_1e8=1;
// wvl_1e7=1;
// wvl_1e6=1;
// wvl_1e5=1;
// wvl_1e4=1;
// wvl_1e3=1;
// wvl_1e2=1;
// wvl_1e1=1;
// wvl_1e0=1;
}
|
