1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
|
/*
Copyright 2007, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
This program benchmarks the write and read of reasonable large files,
with different chunking parameters set.
$Id: bm_chunking.c,v 1.4 2009/08/06 17:29:32 ed Exp $
*/
#include <config.h>
#include <nc_tests.h>
#include <netcdf.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <sys/time.h> /* Extra high precision time info. */
#include <math.h>
/* This is one megabyte (2^20), in decimal. */
#define MEGABYTE 1048576
/* Define this to debug. */
#define DIM_LEN (MEGABYTE*32)
/*#define DIM_LEN (1024*8)*/
/* Size, in bytes, of a double. */
#define DOUBLE_SIZE 8
/* We will create this file. */
#define FILE_NAME "bm_chunking.nc"
/* Subtract the `struct timeval' values X and Y, storing the result in
RESULT. Return 1 if the difference is negative, otherwise 0. This
function from the GNU documentation. */
int
timeval_subtract (result, x, y)
struct timeval *result, *x, *y;
{
/* Perform the carry for the later subtraction by updating Y. */
if (x->tv_usec < y->tv_usec) {
int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1;
y->tv_usec -= 1000000 * nsec;
y->tv_sec += nsec;
}
if (x->tv_usec - y->tv_usec > 1000000) {
int nsec = (x->tv_usec - y->tv_usec) / 1000000;
y->tv_usec += 1000000 * nsec;
y->tv_sec -= nsec;
}
/* Compute the time remaining to wait.
`tv_usec' is certainly positive. */
result->tv_sec = x->tv_sec - y->tv_sec;
result->tv_usec = x->tv_usec - y->tv_usec;
/* Return 1 if result is negative. */
return x->tv_sec < y->tv_sec;
}
int
main(int argc, char **argv)
{
struct timeval start_time, end_time, diff_time;
printf("\n*** Benchmarking chunking.\n");
printf("\n*** Trying different chunksizes in 4 var, 1D file with "
"large dimension (%d)...\n", DIM_LEN);
{
#define DIM_NAME "Some_really_long_dimension"
#define NUMDIMS 1
#define NUMVARS 4
#define NUM_ATTEMPTS 10
#define NUM_VALUES_TO_WRITE (DIM_LEN/1024)
int ncid, dimids[NUMDIMS], varid[NUMVARS], chunksize[NUMDIMS];
char var_name[NUMVARS][NC_MAX_NAME + 1] = {"England", "Ireland", "Scotland", "Wales"};
size_t start[NUMDIMS], count[NUMDIMS] = {NUM_VALUES_TO_WRITE};
int ndims, nvars, natts, unlimdimid;
nc_type xtype;
char name_in[NC_MAX_NAME + 1];
size_t len;
double pi = 3.1459;
double data[NUM_VALUES_TO_WRITE], data_in[NUM_VALUES_TO_WRITE];
int a, i, j;
/* Generate phoney data. */
for (i = 0; i < NUM_VALUES_TO_WRITE; i++)
data[i] = pi * i;
/* Try NUM_ATTEMPTS different values of chunksize, timing each one. */
for (a = 0; a < NUM_ATTEMPTS; a++)
{
/* Create a netCDF netCDF-4/HDF5 format file, with 4 vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR;
if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, dimids)) ERR;
/* According to Quncey, a square chunk around 1 MB in size is a
* reasonable idea. */
chunksize[0] = DIM_LEN/pow(2, a+6);
for (i = 0; i < NUMVARS; i++)
{
if (nc_def_var(ncid, var_name[i], NC_DOUBLE, NUMDIMS,
dimids, &varid[i])) ERR;
if (nc_def_var_chunking(ncid, i, NC_CHUNKED, chunksize)) ERR;
}
if (nc_enddef(ncid)) ERR;
/* Write the data and time it. */
if (gettimeofday(&start_time, NULL))
ERR;
for (start[0] = 0; start[0] < DIM_LEN; start[0] += NUM_VALUES_TO_WRITE)
for (i = 0; i < NUMVARS; i++)
if (nc_put_vara_double(ncid, i, start, count, data)) ERR;
if (nc_close(ncid)) ERR;
if (gettimeofday(&end_time, NULL)) ERR;
if (timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
printf("chunksize %d write time %d sec %d usec\n", chunksize[0],
(int)diff_time.tv_sec, (int)diff_time.tv_usec);
/* Reopen and check the file metadata. */
if (nc_open(FILE_NAME, 0, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
if (ndims != NUMDIMS || nvars != NUMVARS || natts != 0 || unlimdimid != -1) ERR;
if (nc_inq_dimids(ncid, &ndims, dimids, 1)) ERR;
if (ndims != 1 || dimids[0] != 0) ERR;
if (nc_inq_dim(ncid, 0, name_in, &len)) ERR;
if (strcmp(name_in, DIM_NAME) || len != DIM_LEN) ERR;
for (i = 0; i < NUMVARS; i++)
{
if (nc_inq_var(ncid, i, name_in, &xtype, &ndims, dimids, &natts)) ERR;
if (strcmp(name_in, var_name[i]) || xtype != NC_DOUBLE || ndims != 1 ||
dimids[0] != 0 || natts != 0) ERR;
}
/* Read the data and time it. */
if (gettimeofday(&start_time, NULL)) ERR;
for (start[0] = 0; start[0] < DIM_LEN; start[0] += NUM_VALUES_TO_WRITE)
for (i = 0; i < NUMVARS; i++)
if (nc_get_vara_double(ncid, i, start, count, data_in)) ERR;
if (gettimeofday(&end_time, NULL)) ERR;
if (timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
printf("chunksize %d read time %d sec %d usec\n", chunksize[0],
(int)diff_time.tv_sec, (int)diff_time.tv_usec);
/* Reread, and check the data (but don't time this operation). */
for (start[0] = DIM_LEN - NUM_VALUES_TO_WRITE; start[0] > 0; start[0] -= NUM_VALUES_TO_WRITE)
for (i = 0; i < NUMVARS; i++)
{
if (nc_get_vara_double(ncid, i, start, count, data_in)) ERR;
for (j = 0; j < NUM_VALUES_TO_WRITE; j++)
if (data[j] != data_in[j]) ERR;
}
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}
|
