1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
|
/* This is part of the netCDF package. Copyright 2005-2018 University
Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
conditions of use.
This is a benchmark test which times how long it takes to create
some files.
Ed Hartnett
*/
#include <nc_tests.h>
#include "err_macros.h"
#include <unistd.h>
#include <time.h>
#include <sys/time.h> /* Extra high precision time info. */
#define MAX_LEN 30
#define TMP_FILE_NAME "tst_files2_tmp.out"
#define FILE_NAME "tst_files2_5_0.nc" /* One of the files created by first test. */
#define MILLION 1000000
void *last_sbrk;
/* Prototype from tst_utils.c. */
int nc4_timeval_subtract(struct timeval *result, struct timeval *x,
struct timeval *y);
/* This function uses the ps command to find the amount of memory in
use by the process. From the ps man page:
size SZ approximate amount of swap space that would be required if
the process were to dirty all writable pages and then be
swapped out. This number is very rough!
*/
int
get_mem_used1(int *mem_used)
{
char cmd[NC_MAX_NAME + 1];
char blob[MAX_LEN + 1] = "";
FILE *fp;
/* Run the ps command for this process, putting output (one number)
* into file TMP_FILE_NAME. */
sprintf(cmd, "ps -o size= %d > %s", getpid(), TMP_FILE_NAME);
system(cmd);
/* Read the results and delete temp file. */
if (!(fp = fopen(TMP_FILE_NAME, "r"))) ERR;
fread(blob, MAX_LEN, 1, fp);
sscanf(blob, "%d", mem_used);
fclose(fp);
unlink(TMP_FILE_NAME);
return NC_NOERR;
}
void
get_mem_used2(int *mem_used)
{
char buf[30];
FILE *pf;
snprintf(buf, 30, "/proc/%u/statm", (unsigned)getpid());
pf = fopen(buf, "r");
if (pf) {
unsigned size; /* total program size */
unsigned resident;/* resident set size */
unsigned share;/* shared pages */
unsigned text;/* text (code) */
unsigned lib;/* library */
unsigned data;/* data/stack */
/*unsigned dt; dirty pages (unused in Linux 2.6)*/
fscanf(pf, "%u %u %u %u %u %u", &size, &resident, &share,
&text, &lib, &data);
*mem_used = data;
}
else
*mem_used = -1;
fclose(pf);
}
void
get_mem_used3(int *mem_used)
{
void *vp;
vp = sbrk(0);
*mem_used = ((char *)vp - (char *)last_sbrk)/1024;
}
/* Create a sample file, with num_vars 3D or 4D variables, with dim
* lens of dim_len size. */
#define MAX_DIMS 4
int
create_sample_file(char *file_name, int ndims, int *dim_len,
int num_vars, int mode, int num_recs)
{
int ncid, dimids[MAX_DIMS], *varids;
char varname[NC_MAX_NAME + 1];
char dim_name[NC_MAX_NAME + 1];
float *data_out;
size_t start[MAX_DIMS], count[MAX_DIMS];
int slab_nelems;
int i, d;
if (ndims != MAX_DIMS && ndims != MAX_DIMS - 1) ERR_RET;
/* Create a file. */
if (nc_create(file_name, NC_CLOBBER|mode, &ncid)) ERR_RET;
/* Initialize sample data. Slab of data will be full extent of last
* two dimensions. */
slab_nelems = dim_len[ndims - 1] * dim_len[ndims - 2];
if (!(data_out = malloc(slab_nelems * sizeof(float)))) ERR_RET;
for (i = 0; i < slab_nelems; i++)
data_out[i] = 42.42 + i;
/* Create the dimensions. */
for (d = 0; d < ndims; d++)
{
sprintf(dim_name, "dim_%d", d);
if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR_RET;
}
/* Define num_vars variables. */
if (!(varids = malloc(num_vars * sizeof(int)))) ERR_RET;
for (i = 0; i < num_vars; i++)
{
sprintf(varname, "a_%d", i);
if (nc_def_var(ncid, varname, NC_FLOAT, ndims, dimids,
&varids[i])) ERR_RET;
}
/* Enddef required for classic files. */
if (nc_enddef(ncid)) ERR;
/* Set up start/count to write slabs of data. */
for (d = 0; d < ndims; d++)
{
if (d < ndims - 2)
count[d] = 1;
else
{
start[d] = 0;
count[d] = dim_len[d];
}
}
/* Now write some data to the vars in slabs. */
for (i = 0; i < num_vars; i++)
{
if (ndims == MAX_DIMS)
{
for (start[0] = 0; start[0] < (dim_len[0] ? dim_len[0] : num_recs); start[0]++)
for (start[1] = 0; start[1] < dim_len[1]; start[1]++)
if (nc_put_vara_float(ncid, varids[i], start, count,
data_out)) ERR_RET;
}
else
{
for (start[0] = 0; start[0] < (dim_len[0] ? dim_len[0] : num_recs); start[0]++)
if (nc_put_vara_float(ncid, varids[i], start, count,
data_out)) ERR_RET;
}
}
/* Free data and close file. */
free(data_out);
free(varids);
if (nc_close(ncid)) ERR_RET;
return NC_NOERR;
}
int
main(int argc, char **argv)
{
printf("\n*** Testing netcdf-4 file functions, some more.\n");
last_sbrk = sbrk(0);
printf("*** testing lots of open files...\n");
{
#define NUM_TRIES 6
int *ncid_in;
int mem_used, mem_used2;
/* int mem_per_file; */
int num_files[NUM_TRIES] = {1, 5, 10, 20, 35, 50};
char file_name[NUM_TRIES][NC_MAX_NAME + 1];
int num_vars[NUM_TRIES];
size_t cache_size[NUM_TRIES];
int mode[NUM_TRIES];
char mode_name[NUM_TRIES][8];
int ndims[NUM_TRIES];
int dim_len[NUM_TRIES][MAX_DIMS];
int dim_4d[MAX_DIMS] = {NC_UNLIMITED, 10, 100, 100};
char dimstr[30];
char chunkstr[30];
int num_recs[NUM_TRIES] = {1, 1, 1, 1, 1, 1};
struct timeval start_time, end_time, diff_time;
struct timeval close_start_time, close_end_time, close_diff_time;
int open_us, close_us, create_us;
size_t chunksize[MAX_DIMS];
int storage;
int d, f, t;
printf("dims\t\tchunks\t\tformat\tnum_files\tcache(kb)\tnum_vars\tmem(kb)\t"
"open_time(us/file)\tclose_time(us/file)\tcreate_time(us/file)\n");
for (t = 0; t < NUM_TRIES; t++)
{
strcpy(mode_name[t], "netcdf4");
mode[t] = NC_NETCDF4;
cache_size[t] = 16000000;
num_vars[t] = 10;
ndims[t] = 4;
for (d = 0; d < ndims[t]; d++)
dim_len[t][d] = dim_4d[d];
/* Create sample files. */
if (gettimeofday(&start_time, NULL)) ERR;
for (f = 0; f < num_files[t]; f++)
{
/* Set up filename. */
sprintf(file_name[t], "tst_files2_%d_%d.nc", t, f);
if (create_sample_file(file_name[t], ndims[t], dim_len[t], num_vars[t],
mode[t], num_recs[t])) ERR;
/* How long did it take? */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
create_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t];
}
/* /\* Change the cache settings. *\/ */
/* if (nc_set_chunk_cache(cache_size[t], 20000, .75)) ERR; */
/* We need storage for an array of ncids. */
if (!(ncid_in = malloc(num_files[t] * sizeof(int)))) ERR;
/* How much memory is in use now? */
if (get_mem_used1(&mem_used)) ERR;
/* Open the first file to get chunksizes. */
if (gettimeofday(&start_time, NULL)) ERR;
if (nc_open(file_name[t], 0, &ncid_in[0])) ERR;
if (nc_inq_var_chunking(ncid_in[0], 0, &storage, chunksize)) ERR;
/* Now reopen this file a large number of times. */
for (f = 1; f < num_files[t]; f++)
if (nc_open(file_name[t], 0, &ncid_in[f])) ERR_RET;
/* How long did it take per file? */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
open_us = ((int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec) / num_files[t];
/* How much memory is in use by this process now? */
if (get_mem_used1(&mem_used2)) ERR;
/* Close all netcdf files. */
if (gettimeofday(&close_start_time, NULL)) ERR;
for (f = 0; f < num_files[t]; f++)
if (nc_close(ncid_in[f])) ERR_RET;
/* How long did it take to close all files? */
if (gettimeofday(&close_end_time, NULL)) ERR;
if (nc4_timeval_subtract(&close_diff_time, &close_end_time, &close_start_time)) ERR;
close_us = ((int)close_diff_time.tv_sec * MILLION +
(int)close_diff_time.tv_usec) / num_files[t];
/* We're done with this. */
free(ncid_in);
/* How much memory was used for each open file? */
/* mem_per_file = mem_used2/num_files[t]; */
/* Prepare the dimensions string. */
if (ndims[t] == MAX_DIMS)
sprintf(dimstr, "%dx%dx%dx%d", dim_len[t][0], dim_len[t][1],
dim_len[t][2], dim_len[t][3]);
else
sprintf(dimstr, "%dx%dx%d", dim_len[t][0], dim_len[t][1],
dim_len[t][2]);
/* Prepare the chunksize string. */
if (storage == NC_CHUNKED)
{
if (ndims[t] == MAX_DIMS)
sprintf(chunkstr, "%dx%dx%dx%d", (int)chunksize[0], (int)chunksize[1],
(int)chunksize[2], (int)chunksize[3]);
else
sprintf(chunkstr, "%dx%dx%d", (int)chunksize[0], (int)chunksize[1],
(int)chunksize[2]);
}
else
strcpy(chunkstr, "contig ");
/* Output results. */
printf("%s\t%s\t%s\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\t\t%d\n",
dimstr, chunkstr, mode_name[t], num_files[t], (int)(cache_size[t]/1024),
num_vars[t], mem_used2, open_us, close_us, create_us);
}
}
SUMMARIZE_ERR;
printf("Test for memory consumption...\n");
{
#define NUM_TRIES_100 100
int ncid, i;
int mem_used, mem_used1, mem_used2;
get_mem_used2(&mem_used);
mem_used1 = mem_used;
mem_used2 = mem_used;
printf("start: memuse= %d\t%d\t%d \n",mem_used, mem_used1,
mem_used2);
printf("bef_open\taft_open\taft_close\tused_open\tused_closed\n");
for (i=0; i < NUM_TRIES_100; i++)
{
/* Open the file. NC_NOWRITE tells netCDF we want read-only access
* to the file.*/
get_mem_used2(&mem_used);
nc_set_chunk_cache(10,10,.5);
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
get_mem_used2(&mem_used1);
/* Close the file, freeing all resources. ???? */
if (nc_close(ncid)) ERR;
get_mem_used2(&mem_used2);
if (mem_used2 - mem_used)
printf("try %d - %d\t\t%d\t\t%d\t\t%d\t\t%d \n", i,
mem_used, mem_used1, mem_used2, mem_used1 - mem_used,
mem_used2 - mem_used);
}
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}
|