1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
|
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the files COPYING and Copyright.html. COPYING can be found at the root *
* of the source code distribution tree; Copyright.html can be found at the *
* root level of an installed copy of the electronic HDF5 document set and *
* is linked from the top-level documents page. It can also be found at *
* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
* access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Wednesday, April 8, 1998
*/
#include "h5test.h"
const char *FILENAME[] = {
"big",
"sec2",
"stdio",
NULL
};
#define DNAME "big.data"
#define WRT_N 50
#define WRT_SIZE 4*1024
#define FAMILY_SIZE 1024*1024*1024
/* Define big file as 2GB */
#define BIG_FILE 0x80000000UL
#define MAX_TRIES 100
#if H5_SIZEOF_LONG_LONG >= 8
# define GB8LL ((unsigned long_long)8*1024*1024*1024)
#else
# define GB8LL 0 /*cannot do the test*/
#endif
/* Protocols */
static void usage(void);
static hsize_t values_used[WRT_N];
/*-------------------------------------------------------------------------
* Function: randll
*
* Purpose: Create a random long_long value.
*
* Return: Success: Random value
*
* Failure: never fails
*
* Programmer: Robb Matzke
* Tuesday, November 24, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static hsize_t
randll(hsize_t limit, int current_index)
{
hsize_t acc;
int overlap = 1;
int i;
int tries = 0;
/* Generate up to MAX_TRIES random numbers until one of them */
/* does not overlap with any previous writes */
while(overlap != 0 && tries < MAX_TRIES)
{
acc = rand ();
acc *= rand ();
acc = acc % limit;
overlap = 0;
for(i = 0; i < current_index; i++)
{
if((acc >= values_used[i]) && (acc < values_used[i]+WRT_SIZE))
overlap = 1;
if((acc+WRT_SIZE >= values_used[i]) && (acc+WRT_SIZE < values_used[i
]+WRT_SIZE))
overlap = 1;
}
tries++;
}
values_used[current_index]=acc;
return acc;
}
/*-------------------------------------------------------------------------
* Function: is_sparse
*
* Purpose: Determines if the file system of the current working
* directory supports holes.
*
* Return: Success: Non-zero if holes are supported; zero
* otherwise.
*
* Failure: zero
*
* Programmer: Robb Matzke
* Wednesday, July 15, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static int
is_sparse(void)
{
int fd;
h5_stat_t sb;
if ((fd=HDopen("x.h5", O_RDWR|O_TRUNC|O_CREAT, 0666))<0) return 0;
if (HDlseek(fd, (off_t)(1024*1024), SEEK_SET)!=1024*1024) return 0;
if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
if (HDclose(fd)<0) return 0;
if (HDstat("x.h5", &sb)<0) return 0;
if (HDunlink("x.h5")<0) return 0;
#ifdef H5_HAVE_STAT_ST_BLOCKS
return ((unsigned long)sb.st_blocks*512 < (unsigned long)sb.st_size);
#else
return (0);
#endif
}
/*-------------------------------------------------------------------------
* Function: supports_big
*
* Purpose: Determines if the file system of the current working
* directory supports big files.
*
* Return: Success: Non-zero if big files are supported; zero
* otherwise.
*
* Failure: zero
*
* Programmer: Raymond Lu
* Wednesday, April 18, 2007
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static int
supports_big(void)
{
int fd;
if ((fd=HDopen("y.h5", O_RDWR|O_TRUNC|O_CREAT, 0666))<0) return 0;
/* Write a few bytes at 2GB */
if (HDlseek(fd, BIG_FILE, SEEK_SET)!=BIG_FILE) return 0;
if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
/* Write a few bytes at 4GB */
if (HDlseek(fd, 2*BIG_FILE, SEEK_SET) != 2*BIG_FILE) return 0;
if (5!=HDwrite(fd, "hello", (size_t)5)) return 0;
if (HDclose(fd)<0) return 0;
if (HDunlink("y.h5")<0) return 0;
return (1);
}
/*-------------------------------------------------------------------------
* Function: enough_room
*
* Purpose: Tries to create a bunch of sparse files to see if quotas will
* get in the way. Some systems also have problems opening
* enough files and we'll check that too.
*
* Return: Success: Non-zero
*
* Failure: zero
*
* Programmer: Robb Matzke
* Thursday, August 6, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static int
enough_room(hid_t fapl)
{
int ret_value=0;
int fd[68];
size_t i, size = (size_t)1 << 30;
char filename[1024], name[1024];
/* Initialize file descriptors */
for (i=0; i<NELMTS(fd); i++) fd[i] = -1;
/* Get file name template */
assert(H5FD_FAMILY==H5Pget_driver(fapl));
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
/* Create files */
for (i=0; i<NELMTS(fd); i++) {
HDsnprintf(name, sizeof name, filename, i);
if ((fd[i]=HDopen(name, O_RDWR|O_CREAT|O_TRUNC, 0666))<0) {
goto done;
}
if ((off_t)size != HDlseek(fd[i], (off_t)size, SEEK_SET)) {
goto done;
}
if (1!=HDwrite(fd[i], "X", (size_t)1)) {
goto done;
}
}
ret_value = 1;
done:
for (i=0; i<NELMTS(fd) && fd[i]>=0; i++) {
HDsnprintf(name, sizeof name, filename, i);
if(HDclose(fd[i])<0)
ret_value=0;
HDunlink(name);
}
return ret_value;
}
/*-------------------------------------------------------------------------
* Function: writer
*
* Purpose: Creates a *big* dataset.
*
* Return: Success: 0
*
* Failure: >0
*
* Programmer: Robb Matzke
* Wednesday, April 8, 1998
*
* Modifications:
* Robb Matzke, 15 Jul 1998
* Addresses are written to the file DNAME instead of stdout.
*
*-------------------------------------------------------------------------
*/
static int
writer(char* filename, hid_t fapl, int wrt_n)
{
hsize_t size1[4] = {8, 1024, 1024, 1024};
hsize_t size2[1] = {GB8LL};
hsize_t hs_start[1];
hsize_t hs_size[1];
hid_t file=-1, space1=-1, space2=-1, mem_space=-1, d1=-1, d2=-1;
int *buf = (int*)malloc (sizeof(int) * WRT_SIZE);
int i, j;
FILE *out = fopen(DNAME, "w");
hid_t dcpl;
TESTING("large dataset write");
/*
* We might be on a machine that has 32-bit files, so create an HDF5 file
* which is a family of files. Each member of the family will be 1GB
*/
if ((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl))<0) {
goto error;
}
/* Create simple data spaces according to the size specified above. */
if ((space1 = H5Screate_simple (4, size1, size1))<0 ||
(space2 = H5Screate_simple (1, size2, size2))<0) {
goto error;
}
/* Create the datasets */
/*
* The fix below is provided for bug#921
* H5Dcreate with H5P_DEFAULT creation properties
* will create a set of solid 1GB files; test will crash if quotas are enforced
* or it will take some time to write a file.
* We should create a dataset allocating space late and never writing fill values.
* EIP 4/8/03
if ((d1=H5Dcreate (file, "d1", H5T_NATIVE_INT, space1, H5P_DEFAULT))<0 ||
(d2=H5Dcreate (file, "d2", H5T_NATIVE_INT, space2, H5P_DEFAULT))<0) {
goto error;
}
*/
dcpl = H5Pcreate(H5P_DATASET_CREATE);
H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_LATE);
H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER);
if ((d1=H5Dcreate (file, "d1", H5T_NATIVE_INT, space1, dcpl))<0 ||
(d2=H5Dcreate (file, "d2", H5T_NATIVE_INT, space2, dcpl))<0) {
goto error;
}
/* Write some things to them randomly */
hs_size[0] = WRT_SIZE;
if ((mem_space = H5Screate_simple (1, hs_size, hs_size))<0) goto error;
for (i=0; i<wrt_n; i++) {
hs_start[0] = randll (size2[0], i);
HDfprintf (out, "#%03d 0x%016Hx\n", i, hs_start[0]);
if (H5Sselect_hyperslab (space2, H5S_SELECT_SET, hs_start, NULL,
hs_size, NULL)<0) goto error;
for (j=0; j<WRT_SIZE; j++) {
buf[j] = i+1;
}
if (H5Dwrite (d2, H5T_NATIVE_INT, mem_space, space2,
H5P_DEFAULT, buf)<0) goto error;
}
if (H5Dclose (d1)<0) goto error;
if (H5Dclose (d2)<0) goto error;
if (H5Sclose (mem_space)<0) goto error;
if (H5Sclose (space1)<0) goto error;
if (H5Sclose (space2)<0) goto error;
if (H5Fclose (file)<0) goto error;
free (buf);
fclose(out);
PASSED();
return 0;
error:
H5E_BEGIN_TRY {
H5Dclose(d1);
H5Dclose(d2);
H5Sclose(space1);
H5Sclose(space2);
H5Sclose(mem_space);
H5Fclose(file);
} H5E_END_TRY;
if (buf) free(buf);
if (out) fclose(out);
return 1;
}
/*-------------------------------------------------------------------------
* Function: reader
*
* Purpose: Reads some data from random locations in the dataset.
*
* Return: Success: 0
*
* Failure: >0
*
* Programmer: Robb Matzke
* Friday, April 10, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static int
reader (char *filename, hid_t fapl)
{
FILE *script = NULL;
hid_t file=-1, mspace=-1, fspace=-1, d2=-1;
char ln[128], *s;
hsize_t hs_offset[1];
hsize_t hs_size[1] = {WRT_SIZE};
int *buf = (int*)malloc (sizeof(int) * WRT_SIZE);
int i, j, zero, wrong, nerrors=0;
/* Open script file */
script = fopen (DNAME, "r");
/* Open HDF5 file */
if ((file=H5Fopen(filename, H5F_ACC_RDONLY, fapl))<0) goto error;
/* Open the dataset */
if ((d2 = H5Dopen (file, "d2"))<0) goto error;
if ((fspace = H5Dget_space (d2))<0) goto error;
/* Describe `buf' */
if ((mspace = H5Screate_simple (1, hs_size, hs_size))<0) goto error;
/* Read each region */
while (fgets (ln, (int)sizeof(ln), script)) {
if ('#'!=ln[0]) break;
i = (int)strtol (ln+1, &s, 10);
hs_offset[0] = HDstrtoll (s, NULL, 0);
HDfprintf (stdout, "#%03d 0x%016Hx%47s", i, hs_offset[0], "");
fflush (stdout);
if (H5Sselect_hyperslab (fspace, H5S_SELECT_SET, hs_offset, NULL,
hs_size, NULL)<0) goto error;
if (H5Dread (d2, H5T_NATIVE_INT, mspace, fspace, H5P_DEFAULT, buf)<0) {
goto error;
}
/* Check */
for (j=zero=wrong=0; j<WRT_SIZE; j++) {
if (0==buf[j]) zero++;
else if (buf[j]!=i+1) wrong++;
}
if (zero) {
H5_FAILED();
printf(" %d zero%s\n", zero, 1==zero?"":"s");
} else if (wrong) {
SKIPPED();
puts(" Possible overlap with another region.");
nerrors++;
} else {
PASSED();
}
}
if (H5Dclose (d2)<0) goto error;
if (H5Sclose (mspace)<0) goto error;
if (H5Sclose (fspace)<0) goto error;
if (H5Fclose (file)<0) goto error;
free (buf);
fclose (script);
return nerrors;
error:
H5E_BEGIN_TRY {
H5Dclose(d2);
H5Sclose(mspace);
H5Sclose(fspace);
H5Fclose(file);
} H5E_END_TRY;
if (buf) free(buf);
if (script) fclose(script);
return 1;
}
/*-------------------------------------------------------------------------
* Function: usage
*
* Purpose: Print command usage
*
* Return: void
*
* Programmer: Albert Chent
* Mar 28, 2002
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static void
usage(void)
{
HDfprintf(stdout,
"Usage: big [-h] [-c] [-fsize <fsize>}\n"
"\t-h\tPrint the help page\n"
"\t-c\tFile system Checking skipped. Caution: this test generates\n"
"\t\tmany big files and may fill up the file system.\n"
"\t-fsize\tChange family size default to <fsize> where <fsize> is\n"
"\t\ta positive float point number. Default value is %Hu.\n"
"Examples:\n"
"\tbig -fsize 2.1e9 \t# test with file size just under 2GB\n"
"\tbig -fsize 2.2e9 \t# test with file size just above 2GB\n"
"\tBe sure the file system can support the file size requested\n"
, (hsize_t)FAMILY_SIZE);
}
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose:
*
* Return: Success:
*
* Failure:
*
* Programmer: Robb Matzke
* Friday, April 10, 1998
*
* Modifications:
* Albert Cheng, 2002/03/28
* Added command option -fsize.
* Albert Cheng, 2002/04/19
* Added command option -c.
*
*-------------------------------------------------------------------------
*/
int
main (int ac, char **av)
{
hid_t fapl=-1;
hsize_t family_size;
hsize_t family_size_def; /* default family file size */
double family_size_def_dbl; /* default family file size */
int cflag=1; /* check file system before test */
char filename[1024];
/* parameters setup */
family_size_def = FAMILY_SIZE;
while (--ac > 0){
av++;
if (strcmp("-fsize", *av)==0){
/* specify a different family file size */
ac--; av++;
if (ac > 0){
family_size_def_dbl = atof(*av);
H5_ASSIGN_OVERFLOW(family_size_def,family_size_def_dbl,double,hsize_t);
if (family_size_def <= 0)
family_size_def = (hsize_t)FAMILY_SIZE;
}
else{
printf("***Missing fsize value***\n");
usage();
return 1;
}
}
else if (strcmp("-c", *av)==0){
/* turn off file system check before test */
cflag=0;
}
else if (strcmp("-h", *av)==0){
usage();
return 0;
}else{
usage();
return 1;
}
}
/* Reset library */
h5_reset();
fapl = h5_fileaccess();
/* Test big file with the family driver */
puts("Testing big file with the Family Driver ");
if (H5FD_FAMILY!=H5Pget_driver(fapl)) {
HDfprintf(stdout,
"Changing file drivers to the family driver, %Hu bytes each\n",
family_size_def);
if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT)<0) goto error;
} else if (H5Pget_fapl_family(fapl, &family_size, NULL)<0) {
goto error;
} else if (family_size!=family_size_def) {
HDfprintf(stdout, "Changing family member size from %Hu to %Hu\n",
family_size, family_size_def);
if (H5Pset_fapl_family(fapl, family_size_def, H5P_DEFAULT)<0)
goto error;
}
if (cflag){
/*
* We shouldn't run this test if the file system doesn't support holes
* because we would generate multi-gigabyte files.
*/
puts("Checking if file system is adequate for this test...");
if (sizeof(long_long)<8 || 0==GB8LL) {
puts("Test skipped because sizeof(long_long) is too small. This");
puts("hardware apparently doesn't support 64-bit integer types.");
usage();
goto quit;
}
if (!is_sparse()) {
puts("Test skipped because file system does not support holes.");
usage();
goto quit;
}
if (!enough_room(fapl)) {
puts("Test skipped because of quota (file size or num open files).");
usage();
goto quit;
}
if (sizeof(hsize_t)<=4) {
puts("Test skipped because the hdf5 library was configured with the");
puts("--disable-hsizet flag in order to work around a compiler bug.");
usage();
goto quit;
}
}
/* Do the test with the Family Driver */
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if (writer(filename, fapl, WRT_N)) goto error;
if (reader(filename, fapl)) goto error;
puts("Test passed with the Family Driver.");
/*
* We shouldn't run this test if the file system doesn't support big files
* because we would generate multi-gigabyte files.
*/
puts("\nChecking if file system supports big files...");
if (!supports_big()) {
puts("Tests for sec2 and stdio are skipped because file system does not support big files.");
usage();
goto quit;
}
/* Clean up the test file */
if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
/* Test big file with the SEC2 driver */
puts("Testing big file with the SEC2 Driver ");
fapl = h5_fileaccess();
if(H5Pset_fapl_sec2(fapl)<0)
HDmemset(filename, 0, sizeof(filename));
h5_fixname(FILENAME[2], fapl, filename, sizeof filename);
if (writer(filename, fapl, WRT_N)) goto error;
if (reader(filename, fapl)) goto error;
puts("Test passed with the SEC2 Driver.");
#ifdef H5_HAVE_FSEEKO
/* Clean up the test file */
if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
/* Test big file with the STDIO driver only if fseeko is supported,
* because the OFFSET parameter of fseek has the type LONG, not big
* enough to support big files. */
puts("\nTesting big file with the STDIO Driver ");
fapl = h5_fileaccess();
if(H5Pset_fapl_stdio(fapl)<0)
HDmemset(filename, 0, sizeof(filename));
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
if (writer(filename, fapl, WRT_N)) goto error;
if (reader(filename, fapl)) goto error;
puts("Test passed with the STDIO Driver.");
#endif
quit:
/* End with normal exit code */
if (h5_cleanup(FILENAME, fapl)) remove(DNAME);
return 0;
error:
if (fapl>=0) H5Pclose(fapl);
puts("*** TEST FAILED ***");
return 1;
}
|