1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
|
# This file is part of h5py, a Python interface to the HDF5 library.
#
# http://www.h5py.org
#
# Copyright 2008-2013 Andrew Collette and contributors
#
# License: Standard 3-clause BSD; see "license.txt" for full license terms
# and contributor agreement.
"""
Provides access to the low-level HDF5 "H5D" dataset interface.
"""
include "config.pxi"
# Compile-time imports
from _objects cimport pdefault
from numpy cimport ndarray, import_array, PyArray_DATA, NPY_WRITEABLE
from cpython cimport array
from utils cimport check_numpy_read, check_numpy_write, \
convert_tuple, emalloc, efree
from h5t cimport TypeID, typewrap, py_create
from h5s cimport SpaceID
from h5p cimport PropID, propwrap
from _proxy cimport dset_rw
from ._objects import phil, with_phil
# Initialization
import_array()
# === Public constants and data structures ====================================
COMPACT = H5D_COMPACT
CONTIGUOUS = H5D_CONTIGUOUS
CHUNKED = H5D_CHUNKED
ALLOC_TIME_DEFAULT = H5D_ALLOC_TIME_DEFAULT
ALLOC_TIME_LATE = H5D_ALLOC_TIME_LATE
ALLOC_TIME_EARLY = H5D_ALLOC_TIME_EARLY
ALLOC_TIME_INCR = H5D_ALLOC_TIME_INCR
SPACE_STATUS_NOT_ALLOCATED = H5D_SPACE_STATUS_NOT_ALLOCATED
SPACE_STATUS_PART_ALLOCATED = H5D_SPACE_STATUS_PART_ALLOCATED
SPACE_STATUS_ALLOCATED = H5D_SPACE_STATUS_ALLOCATED
FILL_TIME_ALLOC = H5D_FILL_TIME_ALLOC
FILL_TIME_NEVER = H5D_FILL_TIME_NEVER
FILL_TIME_IFSET = H5D_FILL_TIME_IFSET
FILL_VALUE_UNDEFINED = H5D_FILL_VALUE_UNDEFINED
FILL_VALUE_DEFAULT = H5D_FILL_VALUE_DEFAULT
FILL_VALUE_USER_DEFINED = H5D_FILL_VALUE_USER_DEFINED
IF HDF5_VERSION >= VDS_MIN_HDF5_VERSION:
VIRTUAL = H5D_VIRTUAL
VDS_FIRST_MISSING = H5D_VDS_FIRST_MISSING
VDS_LAST_AVAILABLE = H5D_VDS_LAST_AVAILABLE
# === Dataset operations ======================================================
@with_phil
def create(ObjectID loc not None, object name, TypeID tid not None,
SpaceID space not None, PropID dcpl=None, PropID lcpl=None, PropID dapl = None):
""" (objectID loc, STRING name or None, TypeID tid, SpaceID space,
PropDCID dcpl=None, PropID lcpl=None) => DatasetID
Create a new dataset. If "name" is None, the dataset will be
anonymous.
"""
cdef hid_t dsid
cdef char* cname = NULL
if name is not None:
cname = name
if cname != NULL:
dsid = H5Dcreate2(loc.id, cname, tid.id, space.id,
pdefault(lcpl), pdefault(dcpl), pdefault(dapl))
else:
dsid = H5Dcreate_anon(loc.id, tid.id, space.id,
pdefault(dcpl), pdefault(dapl))
return DatasetID(dsid)
@with_phil
def open(ObjectID loc not None, char* name, PropID dapl=None):
""" (ObjectID loc, STRING name, PropID dapl=None) => DatasetID
Open an existing dataset attached to a group or file object, by name.
If specified, dapl may be a dataset access property list.
"""
return DatasetID(H5Dopen2(loc.id, name, pdefault(dapl)))
# --- Proxy functions for safe(r) threading -----------------------------------
cdef class DatasetID(ObjectID):
"""
Represents an HDF5 dataset identifier.
Objects of this class may be used in any HDF5 function which expects
a dataset identifier. Also, all H5D* functions which take a dataset
instance as their first argument are presented as methods of this
class.
Properties:
dtype: Numpy dtype representing the dataset type
shape: Numpy-style shape tuple representing the dataspace
rank: Integer giving dataset rank
* Hashable: Yes, unless anonymous
* Equality: True HDF5 identity if unless anonymous
"""
property dtype:
""" Numpy dtype object representing the dataset type """
def __get__(self):
# Dataset type can't change
cdef TypeID tid
with phil:
if self._dtype is None:
tid = self.get_type()
self._dtype = tid.dtype
return self._dtype
property shape:
""" Numpy-style shape tuple representing the dataspace """
def __get__(self):
# Shape can change (DatasetID.extend), so don't cache it
cdef SpaceID sid
with phil:
sid = self.get_space()
return sid.get_simple_extent_dims()
property rank:
""" Integer giving the dataset rank (0 = scalar) """
def __get__(self):
cdef SpaceID sid
with phil:
sid = self.get_space()
return sid.get_simple_extent_ndims()
@with_phil
def read(self, SpaceID mspace not None, SpaceID fspace not None,
ndarray arr_obj not None, TypeID mtype=None,
PropID dxpl=None):
""" (SpaceID mspace, SpaceID fspace, NDARRAY arr_obj,
TypeID mtype=None, PropDXID dxpl=None)
Read data from an HDF5 dataset into a Numpy array.
It is your responsibility to ensure that the memory dataspace
provided is compatible with the shape of the Numpy array. Since a
wide variety of dataspace configurations are possible, this is not
checked. You can easily crash Python by reading in data from too
large a dataspace.
If a memory datatype is not specified, one will be auto-created
based on the array's dtype.
The provided Numpy array must be writable and C-contiguous. If
this is not the case, ValueError will be raised and the read will
fail. Keyword dxpl may be a dataset transfer property list.
"""
cdef hid_t self_id, mtype_id, mspace_id, fspace_id, plist_id
cdef void* data
cdef int oldflags
if mtype is None:
mtype = py_create(arr_obj.dtype)
check_numpy_write(arr_obj, -1)
self_id = self.id
mtype_id = mtype.id
mspace_id = mspace.id
fspace_id = fspace.id
plist_id = pdefault(dxpl)
data = PyArray_DATA(arr_obj)
dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 1)
@with_phil
def write(self, SpaceID mspace not None, SpaceID fspace not None,
ndarray arr_obj not None, TypeID mtype=None,
PropID dxpl=None):
""" (SpaceID mspace, SpaceID fspace, NDARRAY arr_obj,
TypeID mtype=None, PropDXID dxpl=None)
Write data from a Numpy array to an HDF5 dataset. Keyword dxpl may
be a dataset transfer property list.
It is your responsibility to ensure that the memory dataspace
provided is compatible with the shape of the Numpy array. Since a
wide variety of dataspace configurations are possible, this is not
checked. You can easily crash Python by writing data from too
large a dataspace.
If a memory datatype is not specified, one will be auto-created
based on the array's dtype.
The provided Numpy array must be C-contiguous. If this is not the
case, ValueError will be raised and the read will fail.
"""
cdef hid_t self_id, mtype_id, mspace_id, fspace_id, plist_id
cdef void* data
cdef int oldflags
if mtype is None:
mtype = py_create(arr_obj.dtype)
check_numpy_read(arr_obj, -1)
self_id = self.id
mtype_id = mtype.id
mspace_id = mspace.id
fspace_id = fspace.id
plist_id = pdefault(dxpl)
data = PyArray_DATA(arr_obj)
dset_rw(self_id, mtype_id, mspace_id, fspace_id, plist_id, data, 0)
@with_phil
def extend(self, tuple shape):
""" (TUPLE shape)
Extend the given dataset so it's at least as big as "shape". Note
that a dataset may only be extended up to the maximum dimensions of
its dataspace, which are fixed when the dataset is created.
"""
cdef int rank
cdef hid_t space_id = 0
cdef hsize_t* dims = NULL
try:
space_id = H5Dget_space(self.id)
rank = H5Sget_simple_extent_ndims(space_id)
if len(shape) != rank:
raise TypeError("New shape length (%d) must match dataset rank (%d)" % (len(shape), rank))
dims = <hsize_t*>emalloc(sizeof(hsize_t)*rank)
convert_tuple(shape, dims, rank)
H5Dextend(self.id, dims)
finally:
efree(dims)
if space_id:
H5Sclose(space_id)
@with_phil
def set_extent(self, tuple shape):
""" (TUPLE shape)
Set the size of the dataspace to match the given shape. If the new
size is larger in any dimension, it must be compatible with the
maximum dataspace size.
"""
cdef int rank
cdef hid_t space_id = 0
cdef hsize_t* dims = NULL
try:
space_id = H5Dget_space(self.id)
rank = H5Sget_simple_extent_ndims(space_id)
if len(shape) != rank:
raise TypeError("New shape length (%d) must match dataset rank (%d)" % (len(shape), rank))
dims = <hsize_t*>emalloc(sizeof(hsize_t)*rank)
convert_tuple(shape, dims, rank)
H5Dset_extent(self.id, dims)
finally:
efree(dims)
if space_id:
H5Sclose(space_id)
@with_phil
def get_space(self):
""" () => SpaceID
Create and return a new copy of the dataspace for this dataset.
"""
return SpaceID(H5Dget_space(self.id))
@with_phil
def get_space_status(self):
""" () => INT space_status_code
Determine if space has been allocated for a dataset.
Return value is one of:
* SPACE_STATUS_NOT_ALLOCATED
* SPACE_STATUS_PART_ALLOCATED
* SPACE_STATUS_ALLOCATED
"""
cdef H5D_space_status_t status
H5Dget_space_status(self.id, &status)
return <int>status
@with_phil
def get_type(self):
""" () => TypeID
Create and return a new copy of the datatype for this dataset.
"""
return typewrap(H5Dget_type(self.id))
@with_phil
def get_create_plist(self):
""" () => PropDCID
Create an return a new copy of the dataset creation property list
used when this dataset was created.
"""
return propwrap(H5Dget_create_plist(self.id))
@with_phil
def get_access_plist(self):
""" () => PropDAID
Create an return a new copy of the dataset access property list.
"""
return propwrap(H5Dget_access_plist(self.id))
@with_phil
def get_offset(self):
""" () => LONG offset or None
Get the offset of this dataset in the file, in bytes, or None if
it doesn't have one. This is always the case for datasets which
use chunked storage, compact datasets, and datasets for which space
has not yet been allocated in the file.
"""
cdef haddr_t offset
offset = H5Dget_offset(self.id)
if offset == HADDR_UNDEF:
return None
return offset
@with_phil
def get_storage_size(self):
""" () => LONG storage_size
Determine the amount of file space required for a dataset. Note
this only counts the space which has actually been allocated; it
may even be zero.
"""
return H5Dget_storage_size(self.id)
IF HDF5_VERSION >= SWMR_MIN_HDF5_VERSION:
@with_phil
def flush(self):
""" no return
Flushes all buffers associated with a dataset to disk.
This function causes all buffers associated with a dataset to be
immediately flushed to disk without removing the data from the cache.
Use this in SWMR write mode to allow readers to be updated with the
dataset changes.
Feature requires: 1.9.178 HDF5
"""
H5Dflush(self.id)
@with_phil
def refresh(self):
""" no return
Refreshes all buffers associated with a dataset.
This function causes all buffers associated with a dataset to be
cleared and immediately re-loaded with updated contents from disk.
This function essentially closes the dataset, evicts all metadata
associated with it from the cache, and then re-opens the dataset.
The reopened dataset is automatically re-registered with the same ID.
Use this in SWMR read mode to poll for dataset changes.
Feature requires: 1.9.178 HDF5
"""
H5Drefresh(self.id)
IF HDF5_VERSION >= (1, 8, 11):
def write_direct_chunk(self, offsets, bytes data, filter_mask=0x00000000, PropID dxpl=None):
""" (offsets, bytes data, uint32_t filter_mask=0x00000000, PropID dxpl=None)
This function bypasses any filters HDF5 would normally apply to
written data. However, calling code may apply filters (e.g. gzip
compression) itself before writing the data.
`filter_mask` is a bit field of up to 32 values. It records which
filters have been applied to this chunk, of the filter pipeline
defined for that dataset. Each bit set to `1` means that the filter
in the corresponding position in the pipeline was not applied.
So the default value of `0` means that all defined filters have
been applied to the data before calling this function.
Feature requires: 1.8.11 HDF5
"""
cdef hid_t dset_id
cdef hid_t dxpl_id
cdef hid_t space_id = 0
cdef hsize_t *offset = NULL
cdef size_t data_size
cdef int rank
dset_id = self.id
dxpl_id = pdefault(dxpl)
space_id = H5Dget_space(self.id)
rank = H5Sget_simple_extent_ndims(space_id)
if len(offsets) != rank:
raise TypeError("offset length (%d) must match dataset rank (%d)" % (len(offsets), rank))
try:
offset = <hsize_t*>emalloc(sizeof(hsize_t)*rank)
convert_tuple(offsets, offset, rank)
H5DOwrite_chunk(dset_id, dxpl_id, filter_mask, offset, len(data), <char *> data)
finally:
efree(offset)
if space_id:
H5Sclose(space_id)
IF HDF5_VERSION >= (1, 10, 2):
def read_direct_chunk(self, offsets, PropID dxpl=None):
""" (offsets, PropID dxpl=None)
Reads data to a bytes array directly from a chunk at position
specified by the `offsets` argument and bypasses any filters HDF5
would normally apply to the written data. However, the written data
may be compressed or not.
Returns a tuple containing the `filter_mask` and the bytes data
which are the raw data storing this chuck.
`filter_mask` is a bit field of up to 32 values. It records which
filters have been applied to this chunk, of the filter pipeline
defined for that dataset. Each bit set to `1` means that the filter
in the corresponding position in the pipeline was not applied to
compute the raw data. So the default value of `0` means that all
defined filters have been applied to the raw data.
Feature requires: 1.10.2 HDF5
"""
cdef hid_t dset_id
cdef hid_t dxpl_id
cdef hid_t space_id = 0
cdef hsize_t *offset = NULL
cdef size_t data_size
cdef int rank
cdef uint32_t filters
cdef hsize_t read_chunk_nbytes
cdef array.array data = array.array('B')
dset_id = self.id
dxpl_id = pdefault(dxpl)
space_id = H5Dget_space(self.id)
rank = H5Sget_simple_extent_ndims(space_id)
if len(offsets) != rank:
raise TypeError("offset length (%d) must match dataset rank (%d)" % (len(offsets), rank))
try:
offset = <hsize_t*>emalloc(sizeof(hsize_t)*rank)
convert_tuple(offsets, offset, rank)
H5Dget_chunk_storage_size(dset_id, offset, &read_chunk_nbytes)
array.resize(data, read_chunk_nbytes)
IF HDF5_VERSION >= (1, 10, 3):
H5Dread_chunk(dset_id, dxpl_id, offset, &filters, data.data.as_voidptr)
ELSE:
H5DOread_chunk(dset_id, dxpl_id, offset, &filters, data.data.as_voidptr)
finally:
efree(offset)
if space_id:
H5Sclose(space_id)
return filters, bytes(data)
|