File: d4data.c

package info (click to toggle)
netcdf 1:4.7.4-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 104,952 kB
  • sloc: ansic: 228,683; sh: 10,980; yacc: 2,561; makefile: 1,319; lex: 1,173; xml: 173; awk: 2
file content (377 lines) | stat: -rw-r--r-- 10,503 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
/*********************************************************************
 *   Copyright 2018, UCAR/Unidata
 *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
 *********************************************************************/

#include "d4includes.h"
#include <stdarg.h>
#include <assert.h>
#include "ezxml.h"
#include "d4includes.h"
#include "d4odom.h"

/**
This code serves two purposes
1. Preprocess the dap4 serialization wrt endianness, etc.
   (NCD4_processdata)
2. Walk a specified variable instance to convert to netcdf4
   memory representation.
   (NCD4_fillinstance)

*/

/***************************************************/
/* Forwards */
static int fillstring(NCD4meta*, void** offsetp, void** dstp, NClist* blobs);
static int fillopfixed(NCD4meta*, d4size_t opaquesize, void** offsetp, void** dstp);
static int fillopvar(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs);
static int fillstruct(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs);
static int fillseq(NCD4meta*, NCD4node* type, void** offsetp, void** dstp, NClist* blobs);

/***************************************************/
/* Macro define procedures */

#ifdef D4DUMPCSUM
static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size)
{
    int i;
    fprintf(stderr,"crc32: ");
    for(i=0;i<size;i++) {fprintf(stderr,"%02x",((unsigned char*)buf)[i]);}
    fprintf(stderr,"\n");
    return NCD4_crc32(crc,buf,size);
}
#define CRC32 debugcrc32
#else
#define CRC32 NCD4_crc32
#endif

#define ISTOPLEVEL(var) ((var)->container == NULL || (var)->container->sort == NCD4_GROUP)

/***************************************************/
/* API */

int
NCD4_processdata(NCD4meta* meta)
{
    int ret = NC_NOERR;
    int i;
    NClist* toplevel = NULL;
    NCD4node* root = meta->root;
    void* offset;

    /* Recursively walk the tree in prefix order 
       to get the top-level variables; also mark as unvisited */
    toplevel = nclistnew();
    NCD4_getToplevelVars(meta,root,toplevel);

    /* If necessary, byte swap the serialized data */
    /* Do we need to swap the dap4 data? */
    meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian);

    /* Compute the  offset and size of the toplevel vars in the raw dap data. */
    offset = meta->serial.dap;
    for(i=0;i<nclistlength(toplevel);i++) {
	NCD4node* var = (NCD4node*)nclistget(toplevel,i);
        if((ret=NCD4_delimit(meta,var,&offset)))
	    FAIL(ret,"delimit failure");
    }

    /* Compute the checksums of the top variables */
    /* must occur before any byte swapping */
    if(meta->localchecksumming) {
	for(i=0;i<nclistlength(toplevel);i++) {
	    unsigned int csum = 0;
	    NCD4node* var = (NCD4node*)nclistget(toplevel,i);
            csum = CRC32(csum,var->data.dap4data.memory,var->data.dap4data.size);
            var->data.localchecksum = csum;
	}
    }

    /* verify checksums */
    if(!meta->ignorechecksums && meta->serial.remotechecksumming) {
        for(i=0;i<nclistlength(toplevel);i++) {
	    NCD4node* var = (NCD4node*)nclistget(toplevel,i);
	    if(var->data.localchecksum != var->data.remotechecksum) {
		nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
		ret = NC_EDAP;
		goto done;
	    }
        }
    }

    /* Swap the data for each top level variable,
    */
    if(meta->swap) {
        if((ret=NCD4_swapdata(meta,toplevel)))
	    FAIL(ret,"byte swapping failed");
    }

done:
    if(toplevel) nclistfree(toplevel);
    return THROW(ret);
}

/*
Build a single instance of a type. The blobs
argument accumulates any malloc'd data so we can
reclaim it in case of an error.

Activity is to walk the variable's data to
produce a copy that is compatible with the
netcdf4 memory format.

Assumes that NCD4_processdata has been called.
*/

int
NCD4_fillinstance(NCD4meta* meta, NCD4node* type, void** offsetp, void** dstp, NClist* blobs)
{
    int ret = NC_NOERR;
    void* offset = *offsetp;
    void* dst = *dstp;
    d4size_t memsize = type->meta.memsize;
    d4size_t dapsize = type->meta.dapsize;

    /* If the type is fixed size, then just copy it  */
    if(type->subsort <= NC_UINT64 || type->subsort == NC_ENUM) {
	/* memsize and dapsize are the same */
	assert(memsize == dapsize);
	memcpy(dst,offset,dapsize);
	offset = INCR(offset,dapsize);
    } else switch(type->subsort) {
        case NC_STRING: /* oob strings */
	    if((ret=fillstring(meta,&offset,&dst,blobs)))
	        FAIL(ret,"fillinstance");
	    break;
	case NC_OPAQUE:
	    if(type->opaque.size > 0) {
	        /* We know the size and its the same for all instances */
	        if((ret=fillopfixed(meta,type->opaque.size,&offset,&dst)))
	            FAIL(ret,"fillinstance");
   	    } else {
	        /* Size differs per instance, so we need to convert each opaque to a vlen */
	        if((ret=fillopvar(meta,type,&offset,&dst,blobs)))
	            FAIL(ret,"fillinstance");
	    }
	    break;
	case NC_STRUCT:
	    if((ret=fillstruct(meta,type,&offset,&dst,blobs)))
                FAIL(ret,"fillinstance");
	    break;
	case NC_SEQ:
	    if((ret=fillseq(meta,type,&offset,&dst,blobs)))
                FAIL(ret,"fillinstance");
	    break;
	default:
	    ret = NC_EINVAL;
            FAIL(ret,"fillinstance");
    }
    *dstp = dst;
    *offsetp = offset; /* return just past this object in dap data */
done:
    return THROW(ret);
}

static int
fillstruct(NCD4meta* meta, NCD4node* type, void** offsetp, void** dstp, NClist* blobs)
{
    int i,ret = NC_NOERR;
    void* offset = *offsetp;
    void* dst = *dstp;
 
#ifdef CLEARSTRUCT
    /* Avoid random data within aligned structs */
    memset(dst,0,type->meta.memsize);
#endif

    /* Walk and read each field taking alignments into account */
    for(i=0;i<nclistlength(type->vars);i++) {
	NCD4node* field = nclistget(type->vars,i);
	NCD4node* ftype = field->basetype;
	void* fdst = INCR(dst,field->meta.offset);
	if((ret=NCD4_fillinstance(meta,ftype,&offset,&fdst,blobs)))
            FAIL(ret,"fillstruct");
    }
    dst = INCR(dst,type->meta.memsize);
    *dstp = dst;
    *offsetp = offset;    
done:
    return THROW(ret);
}

static int
fillseq(NCD4meta* meta, NCD4node* type, void** offsetp, void** dstp, NClist* blobs)
{
    int ret = NC_NOERR;
    d4size_t i,recordcount;    
    void* offset;
    nc_vlen_t* dst;
    NCD4node* vlentype;
    d4size_t recordsize;

    offset = *offsetp;

    dst = (nc_vlen_t*)*dstp;
    vlentype = type->basetype;    
    recordsize = vlentype->meta.memsize;

    /* Get record count (remember, it is already properly swapped) */
    recordcount = GETCOUNTER(offset);
    SKIPCOUNTER(offset);
    dst->len = (size_t)recordcount;

    /* compute the required memory */
    dst->p = d4alloc(recordsize*recordcount);
    if(dst->p == NULL) 
	FAIL(NC_ENOMEM,"fillseq");

    for(i=0;i<recordcount;i++) {
	/* Read each record instance */
	void* recdst = INCR((dst->p),(recordsize * i));
	if((ret=NCD4_fillinstance(meta,vlentype,&offset,&recdst,blobs)))
	    FAIL(ret,"fillseq");
    }
    dst++;
    *dstp = dst;
    *offsetp = offset;
done:
    return THROW(ret);
}

/*
Extract and oob a single string instance
*/
static int
fillstring(NCD4meta* meta, void** offsetp, void** dstp, NClist* blobs)
{
    int ret = NC_NOERR;
    d4size_t count;    
    void* offset = *offsetp;
    char** dst = *dstp;
    char* q;

    /* Get string count (remember, it is already properly swapped) */
    count = GETCOUNTER(offset);
    SKIPCOUNTER(offset);
    /* Transfer out of band */
    q = (char*)d4alloc(count+1);
    if(q == NULL)
	{FAIL(NC_ENOMEM,"out of space");}
    memcpy(q,offset,count);
    q[count] = '\0';
    /* Write the pointer to the string */
    *dst = q;
    dst++;
    *dstp = dst;    
    offset = INCR(offset,count);
    *offsetp = offset;
#if 0
    nclistpush(blobs,q);
#else
    q = NULL;
#endif
done:
    return THROW(ret);
}

static int
fillopfixed(NCD4meta* meta, d4size_t opaquesize, void** offsetp, void** dstp)
{
    int ret = NC_NOERR;
    d4size_t count, actual;
    int delta;
    void* offset = *offsetp;
    void* dst = *dstp;

    /* Get opaque count */
    count = GETCOUNTER(offset);
    SKIPCOUNTER(offset);
    /* verify that it is the correct size */
    actual = count;
    delta = actual - opaquesize;
    if(delta != 0) {
#ifdef FIXEDOPAQUE
	nclog(NCLOGWARN,"opaque changed from %lu to %lu",actual,opaquesize);
	memset(dst,0,opaquesize); /* clear in case we have short case */
	count = (delta < 0 ? actual : opaquesize);
#else
        FAIL(NC_EVARSIZE,"Expected opaque size to be %lld; found %lld",opaquesize,count);
#endif
    }
    /* move */
    memcpy(dst,offset,count);
    dst = INCR(dst,count);
    *dstp = dst;
    offset = INCR(offset,count);
    *offsetp = offset;
#ifndef FIXEDOPAQUE
done:
#endif
    return THROW(ret);
}

/*
Move a dap4 variable length opaque out of band.
We treat as if it was (in cdl) ubyte(*).
*/

static int
fillopvar(NCD4meta* meta, NCD4node* type, void** offsetp, void** dstp, NClist* blobs)
{
    int ret = NC_NOERR;
    d4size_t count;
    nc_vlen_t* vlen;
    void* offset = *offsetp;
    void* dst = *dstp;
    char* q;

    /* alias dst format */
    vlen = (nc_vlen_t*)dst;

    /* Get opaque count */
    count = GETCOUNTER(offset);
    SKIPCOUNTER(offset);
    /* Transfer out of band */
    q = (char*)d4alloc(count);
    if(q == NULL) FAIL(NC_ENOMEM,"out of space");
    memcpy(q,offset,count);
    vlen->p = q;
    vlen->len = (size_t)count;
    q = NULL; /*nclistpush(blobs,q);*/
    dst = INCR(dst,sizeof(nc_vlen_t));
    *dstp = dst;
    offset = INCR(offset,count);
    *offsetp = offset;
done:
    return THROW(ret);
}


/**************************************************/
/* Utilities */
int
NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel)
{
    int ret = NC_NOERR;
    int i;

    if(group == NULL)
	group = meta->root;

    /* Collect vars in this group */
    for(i=0;i<nclistlength(group->vars);i++) {
        NCD4node* node = (NCD4node*)nclistget(group->vars,i);
        nclistpush(toplevel,node);
        node->visited = 0; /* We will set later to indicate written vars */
#ifdef D4DEBUGDATA
fprintf(stderr,"toplevel: var=%s\n",node->name);
#endif
    }
    /* Now, recurse into subgroups; will produce prefix order */
    for(i=0;i<nclistlength(group->groups);i++) {
        NCD4node* g = (NCD4node*)nclistget(group->groups,i);
	if((ret=NCD4_getToplevelVars(meta,g,toplevel))) goto done;
    }
done:
    return THROW(ret);
}