File: d4chunk.c

package info (click to toggle)
netcdf 1:4.7.4-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 104,952 kB
  • sloc: ansic: 228,683; sh: 10,980; yacc: 2,561; makefile: 1,319; lex: 1,173; xml: 173; awk: 2
file content (219 lines) | stat: -rw-r--r-- 7,534 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
/*********************************************************************
 *   Copyright 2018, UCAR/Unidata
 *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
 *********************************************************************/

#include "d4includes.h"
#include "d4chunk.h"

/**************************************************/

/* Header flags */
#define LAST_CHUNK          (1)
#define ERR_CHUNK           (2)
#define LITTLE_ENDIAN_CHUNK (4)
#define NOCHECKSUM_CHUNK    (8)

#define ALL_CHUNK_FLAGS (LAST_CHUNK|ERR_CHUNK|LITTLE_ENDIAN_CHUNK|NOCHECKSUM_CHUNK)

/**************************************************/

/*
Given a packet as read from the wire via http (or a file), convert in
place from chunked format to a single contiguous set of bytes. If an
error packet is recovered, then make that available to the caller and
return an error. Also return whether the data was big endian encoded
and whether it has checksums.
Notes:
*/

/* Define a local struct for convenience */
struct HDR {unsigned int flags; unsigned int count;};

/* Forward */
static void* getheader(void* p, struct HDR* hdr, int hostlittleendian);
static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count);

/**************************************************/

int
NCD4_dechunk(NCD4meta* metadata)
{
    unsigned char* p;
    unsigned char* q;
    struct HDR hdr;

    if(metadata->mode == NCD4_DSR) 
        return THROW(NC_EDMR);

    metadata->serial.errdata = NULL;
    metadata->serial.dmr = NULL;
    metadata->serial.dap = NULL;
    metadata->serial.hostlittleendian = NCD4_isLittleEndian();
    metadata->serial.remotelittleendian = 0; /* do not actually know yet */
    metadata->serial.remotechecksumming = 0; /* do not actually know yet */
    metadata->localchecksumming = 0; /* do not actually know yet */

    /* Assume proper mode has been inferred already. */

    /* Verify the mode; assume that the <?xml...?> is optional */
    q = metadata->serial.rawdata;
    if(memcmp(q,"<?xml",strlen("<?xml"))==0
       || memcmp(q,"<Dataset",strlen("<Dataset"))==0) {
        if(metadata->mode != NCD4_DMR) 
            return THROW(NC_EDMR);
        /* setup as dmr only */
        metadata->serial.dmr = (char*)metadata->serial.rawdata; /* temp */
        /* Avoid strdup since rawdata might contain nul chars */
        if((metadata->serial.dmr = malloc(metadata->serial.rawsize+1)) == NULL)
            return THROW(NC_ENOMEM);    
        memcpy(metadata->serial.dmr,metadata->serial.rawdata,metadata->serial.rawsize);
        metadata->serial.dmr[metadata->serial.rawsize-1] = '\0';
        /* Suppress nuls */
        (void)NCD4_elidenuls(metadata->serial.dmr,metadata->serial.rawsize);
        return THROW(NC_NOERR); 
    }

    /* We must be processing a DAP mode packet */
    p = metadata->serial.rawdata;
    metadata->serial.dap = p;

#ifdef D4DUMPRAW
    NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW");
#endif

    /* Get the DMR chunk header*/
    p = getheader(p,&hdr,metadata->serial.hostlittleendian);
    if(hdr.count == 0)
        return THROW(NC_EDMR);
    if(hdr.flags & ERR_CHUNK) {
        return processerrchunk(metadata, (void*)p, hdr.count);
    }

    metadata->serial.remotechecksumming = ((hdr.flags & NOCHECKSUM_CHUNK) ? 0 : 1);
    metadata->localchecksumming = metadata->serial.remotechecksumming;

    metadata->serial.remotelittleendian = ((hdr.flags & LITTLE_ENDIAN_CHUNK) ? 1 : 0);
    /* Again, avoid strxxx operations on dmr */
    if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL)
        return THROW(NC_ENOMEM);        
    memcpy(metadata->serial.dmr,p,hdr.count);
    metadata->serial.dmr[hdr.count-1] = '\0';
    /* Suppress nuls */
    (void)NCD4_elidenuls(metadata->serial.dmr,hdr.count);

    if(hdr.flags & LAST_CHUNK)
        return THROW(NC_ENODATA);
    /* Read and compress the data chunks */
    p = p + hdr.count; /* point to data chunk header */
    /* Do a sanity check in case the server has shorted us with no data */
    if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) {
        /* Server only sent the DMR part */
        metadata->serial.dapsize = 0;
        return THROW(NC_EDATADDS);
    }
    q = metadata->serial.dap; 
    for(;;) {
        p = getheader(p,&hdr,metadata->serial.hostlittleendian);
        if(hdr.flags & ERR_CHUNK) {
            return processerrchunk(metadata, (void*)p, hdr.count);
        }
        /* data chunk; possibly last; possibly empty */
        if(hdr.count > 0) {
            d4memmove(q,p,hdr.count); /* will overwrite the header */
            p += hdr.count;
            q += hdr.count;
        }
        if(hdr.flags & LAST_CHUNK) break;
    }
    metadata->serial.dapsize = (size_t)DELTA(q,metadata->serial.dap);

#ifdef D4DUMPDMR
    fprintf(stderr,"%s\n",metadata->serial.dmr);
    fflush(stderr);
#endif
#ifdef D4DUMPDAP
    NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP");
#endif
    return THROW(NC_NOERR);    
}

static int
processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count)
{
    metadata->serial.errdata = (char*)d4alloc(count+1);
    if(metadata->serial.errdata == NULL)
        return THROW(NC_ENOMEM);
    memcpy(metadata->serial.errdata,errchunk,count);
    metadata->serial.errdata[count] = '\0';
    return THROW(NC_ENODATA); /* slight lie */
}

/* At the moment, the Hyrax test server
       is serving up the chunk data as little endian.
       So use a heuristic to see which endianness
       makes the more sense.
   This fails for very small dap data sections.
*/
static void*
getheader(void* p, struct HDR* hdr, int hostlittleendian)
{
    unsigned char bytes[4];
#ifdef HYRAXHACK
    struct HDR hyrax;
    unsigned char orig[4];
    memcpy(orig,p,sizeof(bytes));/* save a copy */
#endif
    memcpy(bytes,p,sizeof(bytes));
    p = INCR(p,4); /* on-the-wire hdr is 4 bytes */
    /* assume header is network (big) order */
    hdr->flags = bytes[0]; /* big endian => flags are in byte 0 */
    bytes[0] = 0; /* so we can do byte swap to get count */
    if(hostlittleendian)
        swapinline32(bytes); /* host is little endian */
    hdr->count = *(unsigned int*)bytes; /* get count */
#ifdef HYRAXHACK
    memcpy(bytes,orig,sizeof(bytes)); /* restore */
    hyrax.flags = bytes[3];
    bytes[3] = 0; /* so we can do byte swap to get count */
    if(!hostlittleendian)
        swapinline32(bytes); /* host is big endian */
    hyrax.count = *(unsigned int*)bytes; /* get count */
    /* See which makes more sense */
    if(hyrax.flags <= ALL_CHUNK_FLAGS && hyrax.count >= 0 && hyrax.count < hdr->count) {
        /* Use hyrax version */
        *hdr = hyrax;   
    }
#endif
    return p;
}

/**
Given a raw response, attempt to infer the mode: DMR, DAP, DSR.
Since DSR is not standardizes, it becomes the default.
*/
int
NCD4_infermode(NCD4meta* meta)
{
    d4size_t size = meta->serial.rawsize;
    char* raw = meta->serial.rawdata;

    if(size < 16)
        return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ 
    if(memcmp(raw,"<?xml",strlen("<?xml"))==0
       || memcmp(raw,"<Dataset",strlen("<Dataset"))==0) {
        meta->mode = NCD4_DMR;
        goto done;
    }
    raw += 4; /* Pretend we have a DAP hdr */
    if(memcmp(raw,"<?xml",strlen("<?xml"))==0
       || memcmp(raw,"<Dataset",strlen("<Dataset"))==0) {
        meta->mode = NCD4_DAP;
        goto done;
    }
    /* Default to DSR */
    meta->mode = NCD4_DSR;

done:
    return NC_NOERR;
}