function bundle json2bndl(const strings fields, const string jsonobj,
                          const string root, int ndx)
    bundle b = null
    k = 0
    loop j = 1..nelem(fields) --quiet
        f = fields[j]
        src = sprintf("$.%s[%d].%s", root, ndx, f)
        str = jsonget(jsonobj, src, &k)
        if k
            if f == "name"
                # hopefully this can be trashed later
                b[f] = strsub(str, ",", ",")
            else
                b[f] = str
            endif
        endif
    endloop
    return b
end function

### ------------------------------------------------------------------------

function void print_series_dimensions (const bundle dims)
    strings keys = getkeys(dims)
    if nelem(keys) == 0
        return
    endif
    di = dims[keys[1]]
    if typeof(di) == 5 # bundle
        printf "Dimensions, showing JSON mask form:\n"
        loop i=1..nelem(keys) -q
            di = dims[keys[i]]
            printf "  %s: %s, {\"%s\":[\"%s\"]}\n", di.dimension_label, di.value_label,
              keys[i], di.value
        endloop
    elif typeof(di) == 4 # string
        printf "Dimensions in JSON mask form:\n"
        loop i=1..nelem(keys) -q
            di = dims[keys[i]]
            printf "  {\"%s\":[\"%s\"]}\n", keys[i], di
        endloop
    endif
end function

### ------------------------------------------------------------------------

function void fix_dimensions_bundle (const string J, bundle *sd)
    bundle dinfo = jsongetb(J, "datasets")
    scalar dsmulti = 0
    if nelem(dinfo) > 0
        dsmulti = 1
    else
        # try for the singular?
        dinfo = jsongetb(J, "dataset")
    endif
    if nelem(dinfo) == 0
        return
    endif
    dinfo = dsmulti ? dinfo.datasets : dinfo.dataset
    dsid = sprintf("%s/%s", sd.provider_code, sd.dataset_code)
    # get the 'raw' dimensions bundle
    bundle d0 = sd.dimensions
    strings keys = getkeys(d0)
    ndims = nelem(keys)
    bundle dlb = null
    strings dla = null
    if dsmulti
        catch dld = dinfo[dsid].dimensions_labels
        if $error
            catch dla = dinfo[dsid].dimensions_codes_order
        endif
        dvl = dinfo[dsid].dimensions_values_labels
    else
        catch dld = dinfo.dimensions_labels
        if $error
            catch dla = dinfo.dimensions_codes_order
        endif
        dvl = dinfo.dimensions_values_labels
    endif

    # represent the dimensions as a bundle of bundles
    bundle dims
    loop i=1..ndims -q
        string key = keys[i]
        string value = d0[key]
        bundle b = null
        if nelem(dlb) > 0
            b.dimension_label = dlb[key]
        elif nelem(dla) > 0
            b.dimension_label = dla[i]
        else
            b.dimension_label = key
        endif
        b.value = value
        if typeof(dvl[key]) == 6 # array
            got_vlabel = 0
            loop j=1..nelem(dvl[key]) -q
                if typeof(dvl[key][j]) == 5 # bundle
                    bundle bsub = dvl[key][j]
                    if inbundle(bsub, value)
                        b.value_label = bsub[value]
                        got_vlabel = 1
                        break
                    endif
                elif typeof(dvl[key][j]) == 6 # array
                    strings asub = dvl[key][j]
                    if asub[1] == value
                        b.value_label = asub[2]
                        got_vlabel = 1
                        break
                    endif
                endif
            endloop
            if !got_vlabel
                b.value_label = "<unknown value label>"
            endif
        else
            # otherwise dvl[key] must be a bundle
            b.value_label = dvl[key][value]
        endif
        dims[key] = b
        if key == "REF_AREA"
            # add this key at top level
            sd["Reference Area"] = b.value_label
        endif
    endloop

    # replace the original dimensions bundle
    sd.dimensions = dims
end function

### ------------------------------------------------------------------------

function void process_series_bundle (string json, bundle *b)
    set warnings off
    b.error = 0
    b.frequency = bundle_get_frequency(b)
    t1 = 1
    t2 = nelem(b.period)
    if typeof(b.value) == 6 # array
        # trim leading NAs
        loop j=1..t2 -q
            if b.value[j] == "" || b.value[j] == "NA"
                t1++
            else
                break
            endif
        endloop
        # trim trailing NAs
        loop for (j=t2; j>=t1; j--) -q
            if b.value[j] == "" || b.value[j] == "NA"
                t2--
            else
                break
            endif
        endloop
    endif
    T = t2 - t1 + 1
    b.T = T
    b.has_data = T > 0
    if T > 0
        if t1 > 1
            b.period = b.period[t1:t2]
        endif
        havemat = typeof(b.value) == 3
        matrix vals = zeros(T, 1)
        j = 1
        loop t=t1..t2 -q
            vals[j] = havemat ? b.value[t] : atof(b.value[t])
            j++
        endloop
        b.value = vals
    endif
    # if inbundle(b, "dimensions")
    #    fix_dimensions_bundle(json, &b)
    # endif
end function

### ------------------------------------------------------------------------

function string sanitize_url(const string s)
    string dqu = sprintf("\"")
    ret = strsub(s, " ", "%20")
    ret = strsub(ret, dqu, "%22")
    ret = strsub(ret, ",", "%2C")
    ret = strsub(ret, ":", "%3A")
    ret = strsub(ret, "{", "%7B")
    ret = strsub(ret, "}", "%7D")
    ret = strsub(ret, "[", "%5B")
    ret = strsub(ret, "]", "%5D")
    return ret
end function

function string desanitize_url(const string s)
    string dqu = sprintf("\"")
    ret = strsub(s, "%20", " ")
    ret = strsub(ret, "%22", dqu)
    ret = strsub(ret, "%2C", ",")
    ret = strsub(ret, "%3A", ":")
    ret = strsub(ret, "%7B", "{")
    ret = strsub(ret, "%7D", "}")
    ret = strsub(ret, "%5B", "[")
    ret = strsub(ret, "%5D", "]")
    return ret
end function

### ------------------------------------------------------------------------

function scalar bundle_get_frequency (const bundle b)
    set warnings off
    strings alts = defarray("Frequency", "FREQ", "@frequency")
    string frq = ""
    loop i=1..nelem(alts) -q
        if inbundle(b, alts[i])
            frq = tolower(b[alts[i]][1])
            break
        endif
    endloop

    scalar ret
    if frq == ""
        ret = NA
    elif frq == "a"
        ret = 1
    elif frq == "q"
        ret = 4
    elif frq == "m"
        ret = 12
    elif frq == "d"
        ret = 7
    else
        printf "frq = \"%s\" unrecognized\n", frq
        ret = -1
    endif

    return ret
end function

### ------------------------------------------------------------------------

function string make_dimensions(const bundle spec, const string what)
    ret = ""
    if inbundle(spec, what)
        strings ss = spec[what]
        ret ~= sprintf("\"%s\":[", what)
        loop i = 1 .. nelem(ss) --quiet
            ret ~= sprintf("\"%s\"", ss[i])
            if i < nelem(ss)
                ret ~= ","
            else
                ret ~= "]"
            endif
        endloop
    endif
    return ret
end function

### ------------------------------------------------------------------------

function string build_URL_mid(const bundle spec)
    string ret = ""
    if nelem(spec) == 0
        return ret
    endif
    ret = "dimensions={"
    strings filters = array()
    strings dims = getkeys(spec) # eg, "freq", "indicator", "geo"

    n = 0
    loop i = 1..nelem(dims) --quiet
        s = make_dimensions(spec, dims[i])
        if strlen(s) > 0
            n++
            filters = filters + defarray(s)
        endif
    endloop

    loop i = 1..n --quiet
        ret ~= filters[i]
        if i < n
            ret ~= ", "
        endif
    endloop

    ret = ret ~ "}&"

    return ret
end function

### ------------------------------------------------------------------------

function string build_parms(const string pcode, const string dcode,
                            int limit[0::20], int offset[0],
                            bundle spec)
    string head
    if inbundle(spec, "mask")
        string mask = sanitize_url(spec.mask)
        head = sprintf("series/%s/%s/%s?", pcode, dcode, mask)
    else
        head = sprintf("series/%s/%s?", pcode, dcode)
        string mid = build_URL_mid(spec)
        head ~= sanitize_url(mid)
    endif
    string tail = sprintf("limit=%d&offset=%d&format=json&observations=1&metadata=0", limit, offset)
    return head ~ tail
end function

### ------------------------------------------------------------------------

# This is tricky: we may not get a recognizable error code
# or even an error message on failure to retrieve any data.

function scalar check_json (const string s, string *msg)
    ret = 0
    if instring(s, "404 Error") || instring(s, "404 Not Found")
        ret = 404
    elif instring(s, "400 Bad Request")
        ret = 400
    elif instring(s,"Internal server error")
        msg = "Internal server error"
        ret = 1
    else
        bundle eb = jsongetb(s, "errors")
        if inbundle(eb, "errors")
            # do we have a (non-empty) @errors array?
            if typeof(eb.errors) == 6 && nelem(eb.errors) > 0
                if inbundle(eb.errors[1], "cause")
                    msg = eb.errors[1].cause
                elif inbundle(eb.errors[1], "message")
                    msg = eb.errors[1].message
                endif
                ret = 1
            endif
            if ret && instring(msg, "Could not")
                # back off to warning, as with v21 API?
                ret = 0
            endif
        endif
    endif
    # warnings with v22 ??
    return ret
end function

### ------------------------------------------------------------------------

function string get_dbnomics_message (const string json,
                                      const string deflt)
    scalar m = 0
    string msg = jsonget(json, "$.message", &m)
    if m > 0
        return "dbnomics says: " ~ msg
    else
        return deflt
    endif
end function

### ------------------------------------------------------------------------

function scalar dbn_get_json_from_URL (const string URL, string *json, string *msg)

    bundle req = defbundle("URL", URL, "header", "accept: application/json")
    # check for curl error
    scalar error = curl(&req)
    if error
        msg = req.errmsg
    else
        # also check for dbnomics error condition
        json = req.output
        dbg = ngetenv("DBNOMICS_DEBUG")
        if ok(dbg) && dbg > 0
            printf "%s\n", desanitize_url(URL)
            outfile dbn_debug.json --quiet
                json
            end outfile
        endif
        string dbnmsg
        error = check_json(json, &dbnmsg)
        if dbnmsg != ""
            msg = sprintf("dbnomics says:\n\"%s\"\n", dbnmsg)
        endif
    endif

    return error
end function

### ------------------------------------------------------------------------

function scalar dbn_get_json (const string parms, bool verbose[0],
                              string *json)

    string URL = "https://api.db.nomics.world/v22/" ~ parms
    if verbose
        printf "URL: %s\n\n", URL
    endif
    string msg
    error = dbn_get_json_from_URL(URL, &json, &msg)
    if error
        if msg == ""
            msg = "Failed to get the requested information from dbnomics"
        endif
        funcerr msg
    elif msg != ""
        # got a warning but not an actual error
        print msg
    endif

    return error
end function

### ------------------------------------------------------------------------

function bundles multiple_from_json (const string json, bool verbose[0])

    bundles ret = array(0)
    bundle b = jsongetb(json, "series")
    b = b.series
    if inbundle(b, "docs")
        ret = b.docs
    endif
    loop i=1..nelem(ret) -q
        process_series_bundle(json, &ret[i])
    endloop
    return ret

end function

### ------------------------------------------------------------------------

function bundles general_search(const string key, int limit[0::100],
                                int offset[0], bool verbose[0])

    magic = "results.docs"
    bundles ret = array(0)

    string json
    parms = sprintf("search?q=%s&limit=%d&offset=%d", sanitize_url(key),
      limit, offset)
    err = dbn_get_json(parms, verbose, &json)
    if err
        return ret
    endif

    n = 0
    str = jsonget(json, sprintf("$.%s[*].code", magic), &n)
    ntot = atof(jsonget(json, "$.results.num_found"))
    # fix possibly nonsensical result
    if ntot < n
        ntot = n
    endif

    #printf "n = %d, ntot = %d\n", n, ntot

    if n == 0
        if verbose
            printf "No matches found"
        endif
        return ret
    endif

    bundles ret = array(n)
    strings fields = defarray("code", "nb_matching_series", "name", \
      "provider_code")
    loop i = 1..n --quiet
        ret[i] = json2bndl(fields, json, magic, i-1)
        ret[i].ntot = ntot
    endloop

    if verbose
        printf "Datasets containing \"%s\" (%d-%d out of %d): \n\n",
          key, offset+1, offset+n, ntot
        loop i = 1..n --quiet
            printf "%3d: %s.%s", i, ret[i].provider_code, ret[i].code
            printf " (%d series)\n", atof(ret[i].nb_matching_series)
        endloop
    endif

    return ret
end function

### ------------------------------------------------------------------------

function bundles dset_search(const string key, const string pcode,
                             const string dcode, int limit[0::100],
                             int offset[0], bool verbose[0])

    bundles ret = array(0)
    scalar got = 0
    string json
    parms = sprintf("series/%s/%s?q=%s&limit=%d&offset=%d", pcode, dcode,
      sanitize_url(key),limit, offset)

    err = dbn_get_json(parms, verbose, &json)

    if !err
        bundle bj = jsongetb(json, "series")
        scalar ntot = bj["series"].num_found
        if ntot > 0
            ret = bj["series"].docs
            got = nelem(ret)
            if ntot < got
                # just in case of breakage
                ntot = got
            endif
        endif
        if got > 0
            loop i = 1..got --quiet
                ret[i].ntot = ntot
                brename(ret[i], "series_code", "code")
                brename(ret[i], "series_name", "name")
            endloop
        endif
        if verbose
            if got > 0
                printf "Dataset %s/%s, matching series %d-%d of %d:\n\n",
                  pcode, dcode, offset+1, offset+got, ntot
                loop i = 1..got --quiet
                    printf "%s: %s\n", ret[i].code, ret[i].name
                endloop
            else
                printf "no matching series found\n"
            endif
        endif
    endif

    return ret
end function

### ------------------------------------------------------------------------

function matrix get_widths(bundles bs)
    w1 = 6
    w2 = 12
    w3 = 24
    loop i = 1..nelem(bs) --quiet
        l = strlen(bs[i].provider_code) + strlen(bs[i].dataset_code) + 1
        w1 = xmax(w1, l)
        l = strlen(bs[i].series_code)
        w2 = xmax(w2, l)
        l = strlen(bs[i].series_name)
        w3 = xmin(xmax(w3, l), 40)
    endloop
    return {w1, w2, w3} + 2
end function

### ------------------------------------------------------------------------

function string one_line_bprint(bundle b, matrix widths)
    provider = b.provider_code ~ "/" ~ b.dataset_code
    scode = fixname(b.series_code, 1)
    sname = b.series_name
    if strlen(sname) > 40
        sname = sname[1:37] ~ "..."
    endif
    fmtstring = sprintf("%%-%ds %%-%ds %%-%ds", widths[1], widths[2], widths[3])
    ret = sprintf(fmtstring, provider, scode, sname)

    if b.has_data
        scalar T = b.T
        print T
        string span = sprintf("%s:%s", b.period[1], b.period[T])
        print span
        ret ~= sprintf(" %d observations (pd = %d) [%s]", T, b.frequency, span)
    else
        ret ~= " has no data"
    endif

    return ret
end function