include dbnomics_private.inp

function void dbnomics_bundle_print(const bundle b, bool print_data[0])
   if b.error
      printf "dbnomics_bundle_print: bundle has error flag set\n"
      if inbundle(b, "errmsg")
         printf "%s\n", b.errmsg
      endif
      return
   endif

   scalar nf = 0
   string sn
   string dashline

   if inbundle(b, "series_code")
      sn = sprintf("Series: %s", b.series_code)
      nf++
   elif inbundle(b, "dataset_name")
      sn = b.dataset_name
   endif

   if sn != ""
      # make a line of dashes the same length as @sn
      dashline = regsub(sn, ".", "-")
      print dashline
      printf "%s\n", sn
   endif
   if inbundle(b, "provider_code")
      printf "Provider: %s\n", b.provider_code
      nf++
   endif
   if inbundle(b, "dataset_code")
      nf++
      if inbundle(b, "dataset_name")
         printf "Dataset: %s (%s)\n", b.dataset_code, b.dataset_name
      else
         printf "Dataset: %s\n", b.dataset_code
      endif
   endif
   if nf == 3
      printf "Identifier: %s/%s/%s\n", b.provider_code,
        b.dataset_code, b.series_code
   endif
   if inbundle(b, "series_name")
      printf "Name: %s\n", b.series_name
   endif
   if inbundle(b, "dimensions")
      print_series_dimensions(b.dimensions)
   endif
   if dashline != ""
      print dashline
   endif

   if b.has_data
      n = b.T
      n_ok = sumc(ok(b.value))
      printf "pd = %d; %d observations, %s - %s (%d valid)\n",
        b.frequency, n, b.period[1], b.period[n], n_ok
      if print_data
         loop i=1..n -q
            printf "%s %6.1f\n", b.period[i], b.value[i]
         endloop
      endif
   else
      printf "bundle appears to contain no data\n"
   endif
   printf "\n"
end function

function void dbnomics_printer(const bundle *b, int print_data[0::0])
   # wrapper to satisfy the requirements for bundle-print role
   dbnomics_bundle_print(b, print_data)
end function

### ------------------------------------------------------------------------

function void dbnomics_bundles_print(bundles bs)
   aname = argname(bs)
   n = nelem(bs)
   if n > 0
      printf("\n\nContents of %s:\n\n", aname)
      widths = get_widths(bs)
      fmtstring = sprintf("%%-%ds %%-%ds %%-%ds", widths[1], widths[2], widths[3])
      printf("%5s" ~ fmtstring ~ "\n", "", "Provider", "Code", "Description")
      loop i = 1..nelem(bs) --quiet
         printf "%3d: %s\n", i, one_line_bprint(bs[i], widths)
      endloop
   else
      printf("\n\n%s is an empty array\n", aname)
   endif
end function

### ------------------------------------------------------------------------

function scalar dbnomics_bundle_get_data(const bundle b, series *x, bool verbose[0])
   if !inbundle(b, "frequency")
      printf "dbnomics_bundle_get_data: bundle does not contain a data frequency\n"
      return 1
   endif
   pd = b.frequency
   daily = pd == 7
   use_join = daily

   if daily # daily data need special treatment
      match = maxc($pd .= {5;6;7})
   else
      match = (pd == $pd)
   endif

   if !match
      # FIXME: maybe allow join to handle this?
      funcerr "frequency does not match!"
   endif

   if $version >= 20221 # 2022b or higher
      string tmpfile = "tmpXXXXXX.csv"
      outfile --tempfile=tmpfile
         printf "obs\tdbseries\n"
         set warnings off
         loop i=1 .. b.T -q
            printf "%s\t%.15g\n", b.period[i], b.value[i]
         endloop
      end outfile
   else
      string tmpfile = sprintf("%s/tmp%06d.csv", $dotdir, randgen1(i, 0, 999999))
      outfile @tmpfile --quiet
         printf "obs\tdbseries\n"
         set warnings off
         loop i=1 .. b.T -q
            printf "%s\t%.15g\n", b.period[i], b.value[i]
         endloop
      end outfile
   endif

   if daily
      setopt join --tkey="obs,%Y-%m-%d"
   elif instring(b.period[1], "-Q")
      # quarterly
      setopt join --tkey="obs,%Y-Q%q"
   endif

   string tracestr
   outfile --buffer=tracestr
      if !use_join
         # try "append" first
         print "trying append..."
         catch append @tmpfile --quiet --fixed-sample
         err = $error
         if !err
            tool = "append"
            x = dbseries
         else
            # "append" may fail because data are not ordered properly
            # if that's the case, "join" may still work
            print "append failed, trying join..."
            use_join = 1
         endif
      endif
      if use_join
         catch join @tmpfile x --data="dbseries"
         err = $error
         if !err
            tool = "join"
         else
            series dbseries = NA
         endif
      endif
   end outfile

   remove(tmpfile)

   if inbundle(b, "series_name")
      d = b.series_name
      setinfo x --description="@d"
   endif

   if verbose
      printf "dbnomics_bundle_get_data:\n%s\n", b.series_name
      printf "tool = %s\n", tool
      printf "error = %d\n", err
      printf "trace:\n%s\n\n", tracestr
   endif

   return 0
end function

### ------------------------------------------------------------------------

function bundle dbnomics_get_series (const string datacode, bool verbose[0])

   bundle ret
   string json
   ret.error = 0
   ret.has_data = 0
   string parms = "series?series_ids=" ~ datacode ~ "&format=json&observations=1&metadata=0"
   ret.error = dbn_get_json(parms, verbose, &json)
   if ret.error
      return ret
   endif

   catch bundle b = jsongetb(json, "series")
   if $error
      ret.error = 1
   elif inbundle(b, "series") == 0
      ret.error = 1
   elif nelem(b.series.docs) == 0
      ret.error = 1
   endif
   if !ret.error
      bundles bb = b.series.docs
      ret = bb[1]
      process_series_bundle(json, &ret)
   endif
   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_get_cart(const string URL)
   string json
   bundles ret = null
   string msg = ""
   err = dbn_get_json_from_URL(URL, &json, &msg)
   if err
      printf "%s\n", msg
   else
      ret = multiple_from_json(json, 0)
   endif
   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_get_multiple(const string provider,
                                       const string dset, int limit[0::50], int offset[0],
                                       bundle spec[null])

   if !exists(spec)
      bundle spec = null
   endif
   bundles ret = array(0)
   string parms = build_parms(provider, dset, limit, offset, spec)
   string json
   err = dbn_get_json(parms, 0, &json)
   if err
      printf "dbnomics_get_multiple: got error from dbn_get_json\n"
   else
      ret = multiple_from_json(json, 0)
   endif
   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_get_dataset_dimensions(const string provider,
                                                 const string dset, bool verbose[0])

   parms = sprintf("series/%s/%s?limit=0&offset=0&facets=1", provider, dset)
   string json
   err = dbn_get_json(parms, verbose, &json)

   if !err
      bundle b = jsongetb(json, "dataset")
      bundle bf = jsongetb(json, "series_dimensions_facets")
      b = b.dataset
      bf = bf.series_dimensions_facets
      strings dcodes = b.dimensions_codes_order
      ndims = nelem(dcodes)
      if inbundle(b, "dimensions_labels")
         bundle dlabels = b.dimensions_labels
      else
         strings dlabels = b.dimensions_codes_order
      endif
      bundle vlabels = b.dimensions_values_labels
   else
      ndims = 0
   endif

   bundles ret = array(ndims)
   # loop over the dimensions
   loop i=1..ndims --quiet
      string dci = dcodes[i]
      ret[i].code = dci
      if typeof(dlabels) == 5
         ret[i].label = dlabels[dci]
      else
         ret[i].label = dlabels[i]
      endif
      bundles bfi = bf[dci]
      ret[i].values = bfi
      nv = nelem(bfi)
      ret[i].nvalues = nv
      # loop over the values in each dimension
      if inbundle(vlabels, dci) == 5 # bundle
         bundle btmp = vlabels[dci]
         loop j=1..nv --quiet
            bundle bvj = ret[i].values[j]
            ret[i].values[j].label = btmp[bvj.code]
         endloop
      elif inbundle(vlabels, dci) == 6 # array
         stmp = vlabels[dci]
         loop j=1..nv --quiet
            ret[i].values[j].label = stmp[j][2]
         endloop
      endif
   endloop
   if verbose && !err
      loop i=1..ndims --quiet
         r = ret[i]
         nv = r.nvalues
         printf "%s (%s); %d elements\n", r.code, r.label, nv
         loop j=1..nv --quiet
            bundle vj = r.values[j]
            printf "\t%s (%s, count %d)\n", vj.code, vj.label, vj.count
         endloop
         printf("\n")
      endloop
   endif

   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_get_dataset_content(const string provider,
                                              const string dset,
                                              int limit[0::100],
                                              int offset[0])
   bundles ret = null
   string json
   parms = sprintf("series/%s/%s?limit=%d&offset=%d", provider,
     dset, limit, offset)
   err = dbn_get_json(parms, 0, &json)
   if err
      return ret
   endif

   n = 0
   ns = 0
   # note: the following will give the total for the dataset
   string found = jsonget(json, "$.series.num_found", &n)
   if n == 0
      msg = get_dbnomics_message(json, "No content was found")
      funcerr msg
   endif
   if n > 0
      nfound = atof(found)
      ns = nfound - offset
      ns = ns > limit ? limit : ns
   endif
   if ns > 0
      bundles ret = array(ns)
      strings fields = defarray("series_name", "series_code")
      loop i = 1..ns --quiet
         ret[i] = json2bndl(fields, json, "series.docs", i-1)
         brename(ret[i], "series_code", "code")
         if inbundle(ret[i], "series_name")
            brename(ret[i], "series_name", "name")
         else
            ret[i].name = ret[i].code
         endif
         ret[i].num_found = nfound
      endloop
   endif

   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_providers(bool verbose[0])
   magic = "providers.docs"
   nprov = 0
   string json
   err = dbn_get_json("providers", verbose, &json)

   if !err
      str = jsonget(json, sprintf("$.%s[*].code", magic), &nprov)
      if nprov == 0
         err = 1
      endif
   endif
   if err
      funcerr "no providers were found"
   endif

   bundles ret = array(nprov)
   strings fields = defarray("name", "terms_of_use", "code", "region", "description", "website")

   loop i = 1..nprov --quiet
      ret[i] = json2bndl(fields, json, magic, i-1)
   endloop

   if verbose
      printf "Available providers: \n"
      loop i = 1..nprov --quiet
         printf "%3d: %15s", i, ret[i].code
         if inbundle(ret[i], "name")
            printf " (%s)", ret[i].name
         endif
         printf "\n"
      endloop
   endif

   return ret
end function

### ------------------------------------------------------------------------

function bundles dbnomics_search(const string key, const string dset[null],
                                 int limit[0::100], int offset[0],
                                 bool verbose[0])

   bundles ret = null

   if exists(dset) && dset != ""
      # make search specific to a dataset
      strings codes = strsplit(dset, "/")
      if nelem(codes) < 2
         funcerr "Invalid dataset code (should be provider/dataset)"
      else
         ret = dset_search(key, codes[1], codes[2],
           limit, offset, verbose)
      endif
   else
      # search all providers
      ret = general_search(key, limit, offset, verbose)
   endif

   return ret
end function

### ------------------------------------------------------------------------

function bundle dbnomics_category_tree(const string provider,
                                       bool verbose[0])
   bundle ret
   string json
   ret.error = dbn_get_json("providers/" ~ provider, verbose, &json)
   if !ret.error
      ret = jsongetb(json, "category_tree")
   endif
   return ret
end function

### ------------------------------------------------------------------------

# Note: the "category_tree" approach is a relatively lightweight
# and reliable way of getting a simple listing of the datasets
# available from a given provider. Cf. the datasets/{provider}
# query in the v22 API, which apparently has a default limit of
# 10 datasets.

function bundle dbnomics_dsets_for_provider(const string provider,
                                            bool verbose[0])

   bundle ret
   string json
   ret.error = dbn_get_json("providers/" ~ provider, verbose, &json)

   if !ret.error
      bundle b = jsongetb(json, "category_tree")
      bundles B = _jbterms(b)
      n = nelem(B)
      if n == 0
         msg = sprintf("Found no datasets for provider %s", provider)
         funcerr msg
      endif
      strings dscodes = array(n)
      strings dsnames = array(n)
      n_ok = 0
      j = 1
      loop i=1..n -q
         if inbundle(B[i], "code")
            dscodes[j] = B[i].code
            if inbundle(B[i], "name")
               dsnames[j] = B[i].name
            else
               dsnames[j] = "no description"
            endif
            j++
            n_ok++
         endif
      endloop
      if n_ok == 0
         printf("Found no datasets for provider %s", provider)
      elif n_ok < n
         dscodes = dscodes[1:n_ok]
         dsnames = dsnames[1:n_ok]
      endif
      ret.error = 0
      ret.codes = dscodes
      ret.names = dsnames
   endif

   return ret
end function


### ------------------------------------------------------------------------

function series dbnomics_fetch (const string id, bool verbose[0])
   series ret = NA
   bundle b = dbnomics_get_series(id, verbose)
   if b.error
      msg = sprintf("%s: couldn't fetch", id)
      funcerr msg
   endif
   dbnomics_bundle_get_data(b, &ret, verbose)
   return ret
end function

### ------------------------------------------------------------------------

function list dbnomics_bundles_to_list(bundles bs, string key[null])
   n = nelem(bs)
   list RET = null
   given_names = 0
   if exists(key)
      given_names = strlen(key) > 0
   endif

   digits = floor(log10(n)) + 1
   fmt = "series_" ~ sprintf("%%0%dd", digits)

   loop i = 1 .. n --quiet
      b = bs[i]
      if b.has_data && b.frequency == $pd
         valid_name = 0
         if given_names
            if inbundle(b, key)
               val = b[key]
               if typeof(val) == 4
                  valid_name = 1
               endif
            endif
         endif
         sname = valid_name ? fixname(val) : sprintf(fmt, i)
         series tmp = NA
         err = dbnomics_bundle_get_data(b, &tmp)
         if (!err)
            RET += genseries(sname, tmp)
            desc = b.series_name
            setinfo @sname --description="@desc"
         endif
      endif
   endloop
   return RET
end function