File: download_bigann.py

package info (click to toggle)
hnswlib 0.4.0-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 432 kB
  • sloc: cpp: 2,570; python: 402; makefile: 27; sh: 11
file content (27 lines) | stat: -rw-r--r-- 1,024 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
import os.path
import os

links = ['ftp://ftp.irisa.fr/local/texmex/corpus/bigann_query.bvecs.gz',
         'ftp://ftp.irisa.fr/local/texmex/corpus/bigann_gnd.tar.gz',
         'ftp://ftp.irisa.fr/local/texmex/corpus/bigann_base.bvecs.gz']

os.makedirs('downloads', exist_ok=True)
os.makedirs('bigann', exist_ok=True)
for link in links:
    name = link.rsplit('/', 1)[-1]
    filename = os.path.join('downloads', name)
    if not os.path.isfile(filename):
        print('Downloading: ' + filename)
        try:
            os.system('wget --output-document=' + filename + ' ' + link)
        except Exception as inst:
            print(inst)
            print('  Encountered unknown error. Continuing.')
    else:
        print('Already downloaded: ' + filename)
    if filename.endswith('.tar.gz'):
        command = 'tar -zxf ' + filename + ' --directory bigann'
    else:
        command = 'cat ' + filename + ' | gzip -dc > bigann/' + name.replace(".gz", "")
    print("Unpacking file:", command)
    os.system(command)