File: upload-s3.py

package info (click to toggle)
duckdb 1.5.1-2
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 299,196 kB
  • sloc: cpp: 865,414; ansic: 57,292; python: 18,871; sql: 12,663; lisp: 11,751; yacc: 7,412; lex: 1,682; sh: 747; makefile: 558
file content (65 lines) | stat: -rw-r--r-- 1,444 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import subprocess
import sys
import os

if len(sys.argv) < 3:
    print("Usage: [prefix] [filename1] [filename2] ... ")
    exit(1)


def git_rev_hash():
    return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode("utf-8").strip()


prefix = sys.argv[1].strip()

# Hannes controls this web server
# Files are served at https://download.duckdb.org/...

secret_key = os.getenv('DAV_PASSWORD')
if secret_key is None:
    print("Can't find DAV_PASSWORD in env ")
    exit(2)


files = sys.argv[2:]
basenames = [os.path.basename(x) for x in files]

basenames_set = set(basenames)
if len(basenames) != len(basenames_set):
    print("Name conflict")
    exit(3)


git_hash = git_rev_hash()

folder = 'rev/%s/%s' % (git_hash, prefix)


def curlcmd(cmd, path):
    p = subprocess.Popen(
        ['curl', '--retry', '10']
        + cmd
        + ['http://duckdb:%s@dav10635776.mywebdav.de/duckdb-download/%s' % (secret_key, path)],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    output, err = p.communicate()
    rc = p.returncode

    if p.returncode != 0:
        print(err)
        exit(4)


# create dirs, no recursive create supported by webdav
f_p = ''
for p in folder.split('/'):
    f_p += p + '/'
    curlcmd(['-X', 'MKCOL'], f_p)

for f in files:
    base = os.path.basename(f)
    key = '%s/%s' % (folder, base)
    print("%s\t->\thttps://download.duckdb.org/%s " % (f, key))
    curlcmd(['-T', f], key)