1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
#!/usr/bin/env python
from __future__ import print_function
description = """
Upload all the ExternalData files to the Midas server.
The files corresponding to all the ExternalData content links
in the source tree are archived on the Midas server using the
local ExternalData object store.
Requires pydas: https://github.com/midasplatform/pydas
"""
import argparse
import fnmatch
import os
import sys
import pydas
def connect_to_midas(email=None, api_key=None, midas_url='https://midas3.kitware.com/midas/'):
if not api_key:
print('Please enter your login information for ' + midas_url)
pydas.login(url=midas_url, email=email)
else:
pydas.login(url=midas_url, email=email, api_key=api_key)
session = pydas.session
communicator = session.communicator
return session, communicator
def upload_to_midas(content_link, externaldata_object_store,
repository_dir,
session, communicator):
# get the MD5 checksum
print('Uploading ' + content_link + ' ...')
with open(content_link, 'r') as fp:
md5hash = fp.readline().strip()
print('Checksum: ' + md5hash)
# upload to Midas
def get_child_folder(parent, child_name):
children = communicator.folder_children(session.token, parent['folder_id'])
for folder in children['folders']:
if folder['name'] == child_name:
return folder
return None
itk_community = communicator.get_community_by_name('ITK')
itk_public = get_child_folder(itk_community, 'Public')
itk_folder = get_child_folder(itk_public, 'ITK')
# Where to place the file in Midas. Mirror the source tree location.
folders = os.path.dirname(content_link)
if repository_dir != '':
folders = folders[folders.find(repository_dir)+len(repository_dir)+1:]
folders = folders.split(os.path.sep)
current_folder = itk_folder
for folder in folders:
child_folder = get_child_folder(current_folder, folder)
if child_folder is None:
print('Creating folder: ' + folder)
current_folder = communicator.create_folder(session.token,
folder,
current_folder['folder_id'])
else:
current_folder = child_folder
# get the existing or create a new item to hold the file
item_name = os.path.basename(content_link[:-4])
item_id = None
current_folder_children = communicator.folder_children(session.token,
current_folder['folder_id'])
if current_folder_children.has_key('items'):
for item in current_folder_children['items']:
if item['name'] == item_name:
item_id = item['item_id']
break
if item_id is None:
new_item = communicator.create_item(session.token, item_name,
current_folder['folder_id'])
item_id = new_item['item_id']
object_store = os.path.join(externaldata_object_store, 'MD5', md5hash)
if not os.path.exists(object_store):
sys.stderr.write('Could not find the expected object store.\n')
sys.exit(1)
upload_token = communicator.generate_upload_token(session.token,
item_id,
item_name,
md5hash)
if upload_token != "":
communicator.perform_upload(upload_token,
item_name,
item_id=item_id,
revision='head',
filepath=object_store)
def run(itk_source_dir, externaldata_object_store,
email=None, api_key=None, midas_url='https://midas3.kitware.com/midas/'):
session, communicator = connect_to_midas(email, api_key, midas_url)
md5files = []
for root, dirnames, filenames in os.walk(itk_source_dir):
for filename in fnmatch.filter(filenames, '*.md5'):
md5files.append(os.path.join(root, filename))
# Find the location of top level directory in the repository.
repository_dir = None
example_path = os.path.normpath(os.path.dirname(md5files[0]))
potential_path = example_path
previous_path = None
while previous_path != '':
dir = os.path.join(potential_path, '.git')
if os.path.exists(dir):
repository_dir = potential_path
break
previous_path = potential_path
potential_path = os.path.split(potential_path)[0]
if repository_dir == None:
sys.stderr.write('Could not find ITK repository directory.\n')
sys.exit(1)
for content_link in md5files:
src_examples = os.path.join('src', 'Examples')
if content_link.find(src_examples) == -1:
uncategorized = True
else:
uncategorized = False
if uncategorized:
folders = ['Uncategorized']
else:
group_module_path = content_link.split(src_examples)[1][1:]
hiercharchy = group_module_path.split(os.path.sep)
group = hiercharchy[0]
module = hiercharchy[1]
folders = [group, module]
upload_to_midas(content_link, externaldata_object_store, repository_dir,
session, communicator)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--api-key-file', '-k', type=argparse.FileType('r'),
help="A file that contains your Midas user's API key.")
parser.add_argument('--email', '-e',
help="Email address associated with your Midas account.")
parser.add_argument('itk_source_dir',
help='Path to the ITK source tree.')
parser.add_argument('externaldata_object_store',
help='Path to the ExternalData object store, e.g. ' \
+ 'ExternalData/Objects/ in a build tree.')
parser.add_argument('--midas-url', '-u',
help='URL of the Midas server.',
default='https://midas3.kitware.com/midas/')
args = parser.parse_args()
if args.api_key_file:
api_key = args.api_key_file.readline()
api_key = api_key.strip()
else:
api_key = None
run(args.itk_source_dir, args.externaldata_object_store,
email=args.email, api_key=api_key, midas_url=args.midas_url)
|
