3
from __future__ import print_function
6
Upload binary data to Midas.
8
Since every local Git repository contains a copy of the entire project history,
9
it is important to avoid adding large binary files directly to the repository.
10
Large binary files added and removed through the history of a project will cause
11
the repository to become bloated, take up too much disk space, require excessive
12
time and bandwidth to download, etc.
14
The solution to this problem adopted by this project is to store binary files,
15
such as images, in a separate location outside the repository, then download the
16
files at build time with CMake.
18
A "content link" file containing an identifying MD5 checksum is stored in the
19
Git repository at the path where the file would exist with the ".md5" extension
20
added to the file. CMake will find these content link files at build time,
21
download them from a list of server resources, and create symlinks or copies of
22
the original files at the corresponding location in the build tree.
26
https://midas3.kitware.com/midas/
28
is an ITK community resource where any community member can upload binary data
29
files. This script automates the upload of data to the server and generation of
30
the content link file. Before using this script, please go to the Midas
31
website, register, and join the ITK community.
33
This script requires the Python module pydas:
35
https://github.com/midasplatform/pydas
37
which can be installed with::
41
Pass in the input binary files to be uploaded. These should be files
42
located in the SimpleITK source tree. They will be uploaded and
43
replaced with a *.md5 file that can be added to the current
44
commit with "git add -- path/to/file.md5".
55
def connect_to_midas(email=None, api_key=None):
56
midas_url = 'https://midas3.kitware.com/midas/'
57
#pydas.login(url=midas_url, email=email, api_key=api_key)
59
pydas.login(url=midas_url, email=email, api_key=api_key)
61
print('Error occurred while logging in to ' + midas_url)
63
session = pydas.session
64
communicator = session.communicator
65
return session, communicator
68
def upload_to_midas(input_file, output_file, folders, session, communicator):
69
# get the MD5 checksum
70
print('Computing MD5 checksum...')
72
with open(input_file, 'rb') as fp:
73
for chunk in iter(lambda: fp.read(128 * md5.block_size), b''):
75
md5hash = md5.hexdigest()
76
print('Checksum: ' + md5hash)
79
def get_child_folder(parent, child_name):
80
children = communicator.folder_children(session.token,
82
for folder in children['folders']:
83
if folder['name'] == child_name:
86
itk_community = communicator.get_community_by_name('ITK')
87
itk_public = get_child_folder(itk_community, 'Public')
88
simpleitk = get_child_folder(itk_public, 'SimpleITK')
90
current_folder = simpleitk
91
for folder in folders:
92
child_folder = get_child_folder(current_folder, folder)
93
if child_folder is None:
94
print('Creating folder: ' + folder)
96
communicator.create_folder(session.token,
98
current_folder['folder_id'])
100
current_folder = child_folder
102
# get the existing or create a new item to hold the file
103
item_name = os.path.basename(input_file)
105
current_folder_children = \
106
communicator.folder_children(session.token,
107
current_folder['folder_id'])
108
if 'items' in current_folder_children:
109
for item in current_folder_children['items']:
110
if item['name'] == item_name:
111
item_id = item['item_id']
115
new_item = communicator.create_item(session.token, item_name,
116
current_folder['folder_id'])
117
item_id = new_item['item_id']
119
upload_token = communicator.generate_upload_token(session.token,
123
if upload_token != "":
124
communicator.perform_upload(upload_token,
130
# write the content link file
131
with open(output_file, 'w') as fp:
136
def find_git_dir(filepath):
137
"""Find our best estimate of GIT_DIR to locate the root of the SimpleITK
139
filepath = os.path.abspath(filepath)
140
head, tail = os.path.split(filepath)
142
while head != previous_head:
143
if os.path.exists(os.path.join(head, '.git')):
146
head, tail = os.path.split(head)
147
print('Could not find the root of the SimpleITK repository!')
151
def run(input_files, output_files,
152
email=None, api_key=None,
154
git_dir = find_git_dir(input_files[0])
156
git_email_cmd = subprocess.Popen(['git', 'config', 'user.email'],
158
stdout=subprocess.PIPE)
159
if git_email_cmd.wait() is 0:
160
git_email = git_email_cmd.stdout.readline().strip()
161
email_input = raw_input('Email [' + git_email + ']: ')
162
if email_input == '':
167
session, communicator = connect_to_midas(email, api_key)
169
for ii in range(len(input_files)):
170
input_abspath = os.path.abspath(input_files[ii])
171
folders = input_abspath[len(git_dir)+1:].split(os.path.sep)[:-1]
172
upload_to_midas(input_files[ii], output_files[ii], folders,
173
session, communicator)
176
os.remove(input_files[ii])
179
if __name__ == '__main__':
180
usage = "%prog [options] input1 input2 ... inputN"
181
parser = optparse.OptionParser(usage=usage)
182
parser.add_option('--api-key-file', '-k', dest='api_key_file',
183
help="A file that contains your Midas user's API key.")
184
parser.add_option('--email', '-e',
185
help="Email address associated with your Midas account.")
186
parser.add_option('--no-delete', '-n', action='store_true',
188
help='Do not remove the input files after upload.')
189
(options, input_files) = parser.parse_args()
191
if options.api_key_file:
192
with open(options.api_key_file, 'r') as fp:
193
api_key = fp.readline()
194
api_key = api_key.strip()
199
for ii in range(len(input_files)):
200
output_files.append(input_files[ii] + '.md5')
202
no_delete = options.no_delete
204
run(input_files, output_files,
205
email=options.email, api_key=api_key,
added
removed
Utilities/UploadBinaryData.py
