1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
|
# Part of the A-A-P recipe executive: Generic Version Control
# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING
#
# Functions to get files out of a version control system and put them back.
# Most of the work is directed to one of the modules for a specific version
# control system.
# Uploading/downloading is done by functions in Remote.py.
#
import os
import os.path
import string
from Process import recipe_error
from VersContCvs import *
from Work import getwork
from Node import Node
from Remote import download_file, upload_file, remote_remove
def separate_scheme(url):
"""Isolate the scheme of the URL."""
i = string.find(url, "://")
if i < 0:
# Catch "file:~user".
if len(url) > 5 and url[:5] == "file:":
return "file", url[5:]
raise UserError, _('No :// found in "%s"') % url
scheme = string.lower(url[:i])
for c in scheme:
if not c in string.lowercase:
raise UserError, _('Illegal character before colon in "%s"') % url
return scheme, url[i + 3:]
def repl_file_name(attr, name):
"""Replace all "%file%" in "attr" with "name"."""
while 1:
i = string.find(attr, "%file%")
if i >= 0:
attr = attr[:i] + name + attr[i + 6:]
continue
i = string.find(attr, "%basename%")
if i >= 0:
attr = attr[:i] + os.path.basename(name) + attr[i + 10:]
continue
break
return attr
def verscont_command(recdict, dict, nodelist, use_cache, action):
"""
Try performing "action" on nodes "nodelist" from the semi-URL
"dict["name"]".
May Use a cached file when "use_cache" is non-zero.
Returns list of nodes that failed.
"""
scheme, name = separate_scheme(dict["name"])
# Handle a scheme for which there is a scheme_command() function.
fun = recdict["_no"].get(scheme + "_command")
if fun:
return apply(fun, (recdict, name, dict, nodelist, action))
# Handle the "cvs://" scheme.
if scheme == "cvs":
return cvs_command(recdict, name, dict, nodelist, action)
# Handle uploading.
if action in ["commit", "checkin", "publish", "add"]:
return upload_file(recdict, dict, nodelist)
failed = []
for node in nodelist:
# Assume it's a normal URL, try downloading/uploading/removing the
# file.
name = dict["name"]
dict["name"] = repl_file_name(name, node.name)
if action in ["fetch", "checkout"]:
ok = download_file(recdict, dict, node, use_cache)
elif action == "remove":
ok = remote_remove(recdict, dict, node)
else:
ok = 0
if not ok:
failed.append(node)
# restore the name, %file% might have to be replaced again
dict["name"] = name
return failed
def handle_nodelist(rpstack, recdict, nodelist, use_cache, action, attrnames):
"""Common code for fetching, committing, etc.
"action" is the name of the command: "fetch", "commit", etc.
Perform this action on each node in "nodelist", unless "--nobuild" or
"--touch" is used.
"attrnames" is a list of attribute names that can be used for this
action.
Return list of failed nodes.
"""
faillist = []
# Make a copy of the list of nodes. Items are removed when done.
todolist = nodelist[:]
while todolist:
# Find a bunch of nodes with identical attributes and handle them all
# at once.
attr = ''
thislist = []
this_use_cache = None
for node in todolist[:]:
# Use the first attribute in "attrnames" that exists and isn't
# empty.
nx_attr = ''
for n in attrnames:
if node.attributes.has_key(n):
nx_attr = node.attributes[n]
if nx_attr:
attrname = n
break
# Using cache for this node?
uc = use_cache
if (node.attributes.get("usecache")
or node.attributes.get("constant")):
uc = 1
if node.attributes.get("nocache"):
uc = 0
if not nx_attr:
recipe_error(rpstack, _("Missing %s attribute for %s")
% (attrnames[0], node.short_name()))
todolist.remove(node)
faillist.append(node)
elif ((not attr or nx_attr == attr)
and (this_use_cache is None or this_use_cache == uc)):
thislist.append(node)
todolist.remove(node)
attr = nx_attr
this_use_cache = uc
# Limit to 20 to avoid very long command lines.
if len(thislist) > 20:
break
# Do the list of nodes with identical attributes "attr". There is at
# least one.
from Dictlist import str2dictlist
list = str2dictlist(rpstack, attr)
if not list:
recipe_error(rpstack, _("%s attribute for %s is empty")
% (attrname, thislist[0].short_name()))
# Perform the action, unless started with --nobuild or --touch.
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('skip %s for %s') % (action,
str(map(lambda x: x.short_name(), thislist))))
else:
# Loop over the list of locations. Quit as soon as one worked,
# except for "publish", all locations are used then.
for loc in list:
# Try handling thislist with this location. For publish do
# this for all locations, otherwise stop when all items have
# been done.
res = verscont_command(recdict, loc, thislist,
this_use_cache, action)
if action == "publish":
for r in res:
if not r in faillist:
faillist.append(r)
else:
thislist = res
if not thislist:
break
if action != "publish" and thislist:
# this bunch failed. Still continue doing the remaining ones.
faillist.extend(thislist)
return faillist
def fetch_nodelist(rpstack, recdict, nodelist, use_cache):
"""Fetch nodes in "nodelist" according to its "fetch" attribute.
When there is no "fetch" attribute use "commit".
Only use cached files when "use_cache" is non-zero.
Return list of failed nodes."""
return handle_nodelist(rpstack, recdict, nodelist, use_cache,
"fetch", [ "fetch", "commit" ])
def verscont_nodelist(rpstack, recdict, nodelist, action):
"""Checkout nodes in "nodelist" according to their "commit" attribute.
Return list of failed nodes."""
return handle_nodelist(rpstack, recdict, nodelist, 0, action, [ "commit" ])
def publish_nodelist(rpstack, recdict, nodelist, msg):
"""Publish nodes in "nodelist" according to their "publish" attribute.
When there is no "publish" attribute use "commit".
When "msg" is non-zero give a message for up-to-date nodes.
Return None for nothing done, list of failed nodes otherwise.
"""
# Publishing is only done when the signature is outdated.
# Need to pull a few tricks to be able to call check_need_update().
from Sign import sign_updated, buildcheck_updated
from DoBuild import Update, check_need_update
todolist = []
buildchecklist = []
targetlist = []
failed = []
for node in nodelist:
# Skip nodes that don't exist.
node_name = node.get_name()
if not os.path.exists(node_name):
failed.append(node)
msg_error(recdict,
_('Published file does not exist: "%s"') % node_name)
continue
if node.attributes.has_key("publish"):
s = node.attributes["publish"]
elif node.attributes.has_key("commit"):
s = node.attributes["commit"]
else:
s = ''
# Handle each publish/commit item separately, so that when adding a new
# destination we don't upload to the old destinations as well.
from Dictlist import str2dictlist, dictlist2str
list = str2dictlist(rpstack, s)
if not list:
recipe_error(rpstack, _('publish attribute for "%s" is empty')
% node.short_name())
for attr in list:
# Use a fake virtual target to attach the signature to.
# Use the "remember" attribute, only publish when changed.
# Use the sign file in the directory related to the published file.
attr_name = attr["name"]
attr_str = dictlist2str([ attr ])
# Change "scp://" and "rscync://" to "scheme://", so that changing
# the method for uploading doesn't cause the signature to be
# outdated.
attr_name = re.sub("^[a-z]*://", "scheme://", attr_name)
buildcheck_str = re.sub("\\b[a-z]*://", "scheme://", attr_str)
target = Node(":publish:" + node_name + ">" + attr_name)
target.attributes["virtual"] = 1
target.attributes["remember"] = 1
target.attributes["signfile"] = node.get_sign_fname()
source = node.copy()
source.attributes["_node"] = source
source.attributes["name"] = node_name
source.attributes["publish"] = attr_str
# Need to publish the file when it's signature has changed or the
# relevant attributes have changed (use buildcheck for this).
update = Update()
check_need_update(recdict, update, source.attributes, target,
rootname = ":publish:" + node_name + ">")
update.set_buildcheck(recdict, buildcheck_str, target, 0)
# Find out if there is a reason to publish this source node.
reason = update.upd_reason(0, target)
if reason or msg:
disp_name = node.short_name() + "[" + attr_str + "]"
if reason:
if Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Touching "%s": %s')
% (disp_name, reason))
else:
msg_depend(recdict, _('Publishing "%s": %s')
% (disp_name, reason))
todolist.append(source)
buildchecklist.append(update.buildcheck)
targetlist.append(target)
else:
if msg:
msg_depend(recdict, _('item is up-to-date: "%s"')
% disp_name)
# For the "--contents" option we skip items for which the
# "publish" attribute changed but not the contents. Need to
# update the signature anyway, otherwise it will still be
# published next time.
if (Global.cmd_args.options.get("contents")
and not Global.cmd_args.options.get("nobuild")):
sign_updated(recdict, source, source.attributes, target)
if update.buildcheck:
buildcheck_updated(recdict, target, update.buildcheck)
if not todolist:
if failed:
return failed # all outdated files do not exist
return None # all files exist and targets are up-to-date
failed.extend(handle_nodelist(rpstack, recdict, todolist, 0,
"publish", [ "publish", "commit" ]))
# Update signatures, unless --nobuild was used withouth --touch.
if (not Global.cmd_args.options.get("nobuild")
or Global.cmd_args.options.get("touch")):
i = 0
while i < len(todolist):
# Remember the source signature and the buildcheck signature.
# But only for nodes that didn't fail.
node = todolist[i]
if not node in failed:
target = targetlist[i]
buildcheck = buildchecklist[i]
sign_updated(recdict, node, node.attributes, target)
if buildcheck:
buildcheck_updated(recdict, target, buildcheck)
i = i + 1
return failed
def verscont_remove_add(rpstack, recdict, dir, recursive, action):
"""When "action" is "remove: Remove all files in directory "dir" of VCS
that don't belong there.
When "action" is "add:" Add all files in the recipe in directory "dir"
to the VCS that are missing.
"dir" is a dictionary for the directory and its attributes.
Enter directories recursively when "recursive" is non-zero.
"""
if not dir.has_key("commit"):
recipe_error(rpstack, _("no commit attribute for %s") % dir["name"])
from Dictlist import str2dictlist
commit_list = str2dictlist(rpstack, dir["commit"])
if not commit_list:
recipe_error(rpstack, _("commit attribute for %s is empty")
% dir["name"])
dirname = os.path.abspath(dir["name"])
list = None
for commit_item in commit_list:
scheme, name = separate_scheme(commit_item["name"])
# Handle a scheme for which there is a scheme_list() function.
fun = recdict["_no"].get(scheme + "_list")
if fun:
list = apply(fun, (recdict, name, commit_item, dirname, recursive))
break
# Handle the "cvs://" scheme.
if scheme == "cvs":
list = cvs_list(recdict, name, commit_item, dirname, recursive)
break
if list is None:
recipe_error(rpstack, _("No working item in commit attribute for %s")
% dir["name"])
work = getwork(recdict)
ok = 1
if action == "remove":
# Remove: Loop over all items found in the VCS.
for item in list:
node = work.find_node(item)
if not node or not node.attributes.has_key("commit"):
if not node:
node = Node(item)
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Remove "%s"') % node.short_name())
elif verscont_command(recdict, commit_item, [ node ],
0, "remove"):
ok = 0
else:
# Add: Loop over all nodes and check if they are in the VCS.
# Only do this for a node that:
# - has the "commit" attribute
# - has a full name that is longer than the directory name
# - working recursive and the node is below the directory
# - not working recursive and the node is in the directory
# - the node does not appear in the list from the VCS
dirname_len = len(dirname)
for node in work.nodes.values():
if (node.attributes.has_key("commit")
and len(node.absname) > dirname_len
and ((recursive
and node.absname[:dirname_len] == dirname
and node.absname[dirname_len] == '/')
or (not recursive
and os.path.dirname(node.absname) == dirname))
and not node.absname in list):
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Add "%s"') % node.short_name())
elif verscont_command(recdict, commit_item, [ node ], 0, "add"):
ok = 0
return ok
# vim: set sw=4 et sts=4 tw=79 fo+=l:
|
