1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
// Copyright 2015 Canonical Ltd.
// Copyright 2015 Cloudbase Solutions SRL
// Licensed under the LGPLv3, see LICENCE file for details.
package manager
import (
"fmt"
"os"
"os/exec"
"strings"
"time"
"github.com/juju/errors"
"github.com/juju/loggo"
"github.com/juju/utils"
)
var (
logger = loggo.GetLogger("juju.utils.packaging.manager")
AttemptStrategy = utils.AttemptStrategy{
Delay: 10 * time.Second,
Min: 30,
}
)
// CommandOutput is cmd.Output. It was aliased for testing purposes.
var CommandOutput = (*exec.Cmd).CombinedOutput
// processStateSys is ps.Sys. It was aliased for testing purposes.
var ProcessStateSys = (*os.ProcessState).Sys
// RunCommand is utils.RunCommand. It was aliased for testing purposes.
var RunCommand = utils.RunCommand
// exitStatuser is a mini-interface for the ExitStatus() method.
type exitStatuser interface {
ExitStatus() int
}
// RunCommandWithRetry is a helper function which tries to execute the given command.
// It tries to do so for 30 times with a 10 second sleep between commands.
// It returns the output of the command, the exit code, and an error, if one occurs,
// logging along the way.
// It was aliased for testing purposes.
var RunCommandWithRetry = func(cmd string, getFatalError func(string) error) (output string, code int, err error) {
var out []byte
// split the command for use with exec
args := strings.Fields(cmd)
if len(args) <= 1 {
return "", 1, errors.New(fmt.Sprintf("too few arguments: expected at least 2, got %d", len(args)))
}
logger.Infof("Running: %s", cmd)
// Retry operation 30 times, sleeping every 10 seconds between attempts.
// This avoids failure in the case of something else having the dpkg lock
// (e.g. a charm on the machine we're deploying containers to).
for a := AttemptStrategy.Start(); a.Next(); {
// Create the command for each attempt, because we need to
// call cmd.CombinedOutput only once. See http://pad.lv/1394524.
command := exec.Command(args[0], args[1:]...)
out, err = CommandOutput(command)
if err == nil {
return string(out), 0, nil
}
exitError, ok := err.(*exec.ExitError)
if !ok {
err = errors.Annotatef(err, "unexpected error type %T", err)
break
}
waitStatus, ok := ProcessStateSys(exitError.ProcessState).(exitStatuser)
if !ok {
err = errors.Annotatef(err, "unexpected process state type %T", exitError.ProcessState.Sys())
break
}
// Both apt-get and yum return 100 on abnormal execution due to outside
// issues (ex: momentary dns failure).
code = waitStatus.ExitStatus()
if code != 100 {
break
}
if getFatalError != nil {
if fatalErr := getFatalError(string(out)); fatalErr != nil {
err = errors.Annotatef(fatalErr, "encountered fatal error")
break
}
}
logger.Infof("Retrying: %s", cmd)
}
if err != nil {
logger.Errorf("packaging command failed: %v; cmd: %q; output: %s",
err, cmd, string(out))
return "", code, errors.Errorf("packaging command failed: %v", err)
}
return string(out), 0, nil
}
|
