1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
|
// This file is part of CAF, the C++ Actor Framework. See the file LICENSE in
// the main distribution directory for license terms and copyright or visit
// https://github.com/actor-framework/actor-framework/blob/master/LICENSE.
#include <unistd.h>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "caf/all.hpp"
#include "caf/detail/base64.hpp"
#include "caf/io/all.hpp"
using namespace caf;
using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
class host_desc {
public:
std::string host;
int cpu_slots;
host_desc(std::string host_addr, int slots)
: host(std::move(host_addr)), cpu_slots(slots) {
// nop
}
host_desc(host_desc&&) = default;
host_desc(const host_desc&) = default;
host_desc& operator=(host_desc&&) = default;
host_desc& operator=(const host_desc&) = default;
static void append(vector<host_desc>& xs, const string& line) {
vector<string> fields;
split(fields, line, is_any_of(" "), token_compress_on);
if (fields.empty())
return;
host_desc hd;
hd.host = std::move(fields.front());
hd.cpu_slots = 0;
xs.emplace_back(std::move(hd));
}
private:
host_desc() = default;
};
std::vector<host_desc> read_hostfile(const string& fname) {
std::vector<host_desc> result;
std::ifstream in{fname};
std::string line;
while (std::getline(in, line))
host_desc::append(result, line);
return result;
}
bool run_ssh(actor_system& system, const string& wdir, const string& cmd,
const string& host) {
std::cout << "runssh, wdir: " << wdir << " cmd: " << cmd << " host: " << host
<< std::endl;
// pack command before sending it to avoid any issue with shell escaping
string full_cmd = "cd ";
full_cmd += wdir;
full_cmd += '\n';
full_cmd += cmd;
auto packed = detail::base64::encode(full_cmd);
std::ostringstream oss;
oss << "ssh -Y -o ServerAliveInterval=60 " << host << R"( "echo )" << packed
<< R"( | base64 --decode | /bin/sh")";
// return system(oss.str().c_str());
string line;
std::cout << "popen: " << oss.str() << std::endl;
auto fp = popen(oss.str().c_str(), "r");
if (fp == nullptr)
return false;
char buf[512];
auto eob = buf + sizeof(buf); // end-of-buf
auto pred = [](char c) { return c == 0 || c == '\n'; };
scoped_actor self{system};
while (fgets(buf, sizeof(buf), fp) != nullptr) {
auto i = buf;
auto e = std::find_if(i, eob, pred);
line.insert(line.end(), i, e);
while (e != eob && *e != 0) {
aout(self) << line << std::endl;
line.clear();
i = e + 1;
e = std::find_if(i, eob, pred);
line.insert(line.end(), i, e);
}
}
pclose(fp);
std::cout << "host down: " << host << std::endl;
if (!line.empty())
aout(self) << line << std::endl;
return true;
}
void bootstrap(actor_system& system, const string& wdir,
const host_desc& master, vector<host_desc> slaves,
const string& cmd, vector<string> args) {
using io::network::interfaces;
if (!args.empty())
args.erase(args.begin());
scoped_actor self{system};
// open a random port and generate a list of all
// possible addresses slaves can use to connect to us
auto port_res = system.middleman().publish(self, 0);
if (!port_res) {
cerr << "fatal: unable to publish actor: " << to_string(port_res.error())
<< endl;
return;
}
auto port = *port_res;
// run a slave process at master host if user defined slots > 1 for it
if (master.cpu_slots > 1)
slaves.emplace_back(master.host, master.cpu_slots - 1);
for (auto& slave : slaves) {
using namespace caf::io::network;
// build SSH command and pack it to avoid any issue with shell escaping
std::thread{[=, &system](actor bootstrapper) {
std::ostringstream oss;
oss << cmd;
if (slave.cpu_slots > 0)
oss << " --caf.scheduler.max-threads=" << slave.cpu_slots;
oss << " --caf.slave-mode"
<< " --caf.slave-name=" << slave.host
<< " --caf.bootstrap-node=";
bool is_first = true;
interfaces::traverse({protocol::ipv4, protocol::ipv6},
[&](const char*, protocol::network,
bool lo, const char* x) {
if (lo)
return;
if (!is_first)
oss << ",";
else
is_first = false;
oss << x << "/" << port;
});
for (auto& arg : args)
oss << " " << arg;
if (!run_ssh(system, wdir, oss.str(), slave.host))
anon_send(bootstrapper, slave.host);
},
actor{self}}
.detach();
}
std::string slaveslist;
for (size_t i = 0; i < slaves.size(); ++i) {
self->receive(
[&](const string& host, uint16_t slave_port) {
if (!slaveslist.empty())
slaveslist += ',';
slaveslist += host;
slaveslist += '/';
slaveslist += std::to_string(slave_port);
},
[](const string& node) {
cerr << "unable to launch process via SSH at node " << node << endl;
});
}
// run (and wait for) master
std::ostringstream oss;
oss << cmd << " --caf.slave-nodes=" << slaveslist << " " << join(args, " ");
run_ssh(system, wdir, oss.str(), master.host);
}
#define RETURN_WITH_ERROR(output) \
do { \
::std::cerr << output << ::std::endl; \
return EXIT_FAILURE; \
} while (true)
namespace {
struct config : actor_system_config {
config() {
opt_group{custom_options_, "global"}
.add(hostfile, "hostfile", "path to hostfile")
.add(wdir, "wdir", "working directory");
}
string hostfile;
string wdir;
};
} // namespace
int main(int argc, char** argv) {
config cfg;
if (auto err = cfg.parse(argc, argv)) {
std::cerr << "error parsing command line: " << to_string(err) << '\n';
return EXIT_FAILURE;
}
if (cfg.cli_helptext_printed)
return EXIT_SUCCESS;
if (cfg.slave_mode)
RETURN_WITH_ERROR("cannot use slave mode in caf-run tool");
std::unique_ptr<char, void (*)(void*)> pwd{getcwd(nullptr, 0), ::free};
if (cfg.hostfile.empty())
RETURN_WITH_ERROR("no hostfile specified or hostfile is empty");
auto& remainder = cfg.remainder;
if (remainder.empty())
RETURN_WITH_ERROR("empty command line");
auto cmd = std::move(remainder.front());
vector<string> xs;
for (auto i = cfg.remainder.begin() + 1; i != cfg.remainder.end(); ++i)
xs.emplace_back(std::move(*i));
auto hosts = read_hostfile(cfg.hostfile);
if (hosts.empty())
RETURN_WITH_ERROR("no valid entry in hostfile");
actor_system system{cfg};
auto master = hosts.front();
hosts.erase(hosts.begin());
bootstrap(system, (cfg.wdir.empty()) ? pwd.get() : cfg.wdir.c_str(), master,
std::move(hosts), cmd, xs);
return EXIT_SUCCESS;
}
|
