File: master_checks.c

package info (click to toggle)
uwsgi 2.0.31-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 6,564 kB
  • sloc: ansic: 87,066; python: 7,004; cpp: 1,133; java: 708; perl: 678; sh: 585; ruby: 555; makefile: 148; xml: 130; cs: 121; objc: 37; php: 28; erlang: 20; javascript: 11
file content (453 lines) | stat: -rw-r--r-- 13,174 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
#include "uwsgi.h"

extern struct uwsgi_server uwsgi;

// check if all of the workers are dead and exit uWSGI
void uwsgi_master_check_death() {
	if (uwsgi_instance_is_dying) {
		int i;
		for(i=1;i<=uwsgi.numproc;i++) {
			if (uwsgi.workers[i].pid > 0) {
				return;
			}
		}
		for(i=0;i<uwsgi.mules_cnt;i++) {
			if (uwsgi.mules[i].pid > 0) {
				return;
			}
		}
		uwsgi_log("goodbye to uWSGI.\n");
		exit(uwsgi.status.dying_for_need_app ? UWSGI_FAILED_APP_CODE : 0);
	}
}

// check if all of the workers are dead, and trigger a reload
int uwsgi_master_check_reload(char **argv) {
        if (uwsgi_instance_is_reloading) {
                int i;
                for(i=1;i<=uwsgi.numproc;i++) {
                        if (uwsgi.workers[i].pid > 0) {
                                return 0;
                        }
                }
		for(i=0;i<uwsgi.mules_cnt;i++) {
			if (uwsgi.mules[i].pid > 0) {
				return 0;
			}
		}
		uwsgi_reload(argv);
		// never here (unless in shared library mode)
		return -1;
        }
	return 0;
}

// check for chain reload
void uwsgi_master_check_chain() {
	static time_t last_check = 0;

	if (!uwsgi.status.chain_reloading) return;

	// we need to ensure the previous worker (if alive) is accepting new requests
	// before going on
	if (uwsgi.status.chain_reloading > 1) {
		struct uwsgi_worker *previous_worker = &uwsgi.workers[uwsgi.status.chain_reloading-1];
		// is the previous worker alive ?
		if (previous_worker->pid > 0 && !previous_worker->cheaped) {
			// the worker has been respawned but it is still not ready
			if (previous_worker->accepting == 0) {
				time_t now = uwsgi_now();
				if (now != last_check) {
					uwsgi_log_verbose("chain is still waiting for worker %d...\n", uwsgi.status.chain_reloading-1);
					last_check = now;
				}
				return;
			}
		}
	}

	// if all the processes are recycled, the chain is over
	if (uwsgi.status.chain_reloading > uwsgi.numproc) {
		uwsgi.status.chain_reloading = 0;
                uwsgi_log_verbose("chain reloading complete\n");
		return;
	}

	uwsgi_block_signal(SIGHUP);
	int i;
	for(i=uwsgi.status.chain_reloading;i<=uwsgi.numproc;i++) {
		struct uwsgi_worker *uw = &uwsgi.workers[i];
		if (uw->pid > 0 && !uw->cheaped && uw->accepting) {
			// the worker could have been already cursed
			if (uw->cursed_at == 0) {
				uwsgi_log_verbose("chain next victim is worker %d\n", i);
				uwsgi_curse(i, SIGHUP);
			}
			break;
		}
		else {
			uwsgi.status.chain_reloading++;
		}
        }
	uwsgi_unblock_signal(SIGHUP);
}


// special function for assuming all of the workers are dead
void uwsgi_master_commit_status() {
	int i;
	for(i=1;i<=uwsgi.numproc;i++) {
		uwsgi.workers[i].pid = 0;
	}
}

void uwsgi_master_check_idle() {

	static time_t last_request_timecheck = 0;
	static uint64_t last_request_count = 0;
	int i;
	int waitpid_status;

	if (!uwsgi.idle || uwsgi.status.is_cheap)
		return;

	uwsgi.current_time = uwsgi_now();
	if (!last_request_timecheck)
		last_request_timecheck = uwsgi.current_time;

	// security check, stop the check if there are busy workers
	for (i = 1; i <= uwsgi.numproc; i++) {
		if (uwsgi.workers[i].cheaped == 0 && uwsgi.workers[i].pid > 0) {
			if (uwsgi_worker_is_busy(i)) {
				return;
			}
		}
	}

	if (last_request_count != uwsgi.workers[0].requests) {
		last_request_timecheck = uwsgi.current_time;
		last_request_count = uwsgi.workers[0].requests;
	}
	// a bit of over-engeneering to avoid clock skews
	else if (last_request_timecheck < uwsgi.current_time && (uwsgi.current_time - last_request_timecheck > uwsgi.idle)) {
		uwsgi_log("workers have been inactive for more than %d seconds (%llu-%llu)\n", uwsgi.idle, (unsigned long long) uwsgi.current_time, (unsigned long long) last_request_timecheck);
		uwsgi.status.is_cheap = 1;
		if (uwsgi.die_on_idle) {
			if (uwsgi.has_emperor) {
				char byte = 22;
				if (write(uwsgi.emperor_fd, &byte, 1) != 1) {
					uwsgi_error("write()");
					kill_them_all(0);
				}
			}
			else {
				kill_them_all(0);
			}
			return;
		}
		for (i = 1; i <= uwsgi.numproc; i++) {
			uwsgi.workers[i].cheaped = 1;
			if (uwsgi.workers[i].pid == 0)
				continue;
			// first send SIGINT
			kill(uwsgi.workers[i].pid, SIGINT);
			// and start waiting upto 3 seconds
			int j;
			for(j=0;j<3;j++) {
				sleep(1);
				int ret = waitpid(uwsgi.workers[i].pid, &waitpid_status, WNOHANG);
				if (ret == 0) continue;
				if (ret == (int) uwsgi.workers[i].pid) goto done;
				// on error, directly send SIGKILL
				break;
			}
			kill(uwsgi.workers[i].pid, SIGKILL);
			if (waitpid(uwsgi.workers[i].pid, &waitpid_status, 0) < 0) {
				if (errno != ECHILD)
					uwsgi_error("uwsgi_master_check_idle()/waitpid()");
			}
			else {
done:
				uwsgi.workers[i].pid = 0;
				uwsgi.workers[i].rss_size = 0;
				uwsgi.workers[i].vsz_size = 0;
			}
		}
		uwsgi_add_sockets_to_queue(uwsgi.master_queue, -1);
		uwsgi_log("cheap mode enabled: waiting for socket connection...\n");
		last_request_timecheck = 0;
	}

}

int uwsgi_master_check_harakiri(int w, int c, time_t harakiri) {
/**
 * Triggers a harakiri when the following conditions are met:
 * - harakiri timeout > current time
 * - listen queue pressure (ie backlog > harakiri_queue_threshold)
 *
 * The first harakiri attempt on a worker will be graceful if harakiri_graceful_timeout > 0,
 * then the worker has harakiri_graceful_timeout seconds to shutdown cleanly, otherwise
 * a second harakiri will trigger a SIGKILL
 *
 */
#ifdef __linux__
	int backlog = uwsgi.shared->backlog;
#else
	int backlog = 0;
#endif
	if (harakiri == 0 || harakiri > (time_t) uwsgi.current_time) {
		return 0;
	}
	// no pending harakiri for the worker and no backlog pressure, safe to skip
	if (uwsgi.workers[w].pending_harakiri == 0 &&  backlog < uwsgi.harakiri_queue_threshold) {
		uwsgi_log_verbose("HARAKIRI: Skipping harakiri on worker %d. Listen queue is smaller than the threshold (%d < %d)\n",
			w, backlog, uwsgi.harakiri_queue_threshold);
		return 0;
	}

	trigger_harakiri(w);
	if (uwsgi.harakiri_graceful_timeout > 0) {
		uwsgi.workers[w].harakiri = harakiri + uwsgi.harakiri_graceful_timeout;
		uwsgi_log_verbose("HARAKIRI: graceful termination attempt on worker %d with signal %d. Next harakiri: %d\n",
			w, uwsgi.harakiri_graceful_signal, uwsgi.workers[w].harakiri);
	}
	return 1;
}

int uwsgi_master_check_workers_deadline() {
	int i,j;
	int ret = 0;
	for (i = 1; i <= uwsgi.numproc; i++) {
		for(j=0;j<uwsgi.cores;j++) {
			/* first check for harakiri */
			if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].harakiri)) {
				uwsgi_log_verbose("HARAKIRI triggered by worker %d core %d !!!\n", i, j);
				ret = 1;
				break;
			}
			/* then user-defined harakiri */
			if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].user_harakiri)) {
				uwsgi_log_verbose("HARAKIRI (user) triggered by worker %d core %d !!!\n", i, j);
				ret = 1;
				break;
			}
		}
		// then for evil memory checkers
		if (uwsgi.evil_reload_on_as) {
			if ((rlim_t) uwsgi.workers[i].vsz_size >= uwsgi.evil_reload_on_as) {
				uwsgi_log("*** EVIL RELOAD ON WORKER %d ADDRESS SPACE: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].vsz_size, uwsgi.workers[i].pid);
				kill(uwsgi.workers[i].pid, SIGKILL);
				uwsgi.workers[i].vsz_size = 0;
				ret = 1;
			}
		}
		if (uwsgi.evil_reload_on_rss) {
			if ((rlim_t) uwsgi.workers[i].rss_size >= uwsgi.evil_reload_on_rss) {
				uwsgi_log("*** EVIL RELOAD ON WORKER %d RSS: %lld (pid: %d) ***\n", i, (long long) uwsgi.workers[i].rss_size, uwsgi.workers[i].pid);
				kill(uwsgi.workers[i].pid, SIGKILL);
				uwsgi.workers[i].rss_size = 0;
				ret = 1;
			}
		}
		// check if worker was running longer than allowed lifetime
		if (uwsgi.workers[i].pid > 0 && uwsgi.workers[i].cheaped == 0 && uwsgi.max_worker_lifetime > 0) {
			uint64_t lifetime = uwsgi_now() - uwsgi.workers[i].last_spawn;
			if (lifetime > (uwsgi.max_worker_lifetime + (i-1) * uwsgi.max_worker_lifetime_delta)  && uwsgi.workers[i].manage_next_request == 1) {
				uwsgi_log("worker %d lifetime reached, it was running for %llu second(s)\n", i, (unsigned long long) lifetime);
				uwsgi.workers[i].manage_next_request = 0;
				kill(uwsgi.workers[i].pid, SIGWINCH);
				ret = 1;
			}
		}

		// need to find a better way
		//uwsgi.workers[i].last_running_time = uwsgi.workers[i].running_time;
	}


	return ret;
}


int uwsgi_master_check_gateways_deadline() {

	int i;
	int ret = 0;
	for (i = 0; i < ushared->gateways_cnt; i++) {
		if (ushared->gateways_harakiri[i] > 0) {
			if (ushared->gateways_harakiri[i] < (time_t) uwsgi.current_time) {
				if (ushared->gateways[i].pid > 0) {
					uwsgi_log("*** HARAKIRI ON GATEWAY %s %d (pid: %d) ***\n", ushared->gateways[i].name, ushared->gateways[i].num, ushared->gateways[i].pid);
					kill(ushared->gateways[i].pid, SIGKILL);
					ret = 1;
				}
				ushared->gateways_harakiri[i] = 0;
			}
		}
	}
	return ret;
}

int uwsgi_master_check_mules_deadline() {
	int i;
	int ret = 0;

	for (i = 0; i < uwsgi.mules_cnt; i++) {
		if (uwsgi.mules[i].harakiri > 0) {
			if (uwsgi.mules[i].harakiri < (time_t) uwsgi.current_time) {
				uwsgi_log("*** HARAKIRI ON MULE %d HANDLING SIGNAL %d (pid: %d) ***\n", i + 1, uwsgi.mules[i].signum, uwsgi.mules[i].pid);
				kill(uwsgi.mules[i].pid, SIGKILL);
				uwsgi.mules[i].harakiri = 0;
				ret = 1;
			}
		}
		// user harakiri
		if (uwsgi.mules[i].user_harakiri > 0) {
                        if (uwsgi.mules[i].user_harakiri < (time_t) uwsgi.current_time) {
                                uwsgi_log("*** HARAKIRI ON MULE %d (pid: %d) ***\n", i + 1, uwsgi.mules[i].pid);
                                kill(uwsgi.mules[i].pid, SIGKILL);
                                uwsgi.mules[i].user_harakiri = 0;
				ret = 1;
                        }
                }
	}
	return ret;
}

int uwsgi_master_check_spoolers_deadline() {
	int ret = 0;
	struct uwsgi_spooler *uspool = uwsgi.spoolers;
	while (uspool) {
		if (uspool->harakiri > 0 && uspool->harakiri < (time_t) uwsgi.current_time) {
			uwsgi_log("*** HARAKIRI ON THE SPOOLER (pid: %d) ***\n", uspool->pid);
			kill(uspool->pid, SIGKILL);
			uspool->harakiri = 0;
			ret = 1;
		}
		if (uspool->user_harakiri > 0 && uspool->user_harakiri < (time_t) uwsgi.current_time) {
                        uwsgi_log("*** HARAKIRI ON THE SPOOLER (pid: %d) ***\n", uspool->pid);
                        kill(uspool->pid, SIGKILL);
                        uspool->user_harakiri = 0;
			ret = 1;
                }
		uspool = uspool->next;
	}
	return ret;
}


int uwsgi_master_check_spoolers_death(int diedpid) {

	struct uwsgi_spooler *uspool = uwsgi.spoolers;

	while (uspool) {
		if (uspool->pid > 0 && diedpid == uspool->pid) {
			if (uspool->cursed_at) {
				uspool->pid = 0;
				uspool->cursed_at = 0;
				uspool->no_mercy_at = 0;
			}
			uwsgi_log("OOOPS the spooler is no more...trying respawn...\n");
			uspool->respawned++;
			uspool->pid = spooler_start(uspool);
			return -1;
		}
		uspool = uspool->next;
	}
	return 0;
}

int uwsgi_master_check_emperor_death(int diedpid) {
	if (uwsgi.emperor_pid >= 0 && diedpid == uwsgi.emperor_pid) {
		uwsgi_log_verbose("!!! Emperor died !!!\n");
		uwsgi_emperor_start();
		return -1;
	}
	return 0;
}

int uwsgi_master_check_mules_death(int diedpid) {
	int i;
	for (i = 0; i < uwsgi.mules_cnt; i++) {
		if (!(uwsgi.mules[i].pid == diedpid)) continue;
		if (!uwsgi.mules[i].cursed_at) {
			uwsgi_log("OOOPS mule %d (pid: %d) crippled...trying respawn...\n", i + 1, uwsgi.mules[i].pid);
		}
		uwsgi.mules[i].no_mercy_at = 0;
		uwsgi.mules[i].cursed_at = 0;
		uwsgi_mule(i + 1);
		return -1;
	}
	return 0;
}

int uwsgi_master_check_gateways_death(int diedpid) {
	int i;
	for (i = 0; i < ushared->gateways_cnt; i++) {
		if (ushared->gateways[i].pid == diedpid) {
			gateway_respawn(i);
			return -1;
		}
	}
	return 0;
}

int uwsgi_master_check_daemons_death(int diedpid) {
	/* reload the daemons */
	if (uwsgi_daemon_check_pid_reload(diedpid)) {
		return -1;
	}
	return 0;
}

int uwsgi_worker_is_busy(int wid) {
	int i;
	if (uwsgi.workers[wid].sig) return 1;
	for(i=0;i<uwsgi.cores;i++) {
		if (uwsgi.workers[wid].cores[i].in_request) {
			return 1;
		}
	}
	return 0;
}

int uwsgi_master_check_cron_death(int diedpid) {
	struct uwsgi_cron *uc = uwsgi.crons;
	while (uc) {
		if (uc->pid == (pid_t) diedpid) {
			uwsgi_log("[uwsgi-cron] command \"%s\" running with pid %d exited after %d second(s)\n", uc->command, uc->pid, uwsgi_now() - uc->started_at);
			uc->pid = -1;
			uc->started_at = 0;
			return -1;
		}
		uc = uc->next;
	}
	return 0;
}

int uwsgi_master_check_crons_deadline() {
	int ret = 0;
	struct uwsgi_cron *uc = uwsgi.crons;
	while (uc) {
		if (uc->pid >= 0 && uc->harakiri > 0 && uc->harakiri < (time_t) uwsgi.current_time) {
			uwsgi_log("*** HARAKIRI ON CRON \"%s\" (pid: %d) ***\n", uc->command, uc->pid);
			kill(-uc->pid, SIGKILL);
			ret = 1;
		}
		uc = uc->next;
	}
	return ret;
}

void uwsgi_master_check_mountpoints() {
	struct uwsgi_string_list *usl;
	uwsgi_foreach(usl, uwsgi.mountpoints_check) {
		if (uwsgi_check_mountpoint(usl->value)) {
			uwsgi_log_verbose("mountpoint %s failed, triggering detonation...\n", usl->value);
			uwsgi_nuclear_blast();
			//never here
			exit(1);
		}
	}
}