File: NetworkUtilsRemote.cc

package info (click to toggle)
trafficserver 9.2.5%2Bds-1
  • links: PTS, VCS
  • area: main
  • in suites: sid
  • size: 53,008 kB
  • sloc: cpp: 345,484; ansic: 31,134; python: 24,200; sh: 7,271; makefile: 3,045; perl: 2,261; java: 277; pascal: 119; sql: 94; xml: 2
file content (719 lines) | stat: -rw-r--r-- 23,903 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
/** @file

  A brief file description

  @section license License

  Licensed to the Apache Software Foundation (ASF) under one
  or more contributor license agreements.  See the NOTICE file
  distributed with this work for additional information
  regarding copyright ownership.  The ASF licenses this file
  to you under the Apache License, Version 2.0 (the
  "License"); you may not use this file except in compliance
  with the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
 */

#include "tscore/ink_config.h"
#include "tscore/ink_defs.h"
#include "tscore/ink_sock.h"
#include "tscore/ink_string.h"
#include "tscore/ink_memory.h"
#include "tscore/I_Layout.h"
#include "NetworkUtilsRemote.h"
#include "CoreAPI.h"
#include "CoreAPIShared.h"
#include "MgmtSocket.h"
#include "MgmtMarshall.h"

CallbackTable *remote_event_callbacks;

int main_socket_fd  = -1;
int event_socket_fd = -1;

// need to store for reconnecting scenario
char *main_socket_path  = nullptr; // "<path>/mgmtapi.sock"
char *event_socket_path = nullptr; // "<path>/eventapi.sock"

static void *event_callback_thread(void *arg);

/**********************************************************************
 * Socket Helper Functions
 **********************************************************************/
void
set_socket_paths(const char *path)
{
  // free previously set paths if needed
  ats_free(main_socket_path);
  ats_free(event_socket_path);

  // construct paths based on user input
  // form by replacing "mgmtapi.sock" with "eventapi.sock"
  if (path) {
    main_socket_path  = ats_stringdup(Layout::relative_to(path, MGMTAPI_MGMT_SOCKET_NAME));
    event_socket_path = ats_stringdup(Layout::relative_to(path, MGMTAPI_EVENT_SOCKET_NAME));
  } else {
    main_socket_path  = nullptr;
    event_socket_path = nullptr;
  }

  return;
}

/**********************************************************************
 * socket_test
 *
 * purpose: performs socket write to check status of other end of connection
 * input: None
 * output: return false if socket write failed due to some other error
 *         return true if socket write successful
 * notes: send the API_PING test msg
 **********************************************************************/
static bool
socket_test(int fd)
{
  OpType optype       = OpType::API_PING;
  MgmtMarshallInt now = time(nullptr);

  if (MGMTAPI_SEND_MESSAGE(fd, OpType::API_PING, &optype, &now) == TS_ERR_OKAY) {
    return true; // write was successful; connection still open
  }

  return false;
}

/***************************************************************************
 * connect
 *
 * purpose: connects to the port on traffic server that listens to mgmt
 *          requests & issues out responses and alerts
 * 1) create and set the client socket_fd; connect to TM
 * 2) create and set the client's event_socket_fd; connect to TM
 * output: TS_ERR_OKAY          - if both sockets successfully connect to TM
 *         TS_ERR_NET_ESTABLISH - at least one unsuccessful connection
 * notes: If connection breaks it is responsibility of client to reconnect
 *        otherwise traffic server will assume mgmt stopped request and
 *        goes back to just sitting and listening for connection.
 ***************************************************************************/
TSMgmtError
ts_connect()
{
  struct sockaddr_un client_sock;
  struct sockaddr_un client_event_sock;

  int sockaddr_len;

  // make sure a socket path is set up
  if (!main_socket_path || !event_socket_path) {
    goto ERROR;
  }
  // make sure the length of main_socket_path do not exceed the sizeof(sun_path)
  if (strlen(main_socket_path) > sizeof(client_sock.sun_path) - 1) {
    goto ERROR;
  }
  // make sure the length of event_socket_path do not exceed the sizeof(sun_path)
  if (strlen(event_socket_path) > sizeof(client_event_sock.sun_path) - 1) {
    goto ERROR;
  }
  // create a socket
  main_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (main_socket_fd < 0) {
    goto ERROR; // ERROR - can't open socket
  }
  // setup Unix domain socket
  memset(&client_sock, 0, sizeof(sockaddr_un));
  client_sock.sun_family = AF_UNIX;
  ink_strlcpy(client_sock.sun_path, main_socket_path, sizeof(client_sock.sun_path));
#if defined(darwin) || defined(freebsd)
  sockaddr_len = sizeof(sockaddr_un);
#else
  sockaddr_len = sizeof(client_sock.sun_family) + strlen(client_sock.sun_path);
#endif
  // connect call
  if (connect(main_socket_fd, reinterpret_cast<struct sockaddr *>(&client_sock), sockaddr_len) < 0) {
    close(main_socket_fd);
    main_socket_fd = -1;
    goto ERROR; // connection is down
  }
  // -------- set up the event socket ------------------
  // create a socket
  event_socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
  if (event_socket_fd < 0) {
    close(main_socket_fd); // close the other socket too!
    main_socket_fd = -1;
    goto ERROR; // ERROR - can't open socket
  }
  // setup Unix domain socket
  memset(&client_event_sock, 0, sizeof(sockaddr_un));
  client_event_sock.sun_family = AF_UNIX;
  ink_strlcpy(client_event_sock.sun_path, event_socket_path, sizeof(client_event_sock.sun_path));
#if defined(darwin) || defined(freebsd)
  sockaddr_len = sizeof(sockaddr_un);
#else
  sockaddr_len = sizeof(client_event_sock.sun_family) + strlen(client_event_sock.sun_path);
#endif
  // connect call
  if (connect(event_socket_fd, reinterpret_cast<struct sockaddr *>(&client_event_sock), sockaddr_len) < 0) {
    close(event_socket_fd);
    close(main_socket_fd);
    event_socket_fd = -1;
    main_socket_fd  = -1;
    goto ERROR; // connection is down
  }

  return TS_ERR_OKAY;

ERROR:
  return TS_ERR_NET_ESTABLISH;
}

/***************************************************************************
 * disconnect
 *
 * purpose: disconnect from traffic server; closes sockets and resets their values
 * input: None
 * output: TS_ERR_FAIL, TS_ERR_OKAY
 * notes: doesn't do clean up - all cleanup should be done before here
 ***************************************************************************/
TSMgmtError
disconnect()
{
  int ret;

  if (main_socket_fd > 0) {
    ret            = close(main_socket_fd);
    main_socket_fd = -1;
    if (ret < 0) {
      return TS_ERR_FAIL;
    }
  }

  if (event_socket_fd > 0) {
    ret             = close(event_socket_fd);
    event_socket_fd = -1;
    if (ret < 0) {
      return TS_ERR_FAIL;
    }
  }

  return TS_ERR_OKAY;
}

/***************************************************************************
 * reconnect
 *
 * purpose: reconnects to TM (eg. when TM restarts); does all the necessary
 *          set up for reconnection
 * input: None
 * output: TS_ERR_FAIL, TS_ERR_OKAY
 * notes: necessary events for a new client-TM connection:
 * 1) get new socket_fd using old socket_path by calling connect()
 * 2) relaunch event_poll_thread_main with new socket_fd
 * 3) re-notify TM of all the client's registered callbacks by send msg
 ***************************************************************************/
TSMgmtError
reconnect()
{
  TSMgmtError err;

  err = disconnect();
  if (err != TS_ERR_OKAY) { // problem disconnecting
    return err;
  }

  // use the socket_path that was called by remote client on first init
  // use connect instead of TSInit() b/c if TM restarted, client-side tables
  // would be recreated; just want to reconnect to same socket_path
  err = ts_connect();
  if (err != TS_ERR_OKAY) { // problem establishing connection
    return err;
  }

  // relaunch a new event thread since socket_fd changed
  if (0 == (ts_init_options & TS_MGMT_OPT_NO_EVENTS)) {
    ink_thread_create(&ts_event_thread, event_poll_thread_main, &event_socket_fd, 0, 0, nullptr);
    // re-register the callbacks on the TM side for this new client connection
    if (remote_event_callbacks) {
      err = send_register_all_callbacks(event_socket_fd, remote_event_callbacks);
      if (err != TS_ERR_OKAY) { // problem establishing connection
        return err;
      }
    }
  } else {
    ts_event_thread = ink_thread_null();
  }

  return TS_ERR_OKAY;
}

/***************************************************************************
 * reconnect_loop
 *
 * purpose: attempts to reconnect to TM (eg. when TM restarts) for the
 *          specified number of times
 * input:  num_attempts - number of reconnection attempts to try before quit
 * output: TS_ERR_OKAY - if successfully reconnected within num_attempts
 *         TS_ERR_xx - the reason the reconnection failed
 * notes:
 ***************************************************************************/
TSMgmtError
reconnect_loop(int num_attempts)
{
  int numTries    = 0;
  TSMgmtError err = TS_ERR_FAIL;

  while (numTries < num_attempts) {
    numTries++;
    err = reconnect();
    if (err == TS_ERR_OKAY) {
      return TS_ERR_OKAY; // successful connection
    }
    sleep(1); // to make it slower
  }

  return err; // unsuccessful connection after num_attempts
}

/*************************************************************************
 * connect_and_send
 *
 * purpose:
 * When sending a request, it's possible that the user had restarted
 * Traffic Manager. This means that the connection between TM and
 * the remote client has been broken, so the client needs to re-"connect"
 * to Traffic Manager. So, after "writing" to the socket in each
 * "send_xx_request" function, need to check if the TM socket has
 * been closed or not; the "write" function's errno will indicate if
 * the other end of the socket has been closed or not. If it is closed,
 * then need to try to re"connect", then resend the message request if
 * the "connect" was successful.
 * 1) try connect()
 * 2) if connect() success, then resend the request.
 * output: TS_ERR_NET_xx - connection problem or TS_ERR_OKAY
 * notes:
 * This function is basically called by the special "socket_write_conn" fn
 * which will call this fn if it tries to write to the socket and discovers
 * the local end of the socket is closed
 * Warning: system also sends a SIGPIPE error when try to write to socket
 * which is not open; which will by default terminate the process;
 * client needs to "ignore" the SIGPIPE signal
 **************************************************************************/
static TSMgmtError
main_socket_reconnect()
{
  TSMgmtError err;

  // connects to TM and does all necessary event updates required
  err = reconnect();
  if (err != TS_ERR_OKAY) {
    return err;
  }

  // makes sure the descriptor is writable
  if (mgmt_write_timeout(main_socket_fd, MAX_TIME_WAIT, 0) <= 0) {
    return TS_ERR_NET_TIMEOUT;
  }

  return TS_ERR_OKAY;
}

static TSMgmtError
socket_write_conn(int fd, const void *msg_buf, size_t bytes)
{
  size_t byte_wrote = 0;

  // makes sure the descriptor is writable
  if (mgmt_write_timeout(fd, MAX_TIME_WAIT, 0) <= 0) {
    return TS_ERR_NET_TIMEOUT;
  }

  // write until we fulfill the number
  while (byte_wrote < bytes) {
    ssize_t ret = write(fd, static_cast<const char *>(msg_buf) + byte_wrote, bytes - byte_wrote);

    if (ret == 0) {
      return TS_ERR_NET_EOF;
    }

    if (ret < 0) {
      if (mgmt_transient_error()) {
        continue;
      } else {
        return TS_ERR_NET_WRITE;
      }
    }

    // we are all good here
    byte_wrote += ret;
  }

  return TS_ERR_OKAY;
}

TSMgmtError
mgmtapi_sender::send(void *msg, size_t msglen) const
{
  const unsigned tries = 5;
  TSMgmtError err;

  for (unsigned i = 0; i < tries; ++i) {
    err = socket_write_conn(this->fd, msg, msglen);
    if (err == TS_ERR_OKAY) {
      return err;
    }

    // clean-up sockets
    close(main_socket_fd);
    close(event_socket_fd);
    main_socket_fd  = -1;
    event_socket_fd = -1;

    err = main_socket_reconnect();
    if (err != TS_ERR_OKAY) {
      return err;
    }
  }

  return TS_ERR_NET_ESTABLISH; // can't establish connection
}

/**********************************************************************
 * socket_test_thread
 *
 * purpose: continually polls to check if local end of socket connection
 *          is still open; this thread is created when the client calls
 *          Init() to initialize the API; and will not
 *          die until the client process dies
 * input: none
 * output: if other end is closed, it reconnects to TM
 * notes: uses the current main_socket_fd because the main_socket_fd could be
 *        in flux; basically it is possible that the client will reconnect
 *        from some other call, thus making the main_socket_fd actually
 *        valid when socket_test is called
 * reason: decided to create this "watcher" thread for the socket
 *         connection because if TM is restarted or the client process
 *         is started before the TM process, then the client will not
 *         be able to receive any event notifications until a "request"
 *         is issued. In order to prevent losing an event notifications
 *         that are called in between the time TM is restarted and
 *         client issues a first request, we just run this thread which
 *         will try to reconnect to TM if it is not already connected
 **********************************************************************/
void *
socket_test_thread(void *)
{
  // loop until client process dies
  while (true) {
    if (main_socket_fd == -1 || !socket_test(main_socket_fd)) {
      // ASSUMES that in between the time the socket_test is made
      // and this reconnect call is made, the main_socket_fd remains
      // the same (eg. no one else called reconnect to TM successfully!!
      // WHAT IF in between this time, the client had issued a request
      // calling socket_write_conn which then calls reconnect(); then
      // reconnect will return an "ALREADY CONNECTED" error when it
      // tries to connect, and on the next loop iteration, the socket_test
      // will actually pass because main_socket_fd is valid!!
      reconnect();
    }

    sleep(5);
  }

  ink_thread_exit(nullptr);
  return nullptr;
}

/**********************************************************************
 * MARSHALL REQUESTS
 **********************************************************************/

/*------ events -------------------------------------------------------*/

/**********************************************************************
 * send_register_all_callbacks
 *
 * purpose: determines all events which have at least one callback registered
 *          and sends message to notify TM that this client has a callback
 *          registered for each event
 * input: None
 * output: return TS_ERR_OKAY only if ALL events sent okay
 * 1) get list of all events with callbacks
 * 2) for each event, send a EVENT_REG_CALLBACK message
 **********************************************************************/
TSMgmtError
send_register_all_callbacks(int fd, CallbackTable *cb_table)
{
  LLQ *events_with_cb;
  TSMgmtError err, send_err = TS_ERR_FAIL;
  bool no_errors = true; // set to false if one send is not okay

  events_with_cb = get_events_with_callbacks(cb_table);
  // need to check that the list has all the events registered
  if (!events_with_cb) { // all events have registered callback
    OpType optype                 = OpType::EVENT_REG_CALLBACK;
    MgmtMarshallString event_name = nullptr;

    err = MGMTAPI_SEND_MESSAGE(fd, OpType::EVENT_REG_CALLBACK, &optype, &event_name);
    if (err != TS_ERR_OKAY) {
      return err;
    }
  } else {
    int num_events = queue_len(events_with_cb);
    // iterate through the LLQ and send request for each event
    for (int i = 0; i < num_events; i++) {
      OpType optype                 = OpType::EVENT_REG_CALLBACK;
      MgmtMarshallInt event_id      = *static_cast<int *>(dequeue(events_with_cb));
      MgmtMarshallString event_name = get_event_name(event_id);

      if (event_name) {
        err = MGMTAPI_SEND_MESSAGE(fd, OpType::EVENT_REG_CALLBACK, &optype, &event_name);
        ats_free(event_name); // free memory
        if (err != TS_ERR_OKAY) {
          send_err  = err; // save the type of send error
          no_errors = false;
        }
      }
      // REMEMBER: WON"T GET A REPLY from TM side!
    }
  }

  if (events_with_cb) {
    delete_queue(events_with_cb);
  }

  if (no_errors) {
    return TS_ERR_OKAY;
  } else {
    return send_err;
  }
}

/**********************************************************************
 * send_unregister_all_callbacks
 *
 * purpose: determines all events which have no callback registered
 *          and sends message to notify TM that this client has no
 *          callbacks registered for that event
 * input: None
 * output: TS_ERR_OKAY only if all send requests are okay
 **********************************************************************/
TSMgmtError
send_unregister_all_callbacks(int fd, CallbackTable *cb_table)
{
  LLQ *events_with_cb; // list of events with at least one callback
  int reg_callback[NUM_EVENTS];
  TSMgmtError err, send_err = TS_ERR_FAIL;
  bool no_errors = true; // set to false if at least one send fails

  // init array so that all events don't have any callbacks
  for (int &i : reg_callback) {
    i = 0;
  }

  events_with_cb = get_events_with_callbacks(cb_table);
  if (!events_with_cb) { // all events have a registered callback
    return TS_ERR_OKAY;
  } else {
    int num_events = queue_len(events_with_cb);
    // iterate through the LLQ and mark events that have a callback
    for (int i = 0; i < num_events; i++) {
      int event_id           = *static_cast<int *>(dequeue(events_with_cb));
      reg_callback[event_id] = 1; // mark the event as having a callback
    }
    delete_queue(events_with_cb);
  }

  // send message to TM to mark unregister
  for (int k = 0; k < NUM_EVENTS; k++) {
    if (reg_callback[k] == 0) { // event has no registered callbacks
      OpType optype                 = OpType::EVENT_UNREG_CALLBACK;
      MgmtMarshallString event_name = get_event_name(k);

      err = MGMTAPI_SEND_MESSAGE(fd, OpType::EVENT_UNREG_CALLBACK, &optype, &event_name);
      ats_free(event_name);
      if (err != TS_ERR_OKAY) {
        send_err  = err; // save the type of the sending error
        no_errors = false;
      }
      // REMEMBER: WON"T GET A REPLY!
      // only the event_poll_thread_main does any reading of the event_socket;
      // so DO NOT parse reply b/c a reply won't be sent
    }
  }

  if (no_errors) {
    return TS_ERR_OKAY;
  } else {
    return send_err;
  }
}

/**********************************************************************
 * UNMARSHAL REPLIES
 **********************************************************************/

TSMgmtError
parse_generic_response(OpType optype, int fd)
{
  TSMgmtError err;
  MgmtMarshallInt ival;
  MgmtMarshallData data = {nullptr, 0};

  err = recv_mgmt_message(fd, data);
  if (err != TS_ERR_OKAY) {
    return err;
  }

  err = recv_mgmt_response(data.ptr, data.len, optype, &ival);
  ats_free(data.ptr);

  if (err != TS_ERR_OKAY) {
    return err;
  }

  return static_cast<TSMgmtError>(ival);
}

/**********************************************************************
 * event_poll_thread_main
 *
 * purpose: thread listens on the client's event socket connection;
 *          only reads from the event_socket connection and
 *          processes EVENT_NOTIFY messages; each time client
 *          makes new event-socket connection to TM, must launch
 *          a new event_poll_thread_main thread
 * input:   arg - contains the socket_fd to listen on
 * output:  NULL - if error
 * notes:   each time the client's socket connection to TM is reset
 *          a new thread will be launched as old one dies; there are
 *          only two places where a new thread is created:
 *          1) when client first connects (TSInit call)
 *          2) client reconnects() due to a TM restart
 * Uses blocking socket; so blocks until receives an event notification.
 * Shouldn't need to use select since only waiting for a notification
 * message from event_callback_main thread!
 **********************************************************************/
void *
event_poll_thread_main(void *arg)
{
  int sock_fd;

  sock_fd = *(static_cast<int *>(arg)); // should be same as event_socket_fd

  // the sock_fd is going to be the one we listen for events on
  while (true) {
    TSMgmtError ret;
    TSMgmtEvent *event = nullptr;

    MgmtMarshallData reply = {nullptr, 0};
    OpType optype;
    MgmtMarshallString name = nullptr;
    MgmtMarshallString desc = nullptr;

    // possible sock_fd is invalid if TM restarts and client reconnects
    if (sock_fd < 0) {
      break;
    }

    // Just wait until we get an event or error. The 0 return from select(2)
    // means we timed out ...
    if (mgmt_read_timeout(main_socket_fd, MAX_TIME_WAIT, 0) == 0) {
      continue;
    }

    ret = recv_mgmt_message(main_socket_fd, reply);
    if (ret != TS_ERR_OKAY) {
      break;
    }

    ret = recv_mgmt_request(reply.ptr, reply.len, OpType::EVENT_NOTIFY, &optype, &name, &desc);
    ats_free(reply.ptr);

    if (ret != TS_ERR_OKAY) {
      ats_free(name);
      ats_free(desc);
      break;
    }

    ink_assert(optype == OpType::EVENT_NOTIFY);

    // The new event takes ownership of the message strings.
    event              = TSEventCreate();
    event->name        = name;
    event->id          = get_event_id(name);
    event->description = desc;

    // got event notice; spawn new thread to handle the event's callback functions
    ink_thread_create(nullptr, event_callback_thread, (void *)event, 0, 0, nullptr);
  }

  ink_thread_exit(nullptr);
  return nullptr;
}

/**********************************************************************
 * event_callback_thread
 *
 * purpose: Given an event, determines and calls the registered cb functions
 *          in the CallbackTable for remote events
 * input: arg - should be an TSMgmtEvent with the event info sent from TM msg
 * output: returns when done calling all the callbacks
 * notes: None
 **********************************************************************/
static void *
event_callback_thread(void *arg)
{
  TSMgmtEvent *event_notice;
  EventCallbackT *event_cb;
  int index;

  event_notice = static_cast<TSMgmtEvent *>(arg);
  index        = event_notice->id;
  LLQ *func_q; // list of callback functions need to call

  func_q = create_queue();
  if (!func_q) {
    TSEventDestroy(event_notice);
    return nullptr;
  }

  // obtain lock
  ink_mutex_acquire(&remote_event_callbacks->event_callback_lock);

  TSEventSignalFunc cb;

  // check if we have functions to call
  if (remote_event_callbacks->event_callback_l[index] && (!queue_is_empty(remote_event_callbacks->event_callback_l[index]))) {
    int queue_depth = queue_len(remote_event_callbacks->event_callback_l[index]);

    for (int i = 0; i < queue_depth; i++) {
      event_cb = static_cast<EventCallbackT *>(dequeue(remote_event_callbacks->event_callback_l[index]));
      cb       = event_cb->func;
      enqueue(remote_event_callbacks->event_callback_l[index], event_cb);
      enqueue(func_q, reinterpret_cast<void *>(cb)); // add callback function only to list
    }
  }
  // release lock
  ink_mutex_release(&remote_event_callbacks->event_callback_lock);

  // execute the callback function
  while (!queue_is_empty(func_q)) {
    cb = reinterpret_cast<TSEventSignalFunc>(dequeue(func_q));
    (*cb)(event_notice->name, event_notice->description, event_notice->priority, nullptr);
  }

  // clean up event notice
  TSEventDestroy(event_notice);
  delete_queue(func_q);

  // all done!
  ink_thread_exit(nullptr);
  return nullptr;
}