1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
|
#
# One ZeekControl node.
#
import os
import copy
from ZeekControl import doc
class Node:
"""Class representing one node of the ZeekControl maintained setup. In
standalone mode, there's always exactly one node of type ``standalone``. In
a cluster setup, there is zero or more of type ``logger``, exactly one of
type ``manager``, one or more of type ``proxy``, and zero or more of type
``worker``. The manager will handle writing logs if there are no loggers
defined in a cluster.
A ``Node`` object has a number of keys with values that are set via the
``node.cfg`` file and can be accessed directly (from a plugin) via
corresponding Python attributes (e.g., ``node.name``):
``name`` (string)
The name of the node, which corresponds to the ``[<name>]``
section in ``node.cfg``.
``type`` (string)
The type of the node. In a standalone configuration, the only
allowed type is ``standalone``. In a cluster configuration, the
type must be one of: ``logger``, ``manager``, ``proxy``,
or ``worker``.
``host`` (string)
The hostname or IP address of the system the node is
running on. Every node must specify a host.
``interface`` (string)
The network interface for the Zeek worker (or standalone node) to
use; empty if not set.
``lb_procs`` (integer)
The number of clustered Zeek workers you'd like to start up. If
specified, this number must be greater than zero and a load
balancing method must also be specified. This option is valid only
for worker nodes.
``lb_method`` (string)
The load balancing method to distribute packets to all of the
Zeek workers. This must be one of: ``pf_ring``, ``myricom``,
``custom``, or ``interfaces``. This option can have a value
only if the ``lb_procs`` option has a value.
``lb_interfaces`` (string)
A comma-separated list of network interface names for the Zeek
worker to use. The number of interfaces in this list must
equal the value of the ``lb_procs`` option.
This option can be specified only when the load balancing method
is ``interfaces``.
``pin_cpus`` (string)
A comma-separated list of CPU numbers to which the node's Zeek
processes will be pinned. If not specified, then CPU pinning will
not be used for this node. This option is supported only on
Linux and FreeBSD, and is ignored on all other platforms.
CPU numbering starts at zero (e.g.,
the only valid CPU numbers for a machine with one dual-core
processor would be 0 and 1). If the length of this list does not
match the number of Zeek processes for this node, then some CPUs
could have zero (if too many CPU numbers are specified) or more
than one (if not enough CPU numbers are specified) Zeek processes
pinned to them. Only the specified CPU numbers will be used,
regardless of whether additional CPU cores exist.
``env_vars`` (string)
A comma-separated list of environment variables to set when
running Zeek (e.g., ``env_vars=VAR1=1,VAR2=2``). These
node-specific values override any global values specified in
the ``zeekctl.cfg`` file.
``aux_scripts`` (string)
Any node-specific Zeek script configured for this node.
``zone_id`` (string)
If ZeekControl is managing a cluster comprised of nodes
using non-global IPv6 addresses, then this configures the
:rfc:`4007` ``zone_id`` string that the node associates with
the common zone that all cluster nodes are a part of. This
identifier may differ between nodes.
Any attribute that is not defined in ``node.cfg`` will be empty.
In addition, plugins can override `Plugin.nodeKeys`_ to define their own
node keys, which can then be likewise set in ``node.cfg``. The key names
will be prepended with the plugin's `Plugin.prefix`_ (e.g., for the plugin
``test``, the node key ``foo`` is set by adding ``test.foo=value`` to
``node.cfg``).
Finally, a Node object has the following methods that can be called
from a plugin:
"""
# Valid keys in nodes file. The values will be stored in attributes of the
# same name. Custom keys can be add via addKey().
_keys = {"type": 1, "host": 1, "interface": 1, "aux_scripts": 1,
"zeekbase": 1, "ether": 1, "zone_id": 1,
"lb_procs": 1, "lb_method": 1, "lb_interfaces": 1,
"pin_cpus": 1, "env_vars": 1, "count": 1}
def __init__(self, config, name):
"""Instantiates a new node of the given name."""
self.name = name
self._config = config
for key in Node._keys:
self.__dict__[key] = ""
def __str__(self):
return self.name
def copy(self):
n = Node(self._config, self.name)
for key in self.__dict__:
if key.startswith("_"):
# This is to avoid copying _config, which causes problems.
setattr(n, key, getattr(self, key))
else:
# Must make a copy of some config items (e.g. env_vars) so that
# changes to the value only affect one node.
setattr(n, key, copy.copy(getattr(self, key)))
return n
def items(self):
"""Returns a list of (key, value) tuples, sorted by key, of a node."""
def tostr(v):
if isinstance(v, dict):
return ",".join(["%s=%s" % (key, val) for (key, val) in sorted(v.items())])
else:
return str(v)
return [(k, tostr(self.__dict__[k])) for k in sorted(self.__dict__.keys())]
@doc.api
def describe(self):
"""Returns an extended string representation of the node including all
its keys with values (sorted by key)."""
def fmt(v):
if isinstance(v, list):
v = ",".join(v)
elif isinstance(v, dict):
v = ",".join(["%s=%s" % (key, val) for (key, val) in sorted(v.items())])
return v
# Do not output attributes starting with underscore, because they are
# for internal use and don't provide useful information to the user.
return ("%16s - " % self.name) + " ".join(["%s=%s" % (k, fmt(self.__dict__[k])) for k in sorted(self.__dict__.keys()) if not k.startswith("_")])
def to_dict(self):
d = dict(self.items())
d["name"] = self.name
d["description"] = self.describe()
return d
@doc.api
def cwd(self):
"""Returns a string with the node's working directory."""
return os.path.join(self._config.spooldir, self.name)
def setPID(self, pid):
"""Stores the process ID of the node's Zeek process."""
key = "%s-pid" % self.name
self._config.set_state(key, pid)
key = "%s-host" % self.name
self._config.set_state(key, self.host)
@doc.api
def getPID(self):
"""Returns the process ID of the node's Zeek process if running, and
None otherwise."""
key = "%s-pid" % self.name
return self._config.get_state(key)
def clearPID(self):
"""Clears the stored process ID for the node's Zeek process, indicating
that it is no longer running."""
key = "%s-pid" % self.name
self._config.set_state(key, None)
def setCrashed(self):
"""Marks node's Zeek process as having terminated unexpectedly."""
key = "%s-crashed" % self.name
self._config.set_state(key, True)
def clearCrashed(self):
"""Clears the mark for the node's Zeek process having terminated
unexpectedly."""
key = "%s-crashed" % self.name
self._config.set_state(key, False)
@doc.api
def hasCrashed(self):
"""Returns True if the node's Zeek process has exited abnormally."""
key = "%s-crashed" % self.name
val = self._config.get_state(key)
if val is None:
val = False
return val
def getExpectRunning(self):
"""Returns True if we expect the node's Zeek process to be running."""
key = "%s-expect-running" % self.name
val = self._config.get_state(key)
if val is None:
val = False
return val
def setExpectRunning(self, val):
key = "%s-expect-running" % self.name
self._config.set_state(key, val)
def setPort(self, port):
"""Set the Zeek port this node is using."""
key = "%s-port" % self.name
self._config.set_state(key, port)
@doc.api
def getPort(self):
"""Returns an integer with the port number that this node's
communication system is listening on for incoming connections, or -1 if
no such port has been set yet.
"""
key = "%s-port" % self.name
return self._config.get_state(key) or -1
@staticmethod
def addKey(kw):
"""Adds a supported node key. This is used by the PluginRegistry to
register custom keys."""
# We need to convert to lowercase here because Python's configparser
# automatically converts keys to lowercase when reading node.cfg.
Node._keys[kw.lower()] = 1
# The sorting order for node types used by the sorting functions below
# (specifying the sort order explicitly is useful if we don't want to
# use alphabetical order).
_typeorder = ("standalone", "logger", "manager", "proxy", "worker")
# Sorting key function for a list of nodes.
def sortnode(n):
try:
return _typeorder.index(n.type), n.count
except ValueError:
return len(_typeorder), n.count
# Sorting key function for a list of tuples, where the first tuple element is
# a node.
def sorttuple(t):
n = t[0]
try:
return _typeorder.index(n.type), n.count
except ValueError:
return len(_typeorder), n.count
# Given a list of nodes (all of the same type), return a string that describes
# the list of nodes (in either singular or plural form). This string is
# just for informational output, and doesn't have any other use or meaning.
# For standalone node type, the node name is returned instead.
def nodes_describe(nodes):
nodetype = nodes[0].type
if nodetype == "standalone":
return nodes[0].name
elif nodetype == "manager":
return "manager"
elif nodetype == "logger":
return "logger%s" % ("" if len(nodes) == 1 else "s")
elif nodetype == "proxy":
return "prox%s" % ("y" if len(nodes) == 1 else "ies")
elif nodetype == "worker":
return "worker%s" % ("" if len(nodes) == 1 else "s")
# Return a list of all node types.
def node_types():
return ["logger", "manager", "proxy", "worker", "standalone"]
# Check if the given node is a certain type.
def is_standalone(n):
return n.type == "standalone"
def is_manager(n):
return n.type == "manager"
def is_logger(n):
return n.type == "logger"
def is_proxy(n):
return n.type == "proxy"
def is_worker(n):
return n.type == "worker"
# Given a list of nodes, return separate lists for each type of node.
def separate_types(nodes):
loggers = []
manager = []
proxies = []
workers = []
for n in nodes:
if n.type == "worker":
workers += [n]
elif n.type == "proxy":
proxies += [n]
elif n.type in ("manager", "standalone"):
manager += [n]
elif n.type == "logger":
loggers += [n]
return loggers, manager, proxies, workers
# Map of node groups to node types (here, "_ALL_" is for internal use only and
# matches all node types).
grouptype = {"all": "_ALL_",
"loggers": "logger",
"manager": "manager",
"proxies": "proxy",
"workers": "worker"}
# Return a list of all node groups. These are for convenience when using
# zeekctl commands (e.g. "zeekctl start workers").
def node_groups():
return list(grouptype.keys())
# Return the node type (or "_ALL_", which matches all node types) of a
# specified group name. If the "tag" doesn't match any group name, then None
# is returned.
def group_type(tag):
return grouptype.get(tag)
# Return the name of a node group.
def manager_group():
return "manager"
def logger_group():
return "loggers"
def proxy_group():
return "proxies"
def worker_group():
return "workers"
|