Description: drive-full-checker The admin documentation provides a documentation on how to "prevent[ing] disk full scenarios" over here: https://docs.openstack.org/swift/latest/admin_guide.html#preventing-disk-full-scenarios . Even if the doc provides an actual example, this example is written in Python 2, and its implementation is incomplete. . This patch intend to fill the gap, and allow administrator to use an official implementation of a new "swift-drive-full-checker" tool from /usr/bin directly. Once done, we intend to also patch puppet-swift to use this new tool. Author: Thomas Goirand Forwarded: https://review.opendev.org/c/openstack/swift/+/907523 Last-Update: 2024-02-11 Index: swift/etc/drive-full-checker.conf-sample =================================================================== --- /dev/null +++ swift/etc/drive-full-checker.conf-sample @@ -0,0 +1,51 @@ +[drive-full-checker] +# Mount point of your storage. (string value) +# device_dir = /srv/node +# +# Path to the rsyncd.conf file to manage. (string value) +# rsyncd_conf_path = /etc/rsyncd.conf +# +# You can specify default log routing here if you want: +# log_name = drive-audit +# log_facility = LOG_LOCAL0 +# log_level = INFO +# log_address = /dev/log +# The following caps the length of log lines to the value given; no limit if +# set to 0, the default. +# log_max_line_length = 0 +# +# By default, drive-full-checker logs only to syslog. Setting this option True +# makes drive-audit log to console in addition to syslog. +# log_to_console = False +# + +# Max connections to the Account rsync backend. (integer value) +#account_max_connections = 8 + +# Account server reserved space in GiB. (integer value) +#account_reserved_space = 100 + +# Account section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply +# write "account". (string value) +#account_rsyncd_section_name = " account_{} " + +# Max connections to the Container rsync backend. (integer value) +#container_max_connections = 8 + +# Container server reserved space in GiB. (integer value) +#container_reserved_space = 100 + +# Container section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply +# write "container". (string value) +#container_rsyncd_section_name = " container_{} " + +# Max connections to the Object rsync backend. (integer value) +#object_max_connections = 8 + +# Object server reserved space in GiB. (integer value) +#object_reserved_space = 100 + +# Object section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply +# write "object". (string value) +#object_rsyncd_section_name = " object_{} " + Index: swift/releasenotes/notes/disk-full-checker-d4850f2fb479bb36.yaml =================================================================== --- /dev/null +++ swift/releasenotes/notes/disk-full-checker-d4850f2fb479bb36.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + A new /usr/bin/swift-drive-full-checker utility is now provided by Swift. + This tool watches for partitions in /srv/node (or wherever you configured + it) for drive full scenarios. If a drive has less than the configured + amount of space available, swift-drive-full-checker will amend the matching + entry in rsyncd.conf and set it with `max connections = -1`, so that rsync + will gracefully refuse incoming connections, and the sending replicator + process will re-attempt duplicating data when space becomes available. + Typically, swift-drive-full-checker will be called from a cron job every + 5 minutes (at least), with the parameters defining the amount of data + reserved for each type of data, and the number of connection allowed if + the data partition is not full. Index: swift/setup.cfg =================================================================== --- swift.orig/setup.cfg +++ swift/setup.cfg @@ -90,6 +90,7 @@ console_scripts = swift-recon-cron = swift.cli.recon_cron:main swift-reconciler-enqueue = swift.cli.reconciler_enqueue:main swift-reload = swift.cli.reload:main + swift-drive-full-checker = swift.cli.drive_full_checker:main swift-ring-builder = swift.cli.ringbuilder:error_handling_main swift-ring-builder-analyzer = swift.cli.ring_builder_analyzer:main swift-ring-composer = swift.cli.ringcomposer:main Index: swift/swift/cli/drive_full_checker.py =================================================================== --- /dev/null +++ swift/swift/cli/drive_full_checker.py @@ -0,0 +1,207 @@ +# Copyright (c) 2024, Thomas Goirand +# Copyright (c) 2024, Philippe Serafin +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script check a mounted device is full, and disable +# its matching rsync module if that is the case. The disk +# full limit is the first argument. + +import argparse +import configparser +import io +import os +import shutil +import sys + +from six.moves.configparser import ConfigParser + +from swift.common.utils import config_true_value, ismount, get_logger + +GiB = 1024 * 1024 * 1024 + + +# Params for this function: +# logger: ref to the logger +# cp: config parser object containing the rsyncd.conf representation +# srvnode_dir: name of the drive we're inspecting (for example: sdb) +# free: bytes available in the current srvnode_dir that we're inspecting +# sec_name: name of the rsyncd.conf section we may need to patch +# rs: bytes reserved space in srvnode_dir +# mc: "normal" max connections (ie: when partition isn't full) +# for the given dir entry +def _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free, sec_name, rs, mc): + # Calculate section name (ie: replace '{}' by drive name if present) + if '{}' in sec_name: + search_str = sec_name.format(srvnode_dir).strip('"') + else: + search_str = sec_name.strip('"') + + # If referenced in the rsyncd.conf + # In old setup (Python 2), calling config_parser['something'] + # raises an exception if the something section is not present + # in the config file. Which is why we must do try/except. + # I believe this try/except can be removed on more recent + # Python3 based setups. + try: + if cp[search_str]: + # If partition is full (ie: current_free_space < reserved_space), + # set 'max connections' to -1 to disable rsync + if free < rs: + cm = -1 + else: + cm = mc + + if int(cp[search_str]['max connections']) != cm: + if cm == -1: + logger.info('Disabling ' + search_str) + else: + logger.info('Enabling ' + search_str) + cp[search_str]['max connections'] = str(cm) + except KeyError: + pass + + return cp + + +def configure_rsyncd_conf(account_rs, account_mc, account_secname, + container_rs, container_mc, container_secname, + object_rs, object_mc, object_secname, + storage_p, rsyncd_p, logger, sf=None): + # Load the rsyncd.conf file, adding + # a fake global section. + fake_section = '[fake_section_to_please_configobj]\n' + source_file = sf if sf else rsyncd_p + try: + with open(source_file, 'r') as f: + file_content = fake_section + f.read() + except Exception as err: + print("Unexpected error reading {}: {}".format(source_file, err)) + return 1 + + cp = configparser.RawConfigParser() + if sys.version_info[0] == 2: + cp.read_string(file_content.decode('unicode-escape')) + else: + cp.read_string(file_content) + + # For all dirs in /srv/node + for srvnode_dir in os.listdir(storage_p): + dirpath = os.path.join(storage_p, srvnode_dir) + # If the dir is mounted + if ismount(dirpath): + # Get free space of the partition + # shutil.disk_usage can be mocked in tests. + if sys.version_info[0] == 2: + space = os.statvfs(dirpath) + free = (space.f_bsize * space.f_bavail) + else: + free = shutil.disk_usage(dirpath).free + + # Patch all 3 types of rsync module (a+c+o) + cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free, + account_secname, account_rs, + account_mc) + + cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free, + container_secname, container_rs, + container_mc) + + cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free, + object_secname, object_rs, + object_mc) + + # Prepare our rsyncd.conf file before writing + iow = io.StringIO() + cp.write(iow) + file_out = iow.getvalue().replace(fake_section, '') + + try: + with open(rsyncd_p, 'w') as f: + f.write(file_out) + except Exception as err: + logger.error("Unexpected error {}".format(err)) + return 1 + + +def main(): + # Cli OPT parsing + parser = argparse.ArgumentParser(prog='swift-drive-full-checker', + description='Check if the drives of a ' + 'swift node are full, and ' + 'switches /etc/rsyncd.conf ' + '"max connections" ' + 'accordingly.', + epilog='(c) 2024, Thomas Goirand, ' + 'Philippe Serafin & Infomaniak ' + 'Networks.') + parser.add_argument('-c', '--config-file', + default='/etc/swift/drive-full-checker.conf', + help='Path to the drive-full-checker.conf. Default to ' + '/etc/swift/drive-full-checker.conf') + parser.add_argument('-s', '--source-file', + default='/etc/rsyncd.conf', + help='Path to the source file. Default to ' + '/etc/rsyncd.conf') + args = parser.parse_args() + + # disk-full-checker config file parsing + c = ConfigParser() + if not c.read(args.config_file): + print("Unable to read config file %s" % args.conf_path) + sys.exit(1) + + CONF = dict(c.items('drive-full-checker')) + device_dir = CONF.get('device_dir', '/srv/node') + rsyncd_conf_path = CONF.get('rsyncd_conf_path', '/etc/rsyncd.conf') + + account_max_connections = int(CONF.get('account_max_connections', 8)) + account_reserved_space = int(CONF.get('account_reserved_space', 100)) * GiB + account_rsyncd_section_name = CONF.get('account_rsyncd_section_name', + ' account_{} ') + + container_max_connections = int(CONF.get('container_max_connections', 8)) + container_reserved_space = (int(CONF.get('container_reserved_space', 100)) + * GiB) + container_rsyncd_section_name = CONF.get('container_rsyncd_section_name', + ' container_{} ') + + object_max_connections = int(CONF.get('object_max_connections', 8)) + object_reserved_space = int(CONF.get('object_reserved_space', 100)) * GiB + object_rsyncd_section_name = CONF.get('object_rsyncd_section_name', + ' object_{} ') + + # logging facility setup + log_to_console = config_true_value(CONF.get('log_to_console', False)) + CONF['log_name'] = CONF.get('log_name', 'drive-full-checker') + logger = get_logger(CONF, log_to_console=log_to_console, + log_route='drive-full-checker') + + return configure_rsyncd_conf(account_reserved_space, + account_max_connections, + account_rsyncd_section_name, + container_reserved_space, + container_max_connections, + container_rsyncd_section_name, + object_reserved_space, + object_max_connections, + object_rsyncd_section_name, + device_dir, + rsyncd_conf_path, + logger, + args.source_file) + + +if __name__ == "__main__": + sys.exit(main()) Index: swift/test/unit/cli/test_drive_full_checker.py =================================================================== --- /dev/null +++ swift/test/unit/cli/test_drive_full_checker.py @@ -0,0 +1,174 @@ +# Copyright (c) 2024, Thomas Goirand +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import filecmp +import mock +import os +import tempfile +import unittest +import shutil +import sys + +from test.debug_logger import debug_logger +from swift.cli import drive_full_checker + +GiB = 1024 * 1024 * 1024 + +if sys.version_info[0] == 2: + # os.statvfs is removed from Python >= 3 + freespace_func = 'os.statvfs' +else: + # shutil.disk_usage doesn't exist in Python <= 2 + freespace_func = 'shutil.disk_usage' + + +class TestContainerDeleter(unittest.TestCase): + def setUp(self): + self.logger = debug_logger() + + def _write_rsyncd_conf(self, path, max_conn): + rsyncdconf = """pid file = /var/run/rsyncd.pid +uid = nobody +gid = nobody +use chroot = no +log format = %t %a %m %f %b +syslog facility = local3 +timeout = 300 +address = 192.168.100.2 + +[ account_sdb ] +path = /srv/node +read only = false +write only = no +list = yes +uid = swift +gid = swift +incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +max connections = {max_conn} +timeout = 0 +lock file = /var/lock/account_sdb.lock + +[ container_sdb ] +path = /srv/node +read only = false +write only = no +list = yes +uid = swift +gid = swift +incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +max connections = {max_conn} +timeout = 0 +lock file = /var/lock/container_sdb.lock + +[ object_sdb ] +path = /srv/node +read only = false +write only = no +list = yes +uid = swift +gid = swift +incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r +max connections = {max_conn} +timeout = 0 +lock file = /var/lock/object_sdb.lock + +""" + f = os.open(path, os.O_RDWR | os.O_CREAT) + if sys.version_info[0] == 2: + os.write(f, rsyncdconf.format(max_conn=max_conn)) + else: + os.write(f, bytes(rsyncdconf.format(max_conn=max_conn), 'utf-8')) + os.close(f) + + @mock.patch.object(drive_full_checker, 'ismount', return_value=True) + @mock.patch(freespace_func) + def test_drive_full(self, mock_freespace_func, os_path_ismount): + # Create a temp folder to run our tests + tmpdirname = tempfile.mkdtemp() + + storagepath = tmpdirname + '/srvnode' + os.mkdir(storagepath) + os.mkdir(storagepath + '/sdb') + + rsyncdpath = tmpdirname + "/rsyncd.conf" + + # Write a first rsyncd.conf with 8 connections for a,c,o + self._write_rsyncd_conf(rsyncdpath, 8) + + if sys.version_info[0] == 2: + retval = collections.namedtuple('statvfs_result', + 'f_bsize f_bavail') + mock_freespace_func.return_value = retval(10, 10) + else: + retval = collections.namedtuple('usage', 'total used free') + # This says: 10 bytes remaining + mock_freespace_func.return_value = retval(10, 10, 10) + + drive_full_checker.configure_rsyncd_conf(10 * GiB, 8, ' account_{} ', + 10 * GiB, 8, ' container_{} ', + 10 * GiB, 8, ' object_{} ', + storagepath, rsyncdpath, + self.logger) + + should_be_rsyncdconf = tmpdirname + "/rsyncd_should_be.conf" + self._write_rsyncd_conf(should_be_rsyncdconf, -1) + + # Assert that rsyncd.conf and rsyncd_should_be.conf are the same + self.assertTrue(filecmp.cmp(rsyncdpath, should_be_rsyncdconf)) + + shutil.rmtree(tmpdirname) + + @mock.patch.object(drive_full_checker, 'ismount', return_value=True) + @mock.patch(freespace_func) + def test_drive_with_space(self, mock_freespace_func, os_path_ismount): + # Create a temp folder to run our tests + tmpdirname = tempfile.mkdtemp() + + storagepath = tmpdirname + '/srvnode' + os.mkdir(storagepath) + os.mkdir(storagepath + '/sdb') + + rsyncdpath = tmpdirname + "/rsyncd.conf" + + # Write a first rsyncd.conf with 8 connections for a,c,o + self._write_rsyncd_conf(rsyncdpath, -1) + + if sys.version_info[0] == 2: + retval = collections.namedtuple('statvfs_result', + 'f_bsize f_bavail') + mock_freespace_func.return_value = retval(10 * GiB, 10 * GiB) + else: + retval = collections.namedtuple('usage', 'total used free') + # This says: 10 bytes remaining + mock_freespace_func.return_value = retval(10 * GiB, + 10 * GiB, + 10 * GiB) + + drive_full_checker.configure_rsyncd_conf(10, 8, ' account_{} ', + 10, 8, ' container_{} ', + 10, 8, ' object_{} ', + storagepath, rsyncdpath, + self.logger) + + should_be_rsyncdconf = tmpdirname + "/rsyncd_should_be.conf" + self._write_rsyncd_conf(should_be_rsyncdconf, 8) + + # Assert that rsyncd.conf and rsyncd_should_be.conf are the same + self.assertTrue(filecmp.cmp(rsyncdpath, should_be_rsyncdconf)) + + shutil.rmtree(tmpdirname)