Description: drive-full-checker
 The admin documentation provides a documentation on how to "prevent[ing]
 disk full scenarios" over here:
 https://docs.openstack.org/swift/latest/admin_guide.html#preventing-disk-full-scenarios
 .
 Even if the doc provides an actual example, this example is written in
 Python 2, and its implementation is incomplete.
 .
 This patch intend to fill the gap, and allow administrator to use an
 official implementation of a new "swift-drive-full-checker" tool from
 /usr/bin directly. Once done, we intend to also patch puppet-swift to
 use this new tool.
Author: Thomas Goirand <zigo@debian.org>
Forwarded: https://review.opendev.org/c/openstack/swift/+/907523
Last-Update: 2024-02-11

Index: swift/etc/drive-full-checker.conf-sample
===================================================================
--- /dev/null
+++ swift/etc/drive-full-checker.conf-sample
@@ -0,0 +1,51 @@
+[drive-full-checker]
+# Mount point of your storage. (string value)
+# device_dir = /srv/node
+#
+# Path to the rsyncd.conf file to manage. (string value)
+# rsyncd_conf_path = /etc/rsyncd.conf
+#
+# You can specify default log routing here if you want:
+# log_name = drive-audit
+# log_facility = LOG_LOCAL0
+# log_level = INFO
+# log_address = /dev/log
+# The following caps the length of log lines to the value given; no limit if
+# set to 0, the default.
+# log_max_line_length = 0
+#
+# By default, drive-full-checker logs only to syslog. Setting this option True
+# makes drive-audit log to console in addition to syslog.
+# log_to_console = False
+#
+
+# Max connections to the Account rsync backend. (integer value)
+#account_max_connections = 8
+
+# Account server reserved space in GiB. (integer value)
+#account_reserved_space = 100
+
+# Account section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply
+# write "account". (string value)
+#account_rsyncd_section_name = " account_{} "
+
+# Max connections to the Container rsync backend. (integer value)
+#container_max_connections = 8
+
+# Container server reserved space in GiB. (integer value)
+#container_reserved_space = 100
+
+# Container section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply
+# write "container". (string value)
+#container_rsyncd_section_name = " container_{} "
+
+# Max connections to the Object rsync backend. (integer value)
+#object_max_connections = 8
+
+# Object server reserved space in GiB. (integer value)
+#object_reserved_space = 100
+
+# Object section name in the rsyncd.conf file. The "{}" sign will be replaced by the drive name. If not using per drive sections, simply
+# write "object". (string value)
+#object_rsyncd_section_name = " object_{} "
+
Index: swift/releasenotes/notes/disk-full-checker-d4850f2fb479bb36.yaml
===================================================================
--- /dev/null
+++ swift/releasenotes/notes/disk-full-checker-d4850f2fb479bb36.yaml
@@ -0,0 +1,14 @@
+---
+features:
+  - |
+    A new /usr/bin/swift-drive-full-checker utility is now provided by Swift.
+    This tool watches for partitions in /srv/node (or wherever you configured
+    it) for drive full scenarios. If a drive has less than the configured
+    amount of space available, swift-drive-full-checker will amend the matching
+    entry in rsyncd.conf and set it with `max connections = -1`, so that rsync
+    will gracefully refuse incoming connections, and the sending replicator
+    process will re-attempt duplicating data when space becomes available.
+    Typically, swift-drive-full-checker will be called from a cron job every
+    5 minutes (at least), with the parameters defining the amount of data
+    reserved for each type of data, and the number of connection allowed if
+    the data partition is not full.
Index: swift/setup.cfg
===================================================================
--- swift.orig/setup.cfg
+++ swift/setup.cfg
@@ -90,6 +90,7 @@ console_scripts =
     swift-recon-cron = swift.cli.recon_cron:main
     swift-reconciler-enqueue = swift.cli.reconciler_enqueue:main
     swift-reload = swift.cli.reload:main
+    swift-drive-full-checker = swift.cli.drive_full_checker:main
     swift-ring-builder = swift.cli.ringbuilder:error_handling_main
     swift-ring-builder-analyzer = swift.cli.ring_builder_analyzer:main
     swift-ring-composer = swift.cli.ringcomposer:main
Index: swift/swift/cli/drive_full_checker.py
===================================================================
--- /dev/null
+++ swift/swift/cli/drive_full_checker.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2024, Thomas Goirand <zigo@debian.org>
+# Copyright (c) 2024, Philippe Serafin <philippe.serafin@infomaniak.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This script check a mounted device is full, and disable
+# its matching rsync module if that is the case. The disk
+# full limit is the first argument.
+
+import argparse
+import configparser
+import io
+import os
+import shutil
+import sys
+
+from six.moves.configparser import ConfigParser
+
+from swift.common.utils import config_true_value, ismount, get_logger
+
+GiB = 1024 * 1024 * 1024
+
+
+# Params for this function:
+# logger: ref to the logger
+# cp: config parser object containing the rsyncd.conf representation
+# srvnode_dir: name of the drive we're inspecting (for example: sdb)
+# free: bytes available in the current srvnode_dir that we're inspecting
+# sec_name: name of the rsyncd.conf section we may need to patch
+# rs: bytes reserved space in srvnode_dir
+# mc: "normal" max connections (ie: when partition isn't full)
+#     for the given dir entry
+def _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free, sec_name, rs, mc):
+    # Calculate section name (ie: replace '{}' by drive name if present)
+    if '{}' in sec_name:
+        search_str = sec_name.format(srvnode_dir).strip('"')
+    else:
+        search_str = sec_name.strip('"')
+
+    # If referenced in the rsyncd.conf
+    # In old setup (Python 2), calling config_parser['something']
+    # raises an exception if the something section is not present
+    # in the config file. Which is why we must do try/except.
+    # I believe this try/except can be removed on more recent
+    # Python3 based setups.
+    try:
+        if cp[search_str]:
+            # If partition is full (ie: current_free_space <  reserved_space),
+            # set 'max connections' to -1 to disable rsync
+            if free < rs:
+                cm = -1
+            else:
+                cm = mc
+
+            if int(cp[search_str]['max connections']) != cm:
+                if cm == -1:
+                    logger.info('Disabling ' + search_str)
+                else:
+                    logger.info('Enabling ' + search_str)
+                cp[search_str]['max connections'] = str(cm)
+    except KeyError:
+        pass
+
+    return cp
+
+
+def configure_rsyncd_conf(account_rs, account_mc, account_secname,
+                          container_rs, container_mc, container_secname,
+                          object_rs, object_mc, object_secname,
+                          storage_p, rsyncd_p, logger, sf=None):
+    # Load the rsyncd.conf file, adding
+    # a fake global section.
+    fake_section = '[fake_section_to_please_configobj]\n'
+    source_file = sf if sf else rsyncd_p
+    try:
+        with open(source_file, 'r') as f:
+            file_content = fake_section + f.read()
+    except Exception as err:
+        print("Unexpected error reading {}: {}".format(source_file, err))
+        return 1
+
+    cp = configparser.RawConfigParser()
+    if sys.version_info[0] == 2:
+        cp.read_string(file_content.decode('unicode-escape'))
+    else:
+        cp.read_string(file_content)
+
+    # For all dirs in /srv/node
+    for srvnode_dir in os.listdir(storage_p):
+        dirpath = os.path.join(storage_p, srvnode_dir)
+        # If the dir is mounted
+        if ismount(dirpath):
+            # Get free space of the partition
+            # shutil.disk_usage can be mocked in tests.
+            if sys.version_info[0] == 2:
+                space = os.statvfs(dirpath)
+                free = (space.f_bsize * space.f_bavail)
+            else:
+                free = shutil.disk_usage(dirpath).free
+
+            # Patch all 3 types of rsync module (a+c+o)
+            cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free,
+                                         account_secname, account_rs,
+                                         account_mc)
+
+            cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free,
+                                         container_secname, container_rs,
+                                         container_mc)
+
+            cp = _patch_rsyncdconf_entry(logger, cp, srvnode_dir, free,
+                                         object_secname, object_rs,
+                                         object_mc)
+
+    # Prepare our rsyncd.conf file before writing
+    iow = io.StringIO()
+    cp.write(iow)
+    file_out = iow.getvalue().replace(fake_section, '')
+
+    try:
+        with open(rsyncd_p, 'w') as f:
+            f.write(file_out)
+    except Exception as err:
+        logger.error("Unexpected error {}".format(err))
+        return 1
+
+
+def main():
+    # Cli OPT parsing
+    parser = argparse.ArgumentParser(prog='swift-drive-full-checker',
+                                     description='Check if the drives of a '
+                                                 'swift node are full, and '
+                                                 'switches /etc/rsyncd.conf '
+                                                 '"max connections" '
+                                                 'accordingly.',
+                                     epilog='(c) 2024, Thomas Goirand, '
+                                            'Philippe Serafin & Infomaniak '
+                                            'Networks.')
+    parser.add_argument('-c', '--config-file',
+                        default='/etc/swift/drive-full-checker.conf',
+                        help='Path to the drive-full-checker.conf. Default to '
+                             '/etc/swift/drive-full-checker.conf')
+    parser.add_argument('-s', '--source-file',
+                        default='/etc/rsyncd.conf',
+                        help='Path to the source file. Default to '
+                             '/etc/rsyncd.conf')
+    args = parser.parse_args()
+
+    # disk-full-checker config file parsing
+    c = ConfigParser()
+    if not c.read(args.config_file):
+        print("Unable to read config file %s" % args.conf_path)
+        sys.exit(1)
+
+    CONF = dict(c.items('drive-full-checker'))
+    device_dir = CONF.get('device_dir', '/srv/node')
+    rsyncd_conf_path = CONF.get('rsyncd_conf_path', '/etc/rsyncd.conf')
+
+    account_max_connections = int(CONF.get('account_max_connections', 8))
+    account_reserved_space = int(CONF.get('account_reserved_space', 100)) * GiB
+    account_rsyncd_section_name = CONF.get('account_rsyncd_section_name',
+                                           ' account_{} ')
+
+    container_max_connections = int(CONF.get('container_max_connections', 8))
+    container_reserved_space = (int(CONF.get('container_reserved_space', 100))
+                                * GiB)
+    container_rsyncd_section_name = CONF.get('container_rsyncd_section_name',
+                                             ' container_{} ')
+
+    object_max_connections = int(CONF.get('object_max_connections', 8))
+    object_reserved_space = int(CONF.get('object_reserved_space', 100)) * GiB
+    object_rsyncd_section_name = CONF.get('object_rsyncd_section_name',
+                                          ' object_{} ')
+
+    # logging facility setup
+    log_to_console = config_true_value(CONF.get('log_to_console', False))
+    CONF['log_name'] = CONF.get('log_name', 'drive-full-checker')
+    logger = get_logger(CONF, log_to_console=log_to_console,
+                        log_route='drive-full-checker')
+
+    return configure_rsyncd_conf(account_reserved_space,
+                                 account_max_connections,
+                                 account_rsyncd_section_name,
+                                 container_reserved_space,
+                                 container_max_connections,
+                                 container_rsyncd_section_name,
+                                 object_reserved_space,
+                                 object_max_connections,
+                                 object_rsyncd_section_name,
+                                 device_dir,
+                                 rsyncd_conf_path,
+                                 logger,
+                                 args.source_file)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
Index: swift/test/unit/cli/test_drive_full_checker.py
===================================================================
--- /dev/null
+++ swift/test/unit/cli/test_drive_full_checker.py
@@ -0,0 +1,174 @@
+# Copyright (c) 2024, Thomas Goirand <zigo@debian.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy
+# of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import collections
+import filecmp
+import mock
+import os
+import tempfile
+import unittest
+import shutil
+import sys
+
+from test.debug_logger import debug_logger
+from swift.cli import drive_full_checker
+
+GiB = 1024 * 1024 * 1024
+
+if sys.version_info[0] == 2:
+    # os.statvfs is removed from Python >= 3
+    freespace_func = 'os.statvfs'
+else:
+    # shutil.disk_usage doesn't exist in Python <= 2
+    freespace_func = 'shutil.disk_usage'
+
+
+class TestContainerDeleter(unittest.TestCase):
+    def setUp(self):
+        self.logger = debug_logger()
+
+    def _write_rsyncd_conf(self, path, max_conn):
+        rsyncdconf = """pid file = /var/run/rsyncd.pid
+uid = nobody
+gid = nobody
+use chroot = no
+log format = %t %a %m %f %b
+syslog facility = local3
+timeout = 300
+address = 192.168.100.2
+
+[ account_sdb ]
+path = /srv/node
+read only = false
+write only = no
+list = yes
+uid = swift
+gid = swift
+incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+max connections = {max_conn}
+timeout = 0
+lock file = /var/lock/account_sdb.lock
+
+[ container_sdb ]
+path = /srv/node
+read only = false
+write only = no
+list = yes
+uid = swift
+gid = swift
+incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+max connections = {max_conn}
+timeout = 0
+lock file = /var/lock/container_sdb.lock
+
+[ object_sdb ]
+path = /srv/node
+read only = false
+write only = no
+list = yes
+uid = swift
+gid = swift
+incoming chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+outgoing chmod = Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r
+max connections = {max_conn}
+timeout = 0
+lock file = /var/lock/object_sdb.lock
+
+"""
+        f = os.open(path, os.O_RDWR | os.O_CREAT)
+        if sys.version_info[0] == 2:
+            os.write(f, rsyncdconf.format(max_conn=max_conn))
+        else:
+            os.write(f, bytes(rsyncdconf.format(max_conn=max_conn), 'utf-8'))
+        os.close(f)
+
+    @mock.patch.object(drive_full_checker, 'ismount', return_value=True)
+    @mock.patch(freespace_func)
+    def test_drive_full(self, mock_freespace_func, os_path_ismount):
+        # Create a temp folder to run our tests
+        tmpdirname = tempfile.mkdtemp()
+
+        storagepath = tmpdirname + '/srvnode'
+        os.mkdir(storagepath)
+        os.mkdir(storagepath + '/sdb')
+
+        rsyncdpath = tmpdirname + "/rsyncd.conf"
+
+        # Write a first rsyncd.conf with 8 connections for a,c,o
+        self._write_rsyncd_conf(rsyncdpath, 8)
+
+        if sys.version_info[0] == 2:
+            retval = collections.namedtuple('statvfs_result',
+                                            'f_bsize f_bavail')
+            mock_freespace_func.return_value = retval(10, 10)
+        else:
+            retval = collections.namedtuple('usage', 'total used free')
+            # This says: 10 bytes remaining
+            mock_freespace_func.return_value = retval(10, 10, 10)
+
+        drive_full_checker.configure_rsyncd_conf(10 * GiB, 8, ' account_{} ',
+                                                 10 * GiB, 8, ' container_{} ',
+                                                 10 * GiB, 8, ' object_{} ',
+                                                 storagepath, rsyncdpath,
+                                                 self.logger)
+
+        should_be_rsyncdconf = tmpdirname + "/rsyncd_should_be.conf"
+        self._write_rsyncd_conf(should_be_rsyncdconf, -1)
+
+        # Assert that rsyncd.conf and rsyncd_should_be.conf are the same
+        self.assertTrue(filecmp.cmp(rsyncdpath, should_be_rsyncdconf))
+
+        shutil.rmtree(tmpdirname)
+
+    @mock.patch.object(drive_full_checker, 'ismount', return_value=True)
+    @mock.patch(freespace_func)
+    def test_drive_with_space(self, mock_freespace_func, os_path_ismount):
+        # Create a temp folder to run our tests
+        tmpdirname = tempfile.mkdtemp()
+
+        storagepath = tmpdirname + '/srvnode'
+        os.mkdir(storagepath)
+        os.mkdir(storagepath + '/sdb')
+
+        rsyncdpath = tmpdirname + "/rsyncd.conf"
+
+        # Write a first rsyncd.conf with 8 connections for a,c,o
+        self._write_rsyncd_conf(rsyncdpath, -1)
+
+        if sys.version_info[0] == 2:
+            retval = collections.namedtuple('statvfs_result',
+                                            'f_bsize f_bavail')
+            mock_freespace_func.return_value = retval(10 * GiB, 10 * GiB)
+        else:
+            retval = collections.namedtuple('usage', 'total used free')
+            # This says: 10 bytes remaining
+            mock_freespace_func.return_value = retval(10 * GiB,
+                                                      10 * GiB,
+                                                      10 * GiB)
+
+        drive_full_checker.configure_rsyncd_conf(10, 8, ' account_{} ',
+                                                 10, 8, ' container_{} ',
+                                                 10, 8, ' object_{} ',
+                                                 storagepath, rsyncdpath,
+                                                 self.logger)
+
+        should_be_rsyncdconf = tmpdirname + "/rsyncd_should_be.conf"
+        self._write_rsyncd_conf(should_be_rsyncdconf, 8)
+
+        # Assert that rsyncd.conf and rsyncd_should_be.conf are the same
+        self.assertTrue(filecmp.cmp(rsyncdpath, should_be_rsyncdconf))
+
+        shutil.rmtree(tmpdirname)