File: kopano-search-xapian-compact.py

package info (click to toggle)
kopanocore 8.7.0-3
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 15,400 kB
  • sloc: cpp: 175,422; python: 24,623; perl: 7,319; php: 6,056; sh: 2,172; makefile: 1,294; xml: 45; ansic: 1
file content (107 lines) | stat: -rw-r--r-- 3,550 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/usr/bin/python3
# SPDX-License-Identifier: AGPL-3.0-only
from __future__ import print_function
import fcntl
import glob
import grp
import os
import pwd
import shutil
import subprocess
import sys
import traceback

import kopano

"""
uses 'xapian-compact' to compact given database(s), specified using store GUID(s),
or all databases found in the 'index_path' directory specified in search.cfg.

"""

# TODO: purge removed users (stores)

def _default(name):
    if name == '':
        return 'root'
    elif name is None:
        return 'kopano'
    else:
        return name

def main():
    parser = kopano.parser() # select common cmd-line options
    options, args = parser.parse_args()

    config = kopano.Config(None, service='search', options=options)
    server = kopano.Server(options, config)

    # get index_path, run_as_user, run_as_group from  search.cfg
    search_config = server.config
    if not search_config:
        print('ERROR: search config is not available', file=sys.stderr)
        sys.exit(1)

    index_path = _default(search_config.get('index_path'))
    search_user = _default(search_config.get('run_as_user'))
    uid = pwd.getpwnam(search_user).pw_uid
    search_group = _default(search_config.get('run_as_group'))
    gid = grp.getgrnam(search_group).gr_gid

    if (uid, gid) != (os.getuid(), os.getgid()):
        print('ERROR: script should run under user/group as specified in search.cfg', file=sys.stderr)
        sys.exit(1)

    cleaned_args = [arg for arg in sys.argv[:1] if not arg.startswith('-')]
    # find database(s) corresponding to given store GUID(s)
    dbpaths = []
    if len(cleaned_args) > 1:
        for arg in cleaned_args[1:]:
            dbpath = os.path.join(index_path, server.guid+'-'+arg)
            dbpaths.append(dbpath)
    else:  # otherwise, select all databases for compaction
        dbpaths = []
        for path in glob.glob(os.path.join(index_path, server.guid+'-*')):
            if not path.endswith('.lock'):
                dbpaths.append(path)

    # loop and compact
    count = len(dbpaths)
    errors = 0
    for dbpath in dbpaths:
        try:
            if os.path.isdir(dbpath):
                if len(dbpath.split('-')) > 1:
                    store = dbpath.split('-')[1]
                    try:
                        server.store(store)
                    except kopano.NotFoundError as e:
                        print('deleting:', dbpath)
                        shutil.rmtree(dbpath)
                        os.remove('%s.lock' % dbpath)
                        continue

                print('compact:', dbpath)

                with open('%s.lock' % dbpath, 'w') as lockfile:  # do not index at the same time
                    fcntl.flock(lockfile.fileno(), fcntl.LOCK_EX)
                    shutil.rmtree(dbpath + '.compact', ignore_errors=True)
                    subprocess.call(['xapian-compact', dbpath, dbpath + '.compact'])
                    # quickly swap in compacted database
                    shutil.move(dbpath, dbpath + '.old')
                    shutil.move(dbpath + '.compact', dbpath)
                    shutil.rmtree(dbpath + '.old')
            else:
                print('ERROR: no such database:', dbpath)
                errors += 1
            print
        except Exception as e:
            print('ERROR', file=sys.stderr)
            traceback.print_exc(e)
            errors += 1

    # summarize
    print('done compacting (%d processed, %d errors)' % (count, errors))

if __name__ == '__main__':
    main()