File: memcached-automove

package info (click to toggle)
memcached 1.5.6-1.1
  • links: PTS, VCS
  • area: main
  • in suites: buster
  • size: 2,696 kB
  • sloc: ansic: 16,931; perl: 5,523; sh: 4,302; xml: 1,638; python: 393; makefile: 170
file content (229 lines) | stat: -rwxr-xr-x 8,141 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
#!/usr/bin/python3
# Copyright 2017 Facebook.
# Licensed under the same terms as memcached itself.

import argparse
import socket
import sys
import re
import traceback
from time import sleep

parser = argparse.ArgumentParser(description="daemon for rebalancing slabs")
parser.add_argument("--host", help="host to connect to",
        default="localhost:11211", metavar="HOST:PORT")
parser.add_argument("-s", "--sleep", help="seconds between runs",
                    type=int, default="1")
parser.add_argument("-v", "--verbose", action="store_true")
parser.add_argument("-a", "--automove", action="store_true", default=False,
                    help="enable automatic page rebalancing")
parser.add_argument("-w", "--window", type=int, default="30",
                    help="rolling window size for decision history")
parser.add_argument("-r", "--ratio", type=float, default=0.8,
                    help="ratio limiting distance between low/high class ages")

# TODO:
# - age adjuster function
#   - by ratio of get_hits
#   - by ratio of chunk size

args = parser.parse_args()

host, port = args.host.split(':')

def window_check(history, sid, key):
    total = 0
    for window in history['w']:
        s = window.get(sid)
        if s and s.get(key):
            total += s.get(key)
    return total

def determine_move(history, diffs, totals):
    """ Figure out of a page move is in order.

    - if > 2.5 pages of free space without free chunks reducing for N trials,
      and no evictions for N trials, free to global.
    - use ratio of how far apart age can be between slab classes
    - TODO: if get_hits exists, allowable distance in age from the *youngest* slab
      class is based on the percentage of get_hits the class gets, against the
      factored max distance, ie:
      1% max delta. youngest is 900, oldest allowable is 900+90
      if class has 30% of get_hits, it can be 930
    - youngest evicting slab class gets page moved to it, if outside ratio max
    - use age as average over window. smooths over items falling out of WARM.
      also gives a little weight: if still evicting, a few more pages than
      necessary may be moved, pulling the classes closer together. Hopefully
      avoidnig ping-ponging.
    """
    # rotate windows
    history['w'].append({})
    if (len(history['w']) > args.window):
        history['w'].pop(0)
    w = history['w'][-1]
    oldest = (-1, 0)
    youngest = (-1, sys.maxsize)
    decision = (-1, -1)
    for sid, slab in diffs.items():

        w[sid] = {}
        if 'evicted_d' not in slab or 'total_pages_d' not in slab:
            continue
        # mark this window as dirty if total pages increases or evictions
        # happened
        if slab['total_pages_d'] > 0:
            w[sid]['dirty'] = 1
        if slab['evicted_d'] > 0:
            w[sid]['dirty'] = 1
            w[sid]['ev'] = 1
        w[sid]['age'] = slab['age']
        age = window_check(history, sid, 'age') / len(history['w'])

        # if > 2.5 pages free, and not dirty, reassign to global page pool and
        # break.
        if slab['free_chunks'] > slab['chunks_per_page'] * 2.5:
            if window_check(history, sid, 'dirty') == 0:
                decision = (sid, 0)
                break

        # are we the oldest slab class? (and a valid target)
        if age > oldest[1] and slab['total_pages'] > 2:
            oldest = (sid, age)

        # are we the youngest evicting slab class?
        ev_total = window_check(history, sid, 'ev')
        window_min = args.window / 2
        if age < youngest[1] and ev_total > window_min:
            youngest = (sid, age)
            #if args.verbose:
            #    print("window: {} range: {}".format(ev_total, window_min))

    # is the youngest slab class too young?
    if youngest[0] != -1 and oldest[0] != -1:
        if args.verbose:
            print("old:   [class: {}] [age: {:.2f}]\nyoung: [class: {}] [age: {:.2f}]".format(
                int(oldest[0]), oldest[1], int(youngest[0]), youngest[1]))
        if youngest[1] < oldest[1] * args.ratio and w[youngest[0]].get('ev'):
            decision = (oldest[0], youngest[0])

    if (len(history['w']) >= args.window):
        return decision
    return (-1, -1)


def run_move(s, decision):
    s.write("slabs reassign " + str(decision[0]) + " " + str(decision[1]) + "\r\n")
    line = s.readline().rstrip()
    if args.verbose:
        print("move result:", line)


def diff_stats(before, after):
    """ fills out "diffs" as deltas between before/after,
    and "totals" as the sum of all slab classes.
    "_d" postfix to keys means the delta between before/after.
    non-postfix keys are total as of 'after's reading.
    """
    diffs = {}
    totals = {}
    for slabid in after.keys():
        sb = before.get(slabid)
        sa = after.get(slabid)
        if not (sb and sa):
            continue
        slab = sa.copy()
        for k in sa.keys():
            if k not in sb:
                continue
            if k not in totals:
                totals[k] = 0
                totals[k + '_d'] = 0
            if k + '_d' not in slab:
                slab[k + '_d'] = 0
            if re.search(r"^\d+$", sa[k]):
                totals[k] += int(sa[k])
                slab[k] = int(sa[k])
                slab[k + '_d'] = int(sa[k]) - int(sb[k])
                totals[k + '_d'] += int(sa[k]) - int(sb[k])
        slab['slab'] = slabid
        diffs[slabid] = slab
    return (diffs, totals)


def read_stats(s):
    slabs = {}
    for statcmd in ['items', 'slabs']:
        #print("stat cmd: " + statcmd)
        # FIXME: Formatting
        s.write("stats " + statcmd + "\r\n")
        while True:
            line = s.readline().rstrip()
            if line.startswith("END"):
                break

            m = re.match(r"^STAT (?:items:)?(\d+):(\S+) (\S+)", line)
            if m:
                (slab, var, val) = m.groups()
                if slab not in slabs:
                    slabs[slab] = {}
                slabs[slab][var] = val
            #print("line: " + line)
    return slabs


def pct(num, divisor):
    if not divisor:
        return 0
    return (num / divisor)


def show_detail(diffs, totals):
    """ just a pretty printer for some extra data """
    print("\n  {:2s}: {:8s} (pct  ) {:10s} (pct    ) {:6s} (pct)   {:6s}".format('sb',
                'evicted', 'items', 'pages', 'age'))

    for sid, slab in diffs.items():
        if 'evicted_d' not in slab:
            continue
        print("  {:2d}: {:8d} ({:.2f}%) {:10d} ({:.4f}%) {:6d} ({:.2f}%) {:6d}".format(
              int(sid), slab['evicted_d'], pct(slab['evicted_d'], totals['evicted_d']),
              slab['number'], pct(slab['number'], totals['number']),
              slab['total_pages'], pct(slab['total_pages'],
              totals['total_pages']),
              slab['age']))


stats_pre = {}
history = { 'w': [{}] }
while True:
    try:
        with socket.create_connection((host, port), 5) as c:
            s = c.makefile(mode="rw", buffering=1)
            s.write("slabs automove 0\r\n")
            print(s.readline().rstrip())
            while True:
                stats_post = read_stats(s)
                (diffs, totals) = diff_stats(stats_pre, stats_post)
                if args.verbose:
                    show_detail(diffs, totals)
                decision = determine_move(history, diffs, totals)
                if decision[0] != -1 and decision[1] != -1:
                    print("moving page from, to:", decision)
                    if args.automove:
                        run_move(s, decision)

                # Minimize sleeping if we just moved a page to global pool.
                # Improves responsiveness during flushes/quick changes.
                if decision[1] == 0:
                    sleep(0.05)
                else:
                    sleep(args.sleep)
                stats_pre = stats_post
    except:
        err = sys.exc_info()
        print("disconnected:", err[0], err[1])
        traceback.print_exc()
        stats_pre = {}
        history = { 'w': [{}] }
        sleep(args.sleep)