File: volumes.py

package info (click to toggle)
python-btrfs 11-2
  • links: PTS
  • area: main
  • in suites: buster
  • size: 724 kB
  • sloc: python: 4,445; makefile: 195
file content (272 lines) | stat: -rw-r--r-- 8,880 bytes parent folder | download | duplicates (2)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
# Copyright (C) 2018 Hans van Kranenburg <hans@knorrie.org>
#
# This file is part of the python-btrfs module.
#
# python-btrfs is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# python-btrfs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with python-btrfs.  If not, see <http://www.gnu.org/licenses/>.


from btrfs.ctree import (  # noqa
    BLOCK_GROUP_DATA, BLOCK_GROUP_SYSTEM, BLOCK_GROUP_METADATA,
    SPACE_INFO_GLOBAL_RSV, BLOCK_GROUP_TYPE_MASK,
    BLOCK_GROUP_RAID0, BLOCK_GROUP_RAID1, BLOCK_GROUP_RAID5,
    BLOCK_GROUP_RAID6, BLOCK_GROUP_DUP, BLOCK_GROUP_RAID10,
    BLOCK_GROUP_SINGLE,
    BLOCK_GROUP_PROFILE_MASK,
)
from btrfs.utils import SZ_1G
from collections import namedtuple

BTRFS_MAX_DATA_CHUNK_SIZE = 10 * SZ_1G

_RaidAttr = namedtuple('RaidAttr', [
    'sub_stripes', 'dev_stripes', 'devs_max', 'devs_min', 'tolerated_failures',
    'devs_increment', 'ncopies', 'nparity', 'raid_name', 'bg_flag',
])

RAID_RAID10 = 0
RAID_RAID1 = 1
RAID_DUP = 2
RAID_RAID0 = 3
RAID_SINGLE = 4
RAID_RAID5 = 5
RAID_RAID6 = 6


def _bg_flags_to_raid_index(flags):
    """Convert block group flags to an index to access _raid_array"""
    if flags & BLOCK_GROUP_RAID10:
        return RAID_RAID10
    if flags & BLOCK_GROUP_RAID1:
        return RAID_RAID1
    if flags & BLOCK_GROUP_DUP:
        return RAID_DUP
    if flags & BLOCK_GROUP_RAID0:
        return RAID_RAID0
    if flags & BLOCK_GROUP_RAID5:
        return RAID_RAID5
    if flags & BLOCK_GROUP_RAID6:
        return RAID_RAID6
    return RAID_SINGLE


_raid_array = [
    _RaidAttr(
        sub_stripes=2,
        dev_stripes=1,
        devs_max=0,
        devs_min=4,
        tolerated_failures=1,
        devs_increment=2,
        ncopies=2,
        nparity=0,
        raid_name='raid10',
        bg_flag=BLOCK_GROUP_RAID10,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=1,
        devs_max=2,
        devs_min=2,
        tolerated_failures=1,
        devs_increment=2,
        ncopies=2,
        nparity=0,
        raid_name='raid1',
        bg_flag=BLOCK_GROUP_RAID1,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=2,
        devs_max=1,
        devs_min=1,
        tolerated_failures=0,
        devs_increment=1,
        ncopies=2,
        nparity=0,
        raid_name='dup',
        bg_flag=BLOCK_GROUP_DUP,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=1,
        devs_max=0,
        devs_min=2,
        tolerated_failures=0,
        devs_increment=1,
        ncopies=1,
        nparity=0,
        raid_name='raid0',
        bg_flag=BLOCK_GROUP_RAID0,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=1,
        devs_max=1,
        devs_min=1,
        tolerated_failures=0,
        devs_increment=1,
        ncopies=1,
        nparity=0,
        raid_name='single',
        bg_flag=BLOCK_GROUP_SINGLE,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=1,
        devs_max=0,
        devs_min=2,
        tolerated_failures=1,
        devs_increment=1,
        ncopies=1,
        nparity=1,
        raid_name='raid5',
        bg_flag=BLOCK_GROUP_RAID5,
    ),
    _RaidAttr(
        sub_stripes=1,
        dev_stripes=1,
        devs_max=0,
        devs_min=3,
        tolerated_failures=2,
        devs_increment=1,
        ncopies=1,
        nparity=2,
        raid_name='raid6',
        bg_flag=BLOCK_GROUP_RAID6,
    ),
]


def _raid_attrs(flags):
    return _raid_array[_bg_flags_to_raid_index(flags)]


def chunk_length_to_dev_extent_length(flags, num_stripes, chunk_length):
    """Given information about a :class:`~btrfs.ctree.Chunk`, calculate the
    length of :class:`~btrfs.ctree.DevExtent` objects that store the Chunk
    data.

    For example, a Data, RAID6 block group of 8GiB, distributed over 6 devices,
    will occupy 2GiB * (4+2) = 12 GiB physical allocated bytes since it has
    4GiB of allocated bytes reserved for parity:

    Example::

        >>> btrfs.utils.pretty_size(
        ...     btrfs.volumes.chunk_length_to_dev_extent_length(
        ...         btrfs.ctree.BLOCK_GROUP_RAID6, 6, 8 * btrfs.utils.SZ_1G))
        '2.00GiB'
    """
    # So, we start with a chunk length, which is the amount of usable virtual
    # space.
    attrs = _raid_attrs(flags & BLOCK_GROUP_PROFILE_MASK)
    # The nparity attribute means that we have nparity * dev_extent_length of
    # raw space in total, dedicated for parity. These parity bytes can be
    # distributed over all of the num_stripes. If we subtract nparity from
    # num_stripes, we get the amount of dev_extent_lengths worth that contain
    # only data.
    #
    # For simplicity, let's assume all parity lives within dedicated device
    # extents. In reality it robins around, but that doesn't matter for our
    # calculations.
    #
    # RAID5: D | D | D | P
    # RAID6: D | D | D | P | P
    num_data_stripes = num_stripes - attrs.nparity
    # In case of profiles that duplicate data, we have to correct for that.
    # So, if we multiply the chunk_length (which is virtual space) by the
    # amount of copies of the data, we get the amount of raw bytes that we have
    # to fit in num_data_stripes amount of device extents.
    raw_data_bytes = chunk_length * attrs.ncopies
    dev_extent_length = raw_data_bytes // num_data_stripes
    return dev_extent_length


def chunk_to_dev_extent_length(chunk):
    """Given a :class:`~btrfs.ctree.Chunk` object, calculate the length of
    :class:`~btrfs.ctree.DevExtent` objects that store the chunk data.

    Example::

        >>> with btrfs.FileSystem('/') as fs:
        ...     chunk = list(fs.chunks(min_vaddr=3250585600, nr_items=1))[0]
        ...     print(chunk)
        ...     dev_extent_length = btrfs.volumes.chunk_to_dev_extent_length(chunk)
        ...     print("device extent length is {}".format(
        ...         btrfs.utils.pretty_size(dev_extent_length)))
        ...
        chunk vaddr 3250585600 type DATA|RAID5 length 2147483648 num_stripes 3
        device extent length is 1.00GiB
    """
    return chunk_length_to_dev_extent_length(chunk.type, chunk.num_stripes, chunk.length)


def dev_extent_length_to_chunk_length(flags, num_stripes, stripe_size):
    """This function simply reverses the calculation of
    :class:`~btrfs.ctree.Chunk` length to :class:`~btrfs.ctree.DevExtent`
    length.

    Example::

        >>> btrfs.utils.pretty_size(
        ...     btrfs.volumes.dev_extent_length_to_chunk_length(
        ...         btrfs.ctree.BLOCK_GROUP_RAID6, 6, 2 * btrfs.utils.SZ_1G))
        '8.00GiB'
    """
    attrs = _raid_attrs(flags & BLOCK_GROUP_PROFILE_MASK)
    num_data_stripes = num_stripes - attrs.nparity
    # stripe_size is a synonym for device extent length.
    raw_data_bytes = stripe_size * num_data_stripes
    chunk_length = raw_data_bytes // attrs.ncopies
    return chunk_length


def chunk_to_raw_parity_bytes(chunk):
    """Given a :class:`~btrfs.ctree.Chunk` object, calculate how many bytes are
    reserved for storing parity data for RAID56 profiles.

    This number is relevant to understand how much raw disk space that is
    allocated but not used is actually not usable for data, because it's
    reserved for parity.

    Example::

        >>> with btrfs.FileSystem('/mnt/tutorial') as fs:
        ...  chunk = list(fs.chunks(min_vaddr=3250585600, nr_items=1))[0]
        ...  print(chunk)
        ...  parity_bytes = btrfs.volumes.chunk_to_raw_parity_bytes(chunk)
        ...  print("amount of raw parity bytes: {}".format(
        ...      btrfs.utils.pretty_size(parity_bytes)))
        ...
        chunk vaddr 3250585600 type DATA|RAID5 length 2147483648 num_stripes 3
        amount of raw parity bytes: 1.00GiB
    """
    dev_extent_length = chunk_to_dev_extent_length(chunk)
    attrs = _raid_attrs(chunk.type & BLOCK_GROUP_PROFILE_MASK)
    return dev_extent_length * attrs.nparity


def block_group_profile_ncopies(flags):
    """This function returns how many times the actual data is replicated on
    disk given block group flags as input. E.g. for RAID1 this is 2, but for
    RAID5, this is just 1. The actual data is stored once in case of RAID5, and
    the redundancy is done using parity blocks, not data itself.

    Example::

        >>> btrfs.volumes.block_group_profile_ncopies(btrfs.ctree.BLOCK_GROUP_RAID10)
        2
    """
    attrs = _raid_attrs(flags & BLOCK_GROUP_PROFILE_MASK)
    return attrs.ncopies