1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
|
#!/bin/bash
#Test the split-brain resolution CLI commands.
. $(dirname $0)/../../include.rc
. $(dirname $0)/../../volume.rc
function get_replicate_subvol_number {
local filename=$1
#get_backend_paths
if [ -f $B0/${V0}1/$filename ]
then
echo 0
elif [ -f $B0/${V0}3/$filename ]
then echo 1
else
echo -1
fi
}
cleanup;
AREQUAL_PATH=$(dirname $0)/../../utils
GET_MDATA_PATH=$(dirname $0)/../../utils
CFLAGS=""
test "`uname -s`" != "Linux" && {
CFLAGS="$CFLAGS -lintl";
}
build_tester $AREQUAL_PATH/arequal-checksum.c $CFLAGS
build_tester $GET_MDATA_PATH/get-mdata-xattr.c
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
TEST $CLI volume set $V0 cluster.self-heal-daemon off
TEST $CLI volume set $V0 cluster.data-self-heal off
TEST $CLI volume set $V0 cluster.metadata-self-heal off
TEST $CLI volume set $V0 cluster.entry-self-heal off
TEST $CLI volume set $V0 ctime off
TEST $CLI volume start $V0
TEST $GFS --volfile-id=/$V0 --volfile-server=$H0 $M0
cd $M0
for i in {1..10}
do
echo "Initial content">>file$i
done
replica_0_files_list=(`ls $B0/${V0}1|grep -v '^\.'`)
replica_1_files_list=(`ls $B0/${V0}3|grep -v '^\.'`)
############ Create data split-brain in the files. ###########################
TEST kill_brick $V0 $H0 $B0/${V0}1
for file in ${!replica_0_files_list[*]}
do
echo "B1 is down">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}3
for file in ${!replica_1_files_list[*]}
do
echo "B3 is down">>${replica_1_files_list[$file]}
done
SMALLER_FILE_SIZE=$(stat -c %s file1)
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 2
TEST kill_brick $V0 $H0 $B0/${V0}2
for file in ${!replica_0_files_list[*]}
do
echo "B2 is down">>${replica_0_files_list[$file]}
echo "appending more content to make it the bigger file">>${replica_0_files_list[$file]}
done
TEST kill_brick $V0 $H0 $B0/${V0}4
for file in ${!replica_1_files_list[*]}
do
echo "B4 is down">>${replica_1_files_list[$file]}
echo "appending more content to make it the bigger file">>${replica_1_files_list[$file]}
done
BIGGER_FILE_SIZE=$(stat -c %s file1)
TEST $CLI volume start $V0 force
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 1
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 3
############### Acessing the files should now give EIO. ###############################
TEST ! cat file1
TEST ! cat file2
TEST ! cat file3
TEST ! cat file4
TEST ! cat file5
TEST ! cat file6
TEST ! cat file7
TEST ! cat file8
TEST ! cat file9
TEST ! cat file10
###################
TEST $CLI volume set $V0 cluster.self-heal-daemon on
EXPECT_WITHIN $PROCESS_UP_TIMEOUT "Y" glustershd_up_status
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 0
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 1
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 2
EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status_in_shd $V0 3
################ Heal file1 using the bigger-file option ##############
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "0" echo $?
EXPECT $BIGGER_FILE_SIZE stat -c %s file1
################ Heal file2 using the bigger-file option and its gfid ##############
subvolume=$(get_replicate_subvol_number file2)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file2)
elif [ $subvolume == 1 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file2)
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain bigger-file $GFIDSTR
EXPECT "0" echo $?
################ Heal file3 using the source-brick option ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file3)
if [ $subvolume == 0 ]
then
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 /file3
elif [ $subvolume == 1 ]
then
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file3
################ Heal file4 using the source-brick option and it's gfid ##############
################ Use the brick having smaller file size as source #######
subvolume=$(get_replicate_subvol_number file4)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file4)
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}2 $GFIDSTR
elif [ $subvolume == 1 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file4)
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 $GFIDSTR
fi
EXPECT "0" echo $?
EXPECT $SMALLER_FILE_SIZE stat -c %s file4
################ Heal file5 using the latest-mtime option ##############
subvolume=$(get_replicate_subvol_number file5)
if [ $subvolume == 0 ]
then
mtime1=$(get_mtime $B0/${V0}1/file5)
mtime2=$(get_mtime $B0/${V0}2/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
mtime1=$(get_mtime $B0/${V0}3/file5)
mtime2=$(get_mtime $B0/${V0}4/file5)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
$CLI volume heal $V0 split-brain latest-mtime /file5
EXPECT "0" echo $?
if [ $subvolume == 0 ]
then
mtime1_after_heal=$(get_mtime $B0/${V0}1/file5)
mtime2_after_heal=$(get_mtime $B0/${V0}2/file5)
elif [ $subvolume == 1 ]
then
mtime1_after_heal=$(get_mtime $B0/${V0}3/file5)
mtime2_after_heal=$(get_mtime $B0/${V0}4/file5)
fi
#TODO: To below comparisons on full sub-second resolution
TEST [ $LATEST_MTIME -eq $mtime1_after_heal ]
TEST [ $LATEST_MTIME -eq $mtime2_after_heal ]
mtime_mount_after_heal=$(stat -c %Y file5)
TEST [ $LATEST_MTIME -eq $mtime_mount_after_heal ]
################ Heal file6 using the latest-mtime option and its gfid ##############
subvolume=$(get_replicate_subvol_number file6)
if [ $subvolume == 0 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}1/file6)
mtime1=$(get_mtime $B0/${V0}1/file6)
mtime2=$(get_mtime $B0/${V0}2/file6)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
elif [ $subvolume == 1 ]
then
GFID=$(gf_get_gfid_xattr $B0/${V0}3/file6)
mtime1=$(get_mtime $B0/${V0}3/file6)
mtime2=$(get_mtime $B0/${V0}4/file6)
LATEST_MTIME=$(($mtime1 > $mtime2 ? $mtime1:$mtime2))
fi
GFIDSTR="gfid:$(gf_gfid_xattr_to_str $GFID)"
$CLI volume heal $V0 split-brain latest-mtime $GFIDSTR
EXPECT "0" echo $?
if [ $subvolume == 0 ]
then
mtime1_after_heal=$(get_mtime $B0/${V0}1/file6)
mtime2_after_heal=$(get_mtime $B0/${V0}2/file6)
elif [ $subvolume == 1 ]
then
mtime1_after_heal=$(get_mtime $B0/${V0}3/file6)
mtime2_after_heal=$(get_mtime $B0/${V0}4/file6)
fi
#TODO: To below comparisons on full sub-second resolution
TEST [ $LATEST_MTIME -eq $mtime1_after_heal ]
TEST [ $LATEST_MTIME -eq $mtime2_after_heal ]
mtime_mount_after_heal=$(stat -c %Y file6)
TEST [ $LATEST_MTIME -eq $mtime_mount_after_heal ]
################ Heal remaining SB'ed files of replica_0 using B1 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}1
EXPECT "0" echo $?
################ Heal remaining SB'ed files of replica_1 using B3 as source ##############
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}3
EXPECT "0" echo $?
############### Reading the files should now succeed. ###############################
TEST cat file1
TEST cat file2
TEST cat file3
TEST cat file4
TEST cat file5
TEST cat file6
TEST cat file7
TEST cat file8
TEST cat file9
TEST cat file10
################ File contents on the bricks must be same. ################################
TEST diff <(arequal-checksum -p $B0/$V01 -i .glusterfs) <(arequal-checksum -p $B0/$V02 -i .glusterfs)
TEST diff <(arequal-checksum -p $B0/$V03 -i .glusterfs) <(arequal-checksum -p $B0/$V04 -i .glusterfs)
############### Trying to heal files not in SB should fail. ###############################
$CLI volume heal $V0 split-brain bigger-file /file1
EXPECT "1" echo $?
$CLI volume heal $V0 split-brain source-brick $H0:$B0/${V0}4 /file3
EXPECT "1" echo $?
cd -
TEST rm $AREQUAL_PATH/arequal-checksum
TEST rm $GET_MDATA_PATH/get-mdata-xattr
cleanup
|
