1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
package sortedset
import (
"bytes"
"github.com/segmentio/asm/internal"
)
// Dedupe writes to dst the deduplicated sequence of items of the given size
// read from src, returning the byte slice containing the result.
//
// If dst is too small, a new slice is allocated and returned instead.
//
// The source and destination slices may be the same to perform in-place
// deduplication of the elements. The behavior is undefined for any other
// conditions where the source and destination slices overlap.
//
// The function panics if len(src) is not a multiple of the element size.
func Dedupe(dst, src []byte, size int) []byte {
if !internal.MultipleOf(size, len(src)) {
panic("input length is not a multiple of the item size")
}
if len(dst) < len(src) {
dst = make([]byte, len(src))
}
var n int
switch size {
case 1:
n = dedupe1(dst, src)
case 2:
n = dedupe2(dst, src)
case 4:
n = dedupe4(dst, src)
case 8:
n = dedupe8(dst, src)
case 16:
n = dedupe16(dst, src)
case 32:
n = dedupe32(dst, src)
default:
n = dedupeGeneric(dst, src, size)
}
return dst[:n]
}
func dedupeGeneric(dst, src []byte, size int) int {
if len(src) == 0 {
return 0
}
i := size
j := size
copy(dst, src[:size])
for i < len(src) {
if !bytes.Equal(src[i-size:i], src[i:i+size]) {
copy(dst[j:], src[i:i+size])
j += size
}
i += size
}
return j
}
|
