1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
package xopen
import (
"compress/gzip"
"io"
"os/exec"
)
// from: https://gist.github.com/rasky/d42a52c16683f1a2f4dccdef80e2712d
// fastGzReader is an API-compatible drop-in replacement
// for gzip.Reader, that achieves a higher decoding speed
// by spawning an external gzip instance and pipeing data
// through it.
// Go's native gzip implementation is about 2x slower at
// decompressing data compared to zlib (mostly due to Go compiler
// inefficiencies). So for tasks where the gzip decoding
// speed is important, this is a quick workaround that doesn't
// require cgo.
// gzip is part of the gzip package and comes preinstalled on
// most Linux distributions and on OSX.
type fastGzReader struct {
io.ReadCloser
}
func hasProg(prog ...string) bool {
var cmd *exec.Cmd
if len(prog) > 1 {
cmd = exec.Command(prog[0], prog[1:]...)
} else {
cmd = exec.Command(prog[0])
}
err := cmd.Start()
has := err == nil
cmd.Wait()
return has
}
var hasZlib = hasProg("gzip", "-d")
var hasPigz = hasProg("pigz", "-d")
func newFastGzReader(r io.Reader) (io.ReadCloser, error) {
if hasZlib || hasPigz {
var gz fastGzReader
if err := gz.Reset(r); err != nil {
return nil, err
}
return gz, nil
}
return gzip.NewReader(r)
}
func (gz *fastGzReader) Reset(r io.Reader) error {
if gz.ReadCloser != nil {
gz.Close()
}
var cmd *exec.Cmd
if hasPigz {
cmd = exec.Command("pigz", "-d")
} else {
cmd = exec.Command("gzip", "-d")
}
cmd.Stdin = r
rpipe, err := cmd.StdoutPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
rpipe.Close()
return err
}
gz.ReadCloser = rpipe
return nil
}
|
