1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
//+build ignore
// Copyright 2015, Klaus Post, see LICENSE for details.
//
// Simple stream encoder example
//
// The encoder encodes a single file into a number of shards
// To reverse the process see "stream-decoder.go"
//
// To build an executable use:
//
// go build stream-encoder.go
//
// Simple Encoder/Decoder Shortcomings:
// * If the file size of the input isn't dividable by the number of data shards
// the output will contain extra zeroes
//
// * If the shard numbers isn't the same for the decoder as in the
// encoder, invalid output will be generated.
//
// * If values have changed in a shard, it cannot be reconstructed.
//
// * If two shards have been swapped, reconstruction will always fail.
// You need to supply the shards in the same order as they were given to you.
//
// The solution for this is to save a metadata file containing:
//
// * File size.
// * The number of data/parity shards.
// * HASH of each shard.
// * Order of the shards.
//
// If you save these properties, you should abe able to detect file corruption
// in a shard and be able to reconstruct your data if you have the needed number of shards left.
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"io"
"github.com/klauspost/reedsolomon"
)
var dataShards = flag.Int("data", 4, "Number of shards to split the data into, must be below 257.")
var parShards = flag.Int("par", 2, "Number of parity shards")
var outDir = flag.String("out", "", "Alternative output directory")
func init() {
flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
fmt.Fprintf(os.Stderr, " %s [-flags] filename.ext\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Valid flags:\n")
flag.PrintDefaults()
}
}
func main() {
// Parse command line parameters.
flag.Parse()
args := flag.Args()
if len(args) != 1 {
fmt.Fprintf(os.Stderr, "Error: No input filename given\n")
flag.Usage()
os.Exit(1)
}
if *dataShards > 257 {
fmt.Fprintf(os.Stderr, "Error: Too many data shards\n")
os.Exit(1)
}
fname := args[0]
// Create encoding matrix.
enc, err := reedsolomon.NewStream(*dataShards, *parShards)
checkErr(err)
fmt.Println("Opening", fname)
f, err := os.Open(fname)
checkErr(err)
instat, err := f.Stat()
checkErr(err)
shards := *dataShards + *parShards
out := make([]*os.File, shards)
// Create the resulting files.
dir, file := filepath.Split(fname)
if *outDir != "" {
dir = *outDir
}
for i := range out {
outfn := fmt.Sprintf("%s.%d", file, i)
fmt.Println("Creating", outfn)
out[i], err = os.Create(filepath.Join(dir, outfn))
checkErr(err)
}
// Split into files.
data := make([]io.Writer, *dataShards)
for i := range data {
data[i] = out[i]
}
// Do the split
err = enc.Split(f, data, instat.Size())
checkErr(err)
// Close and re-open the files.
input := make([]io.Reader, *dataShards)
for i := range data {
out[i].Close()
f, err := os.Open(out[i].Name())
checkErr(err)
input[i] = f
defer f.Close()
}
// Create parity output writers
parity := make([]io.Writer, *parShards)
for i := range parity {
parity[i] = out[*dataShards+i]
defer out[*dataShards+i].Close()
}
// Encode parity
err = enc.Encode(input, parity)
checkErr(err)
fmt.Printf("File split into %d data + %d parity shards.\n", *dataShards, *parShards)
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
os.Exit(2)
}
}
|
