1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
|
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package build
import (
"bytes"
"compress/gzip"
"crypto/sha1"
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"time"
"appengine"
"appengine/datastore"
)
const maxDatastoreStringLen = 500
// A Package describes a package that is listed on the dashboard.
type Package struct {
Kind string // "subrepo", "external", or empty for the main Go tree
Name string
Path string // (empty for the main Go tree)
NextNum int // Num of the next head Commit
}
func (p *Package) String() string {
return fmt.Sprintf("%s: %q", p.Path, p.Name)
}
func (p *Package) Key(c appengine.Context) *datastore.Key {
key := p.Path
if key == "" {
key = "go"
}
return datastore.NewKey(c, "Package", key, 0, nil)
}
// LastCommit returns the most recent Commit for this Package.
func (p *Package) LastCommit(c appengine.Context) (*Commit, error) {
var commits []*Commit
_, err := datastore.NewQuery("Commit").
Ancestor(p.Key(c)).
Order("-Time").
Limit(1).
GetAll(c, &commits)
if _, ok := err.(*datastore.ErrFieldMismatch); ok {
// Some fields have been removed, so it's okay to ignore this error.
err = nil
}
if err != nil {
return nil, err
}
if len(commits) != 1 {
return nil, datastore.ErrNoSuchEntity
}
return commits[0], nil
}
// GetPackage fetches a Package by path from the datastore.
func GetPackage(c appengine.Context, path string) (*Package, error) {
p := &Package{Path: path}
err := datastore.Get(c, p.Key(c), p)
if err == datastore.ErrNoSuchEntity {
return nil, fmt.Errorf("package %q not found", path)
}
if _, ok := err.(*datastore.ErrFieldMismatch); ok {
// Some fields have been removed, so it's okay to ignore this error.
err = nil
}
return p, err
}
// A Commit describes an individual commit in a package.
//
// Each Commit entity is a descendant of its associated Package entity.
// In other words, all Commits with the same PackagePath belong to the same
// datastore entity group.
type Commit struct {
PackagePath string // (empty for Go commits)
Hash string
ParentHash string
Num int // Internal monotonic counter unique to this package.
User string
Desc string `datastore:",noindex"`
Time time.Time
// ResultData is the Data string of each build Result for this Commit.
// For non-Go commits, only the Results for the current Go tip, weekly,
// and release Tags are stored here. This is purely de-normalized data.
// The complete data set is stored in Result entities.
ResultData []string `datastore:",noindex"`
FailNotificationSent bool
}
func (com *Commit) Key(c appengine.Context) *datastore.Key {
if com.Hash == "" {
panic("tried Key on Commit with empty Hash")
}
p := Package{Path: com.PackagePath}
key := com.PackagePath + "|" + com.Hash
return datastore.NewKey(c, "Commit", key, 0, p.Key(c))
}
func (c *Commit) Valid() error {
if !validHash(c.Hash) {
return errors.New("invalid Hash")
}
if c.ParentHash != "" && !validHash(c.ParentHash) { // empty is OK
return errors.New("invalid ParentHash")
}
return nil
}
// each result line is approx 105 bytes. This constant is a tradeoff between
// build history and the AppEngine datastore limit of 1mb.
const maxResults = 1000
// AddResult adds the denormalized Reuslt data to the Commit's Result field.
// It must be called from inside a datastore transaction.
func (com *Commit) AddResult(c appengine.Context, r *Result) error {
if err := datastore.Get(c, com.Key(c), com); err != nil {
return fmt.Errorf("getting Commit: %v", err)
}
com.ResultData = trim(append(com.ResultData, r.Data()), maxResults)
if _, err := datastore.Put(c, com.Key(c), com); err != nil {
return fmt.Errorf("putting Commit: %v", err)
}
return nil
}
func trim(s []string, n int) []string {
l := min(len(s), n)
return s[len(s)-l:]
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// Result returns the build Result for this Commit for the given builder/goHash.
func (c *Commit) Result(builder, goHash string) *Result {
for _, r := range c.ResultData {
p := strings.SplitN(r, "|", 4)
if len(p) != 4 || p[0] != builder || p[3] != goHash {
continue
}
return partsToHash(c, p)
}
return nil
}
// Results returns the build Results for this Commit for the given goHash.
func (c *Commit) Results(goHash string) (results []*Result) {
for _, r := range c.ResultData {
p := strings.SplitN(r, "|", 4)
if len(p) != 4 || p[3] != goHash {
continue
}
results = append(results, partsToHash(c, p))
}
return
}
// partsToHash converts a Commit and ResultData substrings to a Result.
func partsToHash(c *Commit, p []string) *Result {
return &Result{
Builder: p[0],
Hash: c.Hash,
PackagePath: c.PackagePath,
GoHash: p[3],
OK: p[1] == "true",
LogHash: p[2],
}
}
// A Result describes a build result for a Commit on an OS/architecture.
//
// Each Result entity is a descendant of its associated Commit entity.
type Result struct {
Builder string // "os-arch[-note]"
Hash string
PackagePath string // (empty for Go commits)
// The Go Commit this was built against (empty for Go commits).
GoHash string
OK bool
Log string `datastore:"-"` // for JSON unmarshaling only
LogHash string `datastore:",noindex"` // Key to the Log record.
RunTime int64 // time to build+test in nanoseconds
}
func (r *Result) Key(c appengine.Context) *datastore.Key {
p := Package{Path: r.PackagePath}
key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash
return datastore.NewKey(c, "Result", key, 0, p.Key(c))
}
func (r *Result) Valid() error {
if !validHash(r.Hash) {
return errors.New("invalid Hash")
}
if r.PackagePath != "" && !validHash(r.GoHash) {
return errors.New("invalid GoHash")
}
return nil
}
// Data returns the Result in string format
// to be stored in Commit's ResultData field.
func (r *Result) Data() string {
return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
}
// A Log is a gzip-compressed log file stored under the SHA1 hash of the
// uncompressed log text.
type Log struct {
CompressedLog []byte
}
func (l *Log) Text() ([]byte, error) {
d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog))
if err != nil {
return nil, fmt.Errorf("reading log data: %v", err)
}
b, err := ioutil.ReadAll(d)
if err != nil {
return nil, fmt.Errorf("reading log data: %v", err)
}
return b, nil
}
func PutLog(c appengine.Context, text string) (hash string, err error) {
h := sha1.New()
io.WriteString(h, text)
b := new(bytes.Buffer)
z, _ := gzip.NewWriterLevel(b, gzip.BestCompression)
io.WriteString(z, text)
z.Close()
hash = fmt.Sprintf("%x", h.Sum(nil))
key := datastore.NewKey(c, "Log", hash, 0, nil)
_, err = datastore.Put(c, key, &Log{b.Bytes()})
return
}
// A Tag is used to keep track of the most recent Go weekly and release tags.
// Typically there will be one Tag entity for each kind of hg tag.
type Tag struct {
Kind string // "weekly", "release", or "tip"
Name string // the tag itself (for example: "release.r60")
Hash string
}
func (t *Tag) Key(c appengine.Context) *datastore.Key {
p := &Package{}
return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c))
}
func (t *Tag) Valid() error {
if t.Kind != "weekly" && t.Kind != "release" && t.Kind != "tip" {
return errors.New("invalid Kind")
}
if !validHash(t.Hash) {
return errors.New("invalid Hash")
}
return nil
}
// Commit returns the Commit that corresponds with this Tag.
func (t *Tag) Commit(c appengine.Context) (*Commit, error) {
com := &Commit{Hash: t.Hash}
err := datastore.Get(c, com.Key(c), com)
return com, err
}
// GetTag fetches a Tag by name from the datastore.
func GetTag(c appengine.Context, tag string) (*Tag, error) {
t := &Tag{Kind: tag}
if err := datastore.Get(c, t.Key(c), t); err != nil {
if err == datastore.ErrNoSuchEntity {
return nil, errors.New("tag not found: " + tag)
}
return nil, err
}
if err := t.Valid(); err != nil {
return nil, err
}
return t, nil
}
// Packages returns packages of the specified kind.
// Kind must be one of "external" or "subrepo".
func Packages(c appengine.Context, kind string) ([]*Package, error) {
switch kind {
case "external", "subrepo":
default:
return nil, errors.New(`kind must be one of "external" or "subrepo"`)
}
var pkgs []*Package
q := datastore.NewQuery("Package").Filter("Kind=", kind)
for t := q.Run(c); ; {
pkg := new(Package)
_, err := t.Next(pkg)
if _, ok := err.(*datastore.ErrFieldMismatch); ok {
// Some fields have been removed, so it's okay to ignore this error.
err = nil
}
if err == datastore.Done {
break
} else if err != nil {
return nil, err
}
if pkg.Path != "" {
pkgs = append(pkgs, pkg)
}
}
return pkgs, nil
}
|