1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
|
// Package thirdparty executes integration tests based on third-party projects that use avo.
package thirdparty
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
)
// GithubRepository specifies a repository on github.
type GithubRepository struct {
Owner string `json:"owner"`
Name string `json:"name"`
}
func (r GithubRepository) String() string {
return path.Join(r.Owner, r.Name)
}
// URL returns the Github repository URL.
func (r GithubRepository) URL() string {
return fmt.Sprintf("https://github.com/%s", r)
}
// CloneURL returns the git clone URL.
func (r GithubRepository) CloneURL() string {
return fmt.Sprintf("https://github.com/%s.git", r)
}
// Metadata about the repository.
type Metadata struct {
// Repository description.
Description string `json:"description,omitempty"`
// Homepage URL. Not the same as the Github page.
Homepage string `json:"homepage,omitempty"`
// Number of Github stars.
Stars int `json:"stars,omitempty"`
}
// Project defines an integration test based on a third-party project using avo.
type Project struct {
// Repository for the project. At the moment, all projects are available on
// github.
Repository GithubRepository `json:"repository"`
// Repository metadata.
Metadata Metadata `json:"metadata"`
// Default git branch. This is used when testing against the latest version.
DefaultBranch string `json:"default_branch,omitempty"`
// Version as a git sha, tag or branch.
Version string `json:"version"`
// If the project test has a known problem, record it by setting this to a
// non-zero avo issue number. If set, the project will be skipped in
// testing.
KnownIssue int `json:"known_issue,omitempty"`
// Packages within the project to test.
Packages []*Package `json:"packages"`
}
func (p *Project) defaults(set bool) {
for _, pkg := range p.Packages {
pkg.defaults(set)
}
}
// Validate project definition.
func (p *Project) Validate() error {
if p.DefaultBranch == "" {
return errors.New("missing default branch")
}
if p.Version == "" {
return errors.New("missing version")
}
if len(p.Packages) == 0 {
return errors.New("missing packages")
}
for _, pkg := range p.Packages {
if err := pkg.Validate(); err != nil {
return fmt.Errorf("package %s: %w", pkg.Name(), err)
}
}
return nil
}
// ID returns an identifier for the project.
func (p *Project) ID() string {
return strings.ReplaceAll(p.Repository.String(), "/", "-")
}
// Skip reports whether the project test should be skipped. If skipped, a known
// issue will be set.
func (p *Project) Skip() bool {
return p.KnownIssue != 0
}
// Reason returns the reason why the test is skipped.
func (p *Project) Reason() string {
return fmt.Sprintf("https://github.com/mmcloughlin/avo/issues/%d", p.KnownIssue)
}
// Step represents a set of commands to run as part of the testing plan for a
// third-party package.
type Step struct {
Name string `json:"name,omitempty"`
WorkingDirectory string `json:"dir,omitempty"`
Commands []string `json:"commands"`
}
// Validate step parameters.
func (s *Step) Validate() error {
if s.Name == "" {
return errors.New("missing name")
}
if len(s.Commands) == 0 {
return errors.New("missing commands")
}
return nil
}
// Package defines an integration test for a package within a project.
type Package struct {
// Sub-package within the project under test. All file path references will
// be relative to this directory. If empty the root of the repository is
// used.
SubPackage string `json:"pkg,omitempty"`
// Path to the module file for the avo generator package. This is necessary
// so the integration test can insert replace directives to point at the avo
// version under test.
Module string `json:"module"`
// Setup steps. These run prior to the insertion of avo replace directives,
// therefore should be used if it's necessary to initialize new go modules
// within the repository.
Setup []*Step `json:"setup,omitempty"`
// Steps to run the avo code generator.
Generate []*Step `json:"generate"`
// Test steps. If empty, defaults to "go test ./...".
Test []*Step `json:"test,omitempty"`
}
// defaults sets or removes default field values.
func (p *Package) defaults(set bool) {
for _, stage := range []struct {
Steps []*Step
DefaultName string
}{
{p.Setup, "Setup"},
{p.Generate, "Generate"},
{p.Test, "Test"},
} {
if len(stage.Steps) == 1 {
stage.Steps[0].Name = applydefault(set, stage.Steps[0].Name, stage.DefaultName)
}
}
}
func applydefault(set bool, s, def string) string {
switch {
case set && s == "":
return def
case !set && s == def:
return ""
default:
return s
}
}
// Validate package definition.
func (p *Package) Validate() error {
if p.Module == "" {
return errors.New("missing module")
}
if len(p.Generate) == 0 {
return errors.New("no generate commands")
}
stages := map[string][]*Step{
"setup": p.Setup,
"generate": p.Generate,
"test": p.Test,
}
for name, steps := range stages {
for _, s := range steps {
if err := s.Validate(); err != nil {
return fmt.Errorf("%s step: %w", name, err)
}
}
}
return nil
}
// Name of the package.
func (p *Package) Name() string {
if p.IsRoot() {
return "root"
}
return p.SubPackage
}
// IsRoot reports whether the package is the root of the containing project.
func (p *Package) IsRoot() bool {
return p.SubPackage == ""
}
// Context specifies execution environment parameters for a third-party test.
type Context struct {
// Path to the avo version under test.
AvoDirectory string
// Path to the checked out third-party repository.
RepositoryDirectory string
}
// Steps generates the list of steps required to execute the integration test
// for this package. Context specifies execution environment parameters.
func (p *Package) Steps(c *Context) []*Step {
var steps []*Step
// Optional setup.
steps = append(steps, p.Setup...)
// Replace avo dependency.
const invalid = "v0.0.0-00010101000000-000000000000"
moddir := filepath.Dir(p.Module)
modfile := filepath.Base(p.Module)
steps = append(steps, &Step{
Name: "Avo Module Replacement",
WorkingDirectory: moddir,
Commands: []string{
"go mod edit -modfile=" + modfile + " -require=github.com/mmcloughlin/avo@" + invalid,
"go mod edit -modfile=" + modfile + " -replace=github.com/mmcloughlin/avo=" + c.AvoDirectory,
"go mod tidy -modfile=" + modfile,
},
})
// Run generation.
steps = append(steps, p.Generate...)
// Display changes.
steps = append(steps, &Step{
Name: "Diff",
Commands: []string{"git diff"},
})
// Tests.
if len(p.Test) > 0 {
steps = append(steps, p.Test...)
} else {
steps = append(steps, &Step{
Name: "Test",
Commands: []string{
"go test ./...",
},
})
}
// Prepend sub-directory to every step.
if p.SubPackage != "" {
for _, s := range steps {
s.WorkingDirectory = filepath.Join(p.SubPackage, s.WorkingDirectory)
}
}
return steps
}
// Test case for a given package within a project.
type Test struct {
Project *Project
Package *Package
}
// ID returns an identifier for the test case.
func (t *Test) ID() string {
pkgpath := path.Join(t.Project.Repository.String(), t.Package.SubPackage)
return strings.ReplaceAll(pkgpath, "/", "-")
}
// Projects is a collection of third-party integration tests.
type Projects []*Project
func (p Projects) defaults(set bool) {
for _, prj := range p {
prj.defaults(set)
}
}
// Validate the project collection.
func (p Projects) Validate() error {
seen := map[string]bool{}
for _, prj := range p {
// Project is valid.
if err := prj.Validate(); err != nil {
return fmt.Errorf("project %s: %w", prj.ID(), err)
}
// No duplicate entries.
id := prj.ID()
if seen[id] {
return fmt.Errorf("duplicate project %q", id)
}
seen[id] = true
}
return nil
}
// Tests returns all test cases for the projects collection.
func (p Projects) Tests() []*Test {
var ts []*Test
for _, prj := range p {
for _, pkg := range prj.Packages {
ts = append(ts, &Test{
Project: prj,
Package: pkg,
})
}
}
return ts
}
// Ranked returns a copy of the projects list ranked in desending order of
// popularity.
func (p Projects) Ranked() Projects {
ranked := append(Projects(nil), p...)
sort.SliceStable(ranked, func(i, j int) bool {
return ranked[i].Metadata.Stars > ranked[j].Metadata.Stars
})
return ranked
}
// Top returns the top n most popular projects.
func (p Projects) Top(n int) Projects {
top := p.Ranked()
if len(top) > n {
top = top[:n]
}
return top
}
// Suite defines a third-party test suite.
type Suite struct {
// Projects to test.
Projects Projects `json:"projects"`
// Time of the last update to project metadata.
MetadataLastUpdate time.Time `json:"metadata_last_update"`
}
func (s *Suite) defaults(set bool) {
s.Projects.defaults(set)
}
// Validate the test suite.
func (s *Suite) Validate() error {
if s.MetadataLastUpdate.IsZero() {
return errors.New("empty metadata update time")
}
if s.MetadataLastUpdate.Location() != time.UTC {
return errors.New("metadata update time not in UTC")
}
if err := s.Projects.Validate(); err != nil {
return err
}
return nil
}
// LoadSuite loads a test suite from JSON format.
func LoadSuite(r io.Reader) (*Suite, error) {
var s *Suite
d := json.NewDecoder(r)
d.DisallowUnknownFields()
if err := d.Decode(&s); err != nil {
return nil, err
}
s.defaults(true)
return s, nil
}
// LoadSuiteFile loads a test suite from a JSON file.
func LoadSuiteFile(filename string) (*Suite, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
defer f.Close()
return LoadSuite(f)
}
// StoreSuite writes a test suite in JSON format.
func StoreSuite(w io.Writer, s *Suite) error {
e := json.NewEncoder(w)
e.SetIndent("", " ")
s.defaults(false)
err := e.Encode(s)
s.defaults(true)
return err
}
// StoreSuiteFile writes a test suite to a JSON file.
func StoreSuiteFile(filename string, s *Suite) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
return StoreSuite(f, s)
}
|