1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#############################################################################
## The top section of this file is identical in the 3 cloudbuild.*yaml files.
## Make sure any edits you make here are copied over to the other files too
## if appropriate.
##
## TODO(al): consider if it's possible to merge these 3 files and control via
## substitutions.
#############################################################################
timeout: 1200s
options:
machineType: N1_HIGHCPU_32
volumes:
- name: go-modules
path: /go
env:
- GOPROXY=https://proxy.golang.org
- PROJECT_ROOT=github.com/google/certificate-transparency-go
- GOPATH=/go
substitutions:
_CLUSTER_NAME: trillian-opensource-ci
_MASTER_ZONE: us-central1-a
# Cloud Build logs sent to GCS bucket
logsBucket: 'gs://trillian-cloudbuild-logs'
steps:
# First build a "ct_testbase" docker image which contains most of the tools we need for the later steps:
- name: 'gcr.io/cloud-builders/docker'
entrypoint: 'bash'
args: ['-c', 'docker pull gcr.io/$PROJECT_ID/ct_testbase:latest || exit 0']
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'-t', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
'--cache-from', 'gcr.io/$PROJECT_ID/ct_testbase:latest',
'-f', './integration/Dockerfile',
'.'
]
# prepare spins up an ephemeral trillian instance for testing use.
- name: gcr.io/$PROJECT_ID/ct_testbase
entrypoint: 'bash'
id: 'prepare'
args:
- '-exc'
- |
# Use latest versions of Trillian docker images built by the Trillian CI cloudbuilders.
docker pull gcr.io/$PROJECT_ID/log_server:latest
docker tag gcr.io/$PROJECT_ID/log_server:latest deployment_trillian-log-server
docker pull gcr.io/$PROJECT_ID/log_signer:latest
docker tag gcr.io/$PROJECT_ID/log_signer:latest deployment_trillian-log-signer
# Bring up an ephemeral trillian instance using the docker-compose config in the Trillian repo:
export TRILLIAN_LOCATION="$$(go list -f '{{.Dir}}' github.com/google/trillian)"
# We need to fix up Trillian's docker-compose to connect to the CloudBuild network to that tests can use it:
echo -e "networks:\n default:\n external:\n name: cloudbuild" >> $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml
docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml pull mysql trillian-log-server trillian-log-signer
docker-compose -f $${TRILLIAN_LOCATION}/examples/deployment/docker-compose.yml up -d mysql trillian-log-server trillian-log-signer
# Install proto related bits and block on Trillian being ready
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'ci-ready'
entrypoint: 'bash'
args:
- '-ec'
- |
go install \
github.com/golang/protobuf/proto \
github.com/golang/protobuf/protoc-gen-go \
github.com/golang/mock/mockgen \
go.etcd.io/etcd/v3 go.etcd.io/etcd/etcdctl/v3 \
github.com/fullstorydev/grpcurl/cmd/grpcurl
# Generate all protoc and mockgen files
go generate -run="protoc" ./...
go generate -run="mockgen" ./...
# Cache all the modules we'll need too
go mod download
go test ./...
# Wait for trillian logserver to be up
until nc -z deployment_trillian-log-server_1 8090; do echo .; sleep 5; done
waitFor: ['prepare']
# Run the presubmit tests
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'default_test'
env:
- 'GOFLAGS='
- 'PRESUBMIT_OPTS=--no-linters --no-generate'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'race_detection'
env:
- 'GOFLAGS=-race'
- 'PRESUBMIT_OPTS=--no-linters'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'etcd_with_coverage'
env:
- 'GOFLAGS='
- 'PRESUBMIT_OPTS=--no-linters --coverage'
- 'WITH_ETCD=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'etcd_with_race'
env:
- 'GOFLAGS=-race'
- 'PRESUBMIT_OPTS=--no-linters'
- 'WITH_ETCD=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'with_pkcs11_and_race'
env:
- 'GOFLAGS=-race --tags=pkcs11'
- 'PRESUBMIT_OPTS=--no-linters'
- 'WITH_PKCS11=true'
- 'TRILLIAN_LOG_SERVERS=deployment_trillian-log-server_1:8090'
- 'TRILLIAN_LOG_SERVER_1=deployment_trillian-log-server_1:8090'
waitFor: ['ci-ready']
# Collect and submit codecoverage reports
- name: 'gcr.io/cloud-builders/curl'
id: 'codecov.io'
entrypoint: bash
args: ['-c', 'bash <(curl -s https://codecov.io/bash)']
env:
- 'VCS_COMMIT_ID=$COMMIT_SHA'
- 'VCS_BRANCH_NAME=$BRANCH_NAME'
- 'VCS_PULL_REQUEST=$_PR_NUMBER'
- 'CI_BUILD_ID=$BUILD_ID'
- 'CODECOV_TOKEN=$_CODECOV_TOKEN' # _CODECOV_TOKEN is specified in the cloud build trigger
waitFor: ['etcd_with_coverage']
- name: gcr.io/$PROJECT_ID/ct_testbase
id: 'ci_complete'
entrypoint: /bin/true
waitFor: ['codecov.io', 'default_test', 'race_detection', 'etcd_with_coverage', 'etcd_with_race', 'with_pkcs11_and_race']
############################################################################
## End of replicated section.
## Below are deployment specific steps for the CD env.
############################################################################
- id: build_ctfe
name: gcr.io/cloud-builders/docker
args:
- build
- --file=trillian/examples/deployment/docker/ctfe/Dockerfile
- --tag=gcr.io/${PROJECT_ID}/ctfe:${TAG_NAME}
- --cache-from=gcr.io/${PROJECT_ID}/ctfe
- .
images:
- gcr.io/${PROJECT_ID}/ctfe:${TAG_NAME}
- gcr.io/${PROJECT_ID}/ct_testbase:latest
|
