1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
|
package base
// This file contains gerrithub related definitions etc
import (
"encoding/json"
"strings"
"github.com/cue-tmp/jsonschema-pub/exp1/githubactions"
)
// trybotWorkflows is a template for trybot-based repos
trybotWorkflows: {
(trybot.key): githubactions.#Workflow & {
on: workflow_dispatch: {}
}
"\(trybot.key)_dispatch": trybotDispatchWorkflow
"push_tip_to_\(trybot.key)": pushTipToTrybotWorkflow
"evict_caches": evictCaches
}
#dispatch: {
type: string
CL: int
patchset: int
targetBranch: *defaultBranch | string
let p = strings.Split("\(CL)", "")
let rightMostTwo = p[len(p)-2] + p[len(p)-1]
ref: *"refs/changes/\(rightMostTwo)/\(CL)/\(patchset)" | string
}
trybotDispatchWorkflow: bashWorkflow & {
#dummyDispatch?: #dispatch
name: "Dispatch \(trybot.key)"
on: {
repository_dispatch: {}
push: {
// To enable testing of the dispatch itself
branches: [testDefaultBranch]
}
}
jobs: {
(trybot.key): {
"runs-on": linuxMachine
let goodDummyData = [if json.Marshal(#dummyDispatch) != _|_ {true}, false][0]
// We set the "on" conditions above, but this would otherwise mean we
// run for all dispatch events.
if: "${{ (\(isTestDefaultBranch) && \(goodDummyData)) || github.event.client_payload.type == '\(trybot.key)' }}"
// See the comment below about the need for cases
let cases = [
{
condition: "!="
expr: "fromJSON(steps.payload.outputs.value)"
nameSuffix: "fake data"
},
{
condition: "=="
expr: "github.event.client_payload"
nameSuffix: "repository_dispatch payload"
},
]
steps: [
writeNetrcFile,
githubactions.#Step & {
name: "Write fake payload"
id: "payload"
if: "github.repository == '\(githubRepositoryPath)' && \(isTestDefaultBranch)"
// Use bash heredocs so that JSON's use of double quotes does
// not get interpreted as shell. Both in the running of the
// command itself, which itself is the echo-ing of a command to
// $GITHUB_OUTPUT.
run: #"""
cat <<EOD >> $GITHUB_OUTPUT
value<<DOE
\#(*json.Marshal(#dummyDispatch) | "null")
DOE
EOD
"""#
},
// GitHub does not allow steps with the same ID, even if (by virtue
// of runtime 'if' expressions) both would not actually run. So
// we have to duplciate the steps that follow with those same
// runtime expressions
//
// Hence we have to create two steps, one to trigger if the
// repository_dispatch payload is set, and one if not (i.e. we use
// the fake payload).
for v in cases {
let localBranchExpr = "local_${{ \(v.expr).targetBranch }}"
let targetBranchExpr = "${{ \(v.expr).targetBranch }}"
githubactions.#Step & {
name: "Trigger \(trybot.name) (\(v.nameSuffix))"
if: "github.event.client_payload.type \(v.condition) '\(trybot.key)'"
run: """
mkdir tmpgit
cd tmpgit
git init -b initialbranch
git config user.name \(botGitHubUser)
git config user.email \(botGitHubUserEmail)
git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(botGitHubUser):${{ secrets.\(botGitHubUserTokenSecretsKey) }} | base64)"
git remote add origin \(gerritHubRepositoryURL)
git fetch origin ${{ \(v.expr).ref }}
git checkout -b \(localBranchExpr) FETCH_HEAD
# Error if we already have dispatchTrailer according to git log logic.
# See earlier check for GitHub expression logic check.
x="$(git log -1 --pretty='%(trailers:key=\(dispatchTrailer),valueonly)')"
if [[ "$x" != "" ]]
then
echo "Ref ${{ \(v.expr).ref }} already has a \(dispatchTrailer)"
exit 1
fi
# Add the trailer because we don't have it yet. GitHub expressions do not have a
# substitute or quote capability. So we do that in shell. We also strip out the
# indenting added by toJSON. We ensure that the type field is first in order
# that we can safely check for specific types of dispatch trailer.
#
# Use bash heredoc so that JSON's use of double quotes does
# not get interpreted as shell.
trailer="$(cat <<EOD | jq -r -c '{type} + .'
${{ toJSON(\(v.expr)) }}
EOD
)"
git log -1 --format=%B | git interpret-trailers --trailer "\(dispatchTrailer): $trailer" | git commit --amend -F -
git log -1
success=false
for try in {1..20}; do
echo "Push to trybot try $try"
if git push -f \(trybotRepositoryURL) \(localBranchExpr):\(targetBranchExpr); then
success=true
break
fi
sleep 1
done
if ! $success; then
echo "Giving up"
exit 1
fi
"""
}
},
]
}
}
}
pushTipToTrybotWorkflow: bashWorkflow & {
on: {
push: branches: protectedBranchPatterns
}
jobs: push: {
"runs-on": linuxMachine
if: "${{github.repository == '\(githubRepositoryPath)'}}"
}
name: "Push tip to \(trybot.key)"
concurrency: "push_tip_to_trybot"
jobs: push: {
steps: [
writeNetrcFile,
githubactions.#Step & {
name: "Push tip to trybot"
run: """
mkdir tmpgit
cd tmpgit
git init -b initialbranch
git config user.name \(botGitHubUser)
git config user.email \(botGitHubUserEmail)
git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(botGitHubUser):${{ secrets.\(botGitHubUserTokenSecretsKey) }} | base64)"
git remote add origin \(gerritHubRepositoryURL)
git remote add trybot \(trybotRepositoryURL)
git fetch origin "${{ github.ref }}"
success=false
for try in {1..20}; do
echo "Push to trybot try $try"
if git push -f trybot "FETCH_HEAD:${{ github.ref }}"; then
success=true
break
fi
sleep 1
done
if ! $success; then
echo "Giving up"
exit 1
fi
"""
},
]
}
}
// evictCaches removes "old" GitHub actions caches from the main repo and the
// accompanying trybot The job is only run in the main repo, because
// that is the only place where the credentials exist.
//
// The GitHub actions caches in the main and trybot repos can get large. So
// large in fact we got the following warning from GitHub:
//
// "Approaching total cache storage limit (34.5 GB of 10 GB Used)"
//
// Yes, you did read that right.
//
// Not only does this have the effect of causing us to breach "limits" it also
// means that we can't be sure that individual caches are not bloated.
//
// Fix that by purging the actions caches on a daily basis at 0200, followed 15
// mins later by a re-run of the tip trybots to repopulate the caches so they
// are warm and minimal.
//
// In testing with @mvdan, this resulted in cache sizes for Linux dropping from
// ~1GB to ~125MB. This is a considerable saving.
//
// Note this currently removes all cache entries, regardless of whether they
// are go-related or not. We should revisit this later.
evictCaches: bashWorkflow & {
name: "Evict caches"
on: {
schedule: [
{cron: "0 2 * * *"},
]
}
jobs: {
test: {
// We only want to run this in the main repo
if: "${{github.repository == '\(githubRepositoryPath)'}}"
"runs-on": linuxMachine
steps: [
for v in checkoutCode {v},
githubactions.#Step & {
name: "Delete caches"
run: """
set -x
echo ${{ secrets.\(botGitHubUserTokenSecretsKey) }} | gh auth login --with-token
gh extension install actions/gh-actions-cache
for i in \(githubRepositoryURL) \(trybotRepositoryURL)
do
echo "Evicting caches for $i"
cd $(mktemp -d)
git init -b initialbranch
git remote add origin $i
for j in $(gh actions-cache list -L 100 | grep refs/ | awk '{print $1}')
do
gh actions-cache delete --confirm $j
done
done
"""
},
githubactions.#Step & {
name: "Trigger workflow runs to repopulate caches"
let branchPatterns = strings.Join(protectedBranchPatterns, " ")
run: """
# Prepare git for pushes to trybot repo. Note
# because we have already checked out code we don't
# need origin. Fetch origin default branch for later use
git config user.name \(botGitHubUser)
git config user.email \(botGitHubUserEmail)
git config http.https://github.com/.extraheader "AUTHORIZATION: basic $(echo -n \(botGitHubUser):${{ secrets.\(botGitHubUserTokenSecretsKey) }} | base64)"
git remote add trybot \(trybotRepositoryURL)
# Now trigger the most recent workflow run on each of the default branches.
# We do this by listing all the branches on the main repo and finding those
# which match the protected branch patterns (globs).
for j in $(\(curlGitHubAPI) -f https://api.github.com/repos/\(githubRepositoryPath)/branches | jq -r '.[] | .name')
do
for i in \(branchPatterns)
do
if [[ "$j" != $i ]]; then
continue
fi
echo Branch: $j
sha=$(\(curlGitHubAPI) "https://api.github.com/repos/\(githubRepositoryPath)/commits/$j" | jq -r '.sha')
echo Latest commit: $sha
echo "Trigger workflow on \(githubRepositoryPath)"
\(curlGitHubAPI) --fail-with-body -X POST https://api.github.com/repos/\(githubRepositoryPath)/actions/workflows/\(trybot.key+workflowFileExtension)/dispatches -d "{\\"ref\\":\\"$j\\"}"
# Ensure that the trybot repo has the latest commit for
# this branch. If the force-push results in a commit
# being pushed, that will trigger the trybot workflows
# so we don't need to do anything, otherwise we need to
# trigger the most recent commit on that branch
git remote -v
git fetch origin refs/heads/$j
git log -1 FETCH_HEAD
success=false
for try in {1..20}; do
echo "Push to trybot try $try"
exitCode=0; push="$(git push -f trybot FETCH_HEAD:$j 2>&1)" || exitCode=$?
echo "$push"
if [[ $exitCode -eq 0 ]]; then
success=true
break
fi
sleep 1
done
if ! $success; then
echo "Giving up"
exit 1
fi
if echo "$push" | grep up-to-date
then
# We are up-to-date, i.e. the push did nothing, hence we need to trigger a workflow_dispatch
# in the trybot repo.
echo "Trigger workflow on \(trybotRepositoryPath)"
\(curlGitHubAPI) --fail-with-body -X POST https://api.github.com/repos/\(trybotRepositoryPath)/actions/workflows/\(trybot.key+workflowFileExtension)/dispatches -d "{\\"ref\\":\\"$j\\"}"
else
echo "Force-push to \(trybotRepositoryPath) did work; nothing to do"
fi
done
done
"""
},
]
}
}
}
writeNetrcFile: githubactions.#Step & {
name: "Write netrc file for \(botGerritHubUser) Gerrithub"
run: """
cat <<EOD > ~/.netrc
machine \(gerritHubHostname)
login \(botGerritHubUser)
password ${{ secrets.\(botGerritHubUserPasswordSecretsKey) }}
EOD
chmod 600 ~/.netrc
"""
}
|