1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
name: CLI REGRESSION TEST $(Date:yyyyMMdd)$(Rev:.r)
resources:
- repo: self
trigger:
branches:
exclude:
- '*'
variables:
- template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml
jobs:
- job: UpdateVersionFiles
displayName: CLI Bump Version Update Version Files
condition: ne(variables['PACKAGE'], '')
pool:
name: ${{ variables.ubuntu_pool }}
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.13'
displayName: "Use Python 3.13"
- task: AzureCLI@2
displayName: 'update version'
inputs:
connectedServiceNameARM: $(azure-cli-live-test-msft-connected-service)
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -ev
# git config
GITHUB_TOKEN=$(az keyvault secret show --vault-name kv-azuresdk --name azclibot-pat --query value -otsv)
git config --global user.email "AzPyCLI@microsoft.com"
git config --global user.name "Azure CLI Team"
git remote add azclibot https://azclibot:${GITHUB_TOKEN}@github.com/azclibot/azure-cli.git
git checkout -b regression_test_$(Build.BuildId)
echo package: $(PACKAGE) $(TARGET_PACKAGE_VERSION)
echo resource_type: $(RESOURCE_TYPE) $(TARGET_API_VERSION)
if [[ -z "$(PACKAGE)" || -z "$(TARGET_PACKAGE_VERSION)" ]]; then
echo "'PACKAGE' and 'TARGET_PACKAGE_VERSION' are required"
exit 1
fi
sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py
sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py
sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.windows.txt
sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Linux.txt
sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Darwin.txt
if [[ -n "$(RESOURCE_TYPE)" ]]; then
python scripts/regression_test/update_api_profile.py "$(RESOURCE_TYPE)" "$(TARGET_API_VERSION)"
fi
git add .
git commit -m "update version"
git push --set-upstream azclibot regression_test_$(Build.BuildId)
- job: RerunTests
displayName: CLI Regression tests
dependsOn: UpdateVersionFiles
condition: in(dependencies.UpdateVersionFiles.result, 'Succeeded', 'Skipped')
timeoutInMinutes: 0
strategy:
maxParallel: 8
matrix:
instance1:
Instance_idx: 1
instance2:
Instance_idx: 2
instance3:
Instance_idx: 3
instance4:
Instance_idx: 4
instance5:
Instance_idx: 5
instance6:
Instance_idx: 6
instance7:
Instance_idx: 7
instance8:
Instance_idx: 8
pool:
name: ${{ variables.ubuntu_pool }}
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '3.13'
displayName: "Use Python 3.13"
- task: AzureCLI@2
displayName: 'Checkout Target Branch'
inputs:
connectedServiceNameARM: $(azure-cli-live-test-msft-connected-service)
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -ev
# git config
if [[ -n "$(CUSTOM_REPO)" && -n "$(CUSTOM_BRANCH)" && -n "$(CUSTOM_GITHUB_TOKEN)" ]]; then
GITHUB_REPO=$(CUSTOM_REPO)
GITHUB_BRANCH=$(CUSTOM_BRANCH)
GITHUB_TOKEN=$(CUSTOM_GITHUB_TOKEN)
else
GITHUB_REPO="azclibot"
GITHUB_BRANCH="regression_test_$(Build.BuildId)"
GITHUB_TOKEN=$(az keyvault secret show --vault-name kv-azuresdk --name azclibot-pat --query value -otsv)
fi
git config --global user.email "AzPyCLI@microsoft.com"
git config --global user.name "Azure CLI Team"
git remote add ${GITHUB_REPO} https://${GITHUB_REPO}:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}/azure-cli.git
git fetch ${GITHUB_REPO} ${GITHUB_BRANCH}
git checkout -b ${GITHUB_BRANCH} ${GITHUB_REPO}/${GITHUB_BRANCH}
- template: ../../.azure-pipelines/templates/azdev_setup.yml
- task: AzureCLI@2
displayName: 'Rerun tests'
inputs:
connectedServiceNameARM: $(azure-cli-live-test-bami-connected-service)
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -ev
source env/bin/activate
if [[ -n "$(CUSTOM_WHL_URL)" ]]; then
pip install $(CUSTOM_WHL_URL) --force-reinstall
fi
az account set -s $(azure-cli-live-test-bami-sub-id)
serial_modules="appservice botservice cloud network azure-cli-core azure-cli-telemetry"
python scripts/ci/automation_full_test.py "8" "$(Instance_idx)" "latest" "$serial_modules" "True"
- task: PublishTestResults@2
inputs:
testResultsFiles: '/$(HOME)/.azdev/env_config/mnt/vss/_work/1/s/env/test_results_*.xml'
testRunTitle: 'CLI Regression test results of instance $(Instance_idx)'
- job: CreatePR
displayName: CLI Regression Test Create PR
dependsOn: RerunTests
condition: in(dependencies.RerunTests.result, 'Succeeded', 'Skipped')
pool:
name: ${{ variables.ubuntu_pool }}
steps:
- task: AzureCLI@2
displayName: 'Create PR'
inputs:
connectedServiceNameARM: $(azure-cli-live-test-msft-connected-service)
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -ev
# git config
if [[ -n "$(CUSTOM_REPO)" && -n "$(CUSTOM_BRANCH)" && -n "$(CUSTOM_GITHUB_TOKEN)" ]]; then
GITHUB_TOKEN=$(CUSTOM_GITHUB_TOKEN)
pr_title="Regression Test for $(CUSTOM_REPO)/$(CUSTOM_BRANCH)"
pr_body="Triggered by CLI Regression Test Pipeline - ADO_BUILD_ID=$(Build.BuildId)\n\nRegression test for $(CUSTOM_REPO)/$(CUSTOM_BRANCH)\nThere may still exist some failed tests, see [details](https://dev.azure.com/azure-sdk/internal/_build/results?buildId=$(Build.BuildId)&view=ms.vss-test-web.build-test-results-tab) in ADO pipeline"
pr_head="$(CUSTOM_REPO):$(CUSTOM_BRANCH)"
else
GITHUB_TOKEN=$(az keyvault secret show --vault-name kv-azuresdk --name azclibot-pat --query value -otsv)
pr_title="Regression Test for $(PACKAGE) to $(TARGET_PACKAGE_VERSION)"
pr_body="Triggered by CLI Regression Test Pipeline - ADO_BUILD_ID=$(Build.BuildId)\n\npackage: $(PACKAGE) $(TARGET_PACKAGE_VERSION)\nresource_type: $(RESOURCE_TYPE) $(TARGET_API_VERSION)\n\nThere may still exist some failed tests, see [details](https://dev.azure.com/azure-sdk/internal/_build/results?buildId=$(Build.BuildId)&view=ms.vss-test-web.build-test-results-tab) in ADO pipeline"
pr_head="azclibot:bump_version_$(Build.BuildId)"
fi
git config --global user.email "AzPyCLI@microsoft.com"
git config --global user.name "Azure CLI Team"
curl \
-H "Authorization: token ${GITHUB_TOKEN}" \
-d "{\"title\": \"${pr_title}\", \"body\": \"${pr_body}\", \"head\": \"${pr_head}\", \"base\": \"dev\"}" \
https://api.github.com/repos/Azure/azure-cli/pulls
|
