1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
|
# Copyright 2017: Mirantis Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import configparser
import os
import re
import requests
from rally.common import logging
from rally import exceptions
from rally.task import utils as task_utils
from rally.verification import context
from rally.verification import utils
from rally_openstack.common import consts
from rally_openstack.common import credential
from rally_openstack.common.services.image import image
from rally_openstack.common.services.network import neutron
from rally_openstack.verification.tempest import config as conf
LOG = logging.getLogger(__name__)
@context.configure("tempest", order=900)
class TempestContext(context.VerifierContext):
"""Context class to create/delete resources needed for Tempest."""
RESOURCE_NAME_FORMAT = "rally_verify_XXXXXXXX_XXXXXXXX"
def __init__(self, ctx):
super(TempestContext, self).__init__(ctx)
openstack_platform = self.verifier.env.data["platforms"]["openstack"]
admin_creds = credential.OpenStackCredential(
permission=consts.EndpointPermission.ADMIN,
**openstack_platform["platform_data"]["admin"])
self.clients = admin_creds.clients()
self.available_services = self.clients.services().values()
self.conf = configparser.ConfigParser(allow_no_value=True)
self.conf.optionxform = str
self.conf_path = self.verifier.manager.configfile
self.data_dir = self.verifier.manager.home_dir
self.image_name = "tempest-image"
self._created_roles = []
self._created_images = []
self._created_flavors = []
self._created_networks = []
def _configure_img_options(self):
try:
tempest_major_version = int(self.verifier.version.split(".", 1)[0])
except ValueError:
# use latest flow by default
tempest_major_version = 27
if tempest_major_version < 27:
self._configure_option("scenario", "img_dir", self.data_dir)
img_file = self.image_name
else:
img_file = self.data_dir + "/" + self.image_name
self._configure_option("scenario", "img_file", img_file,
helper_method=self._download_image)
def setup(self):
self.conf.read(self.conf_path)
utils.create_dir(self.data_dir)
self._create_tempest_roles()
self._configure_option("DEFAULT", "log_file",
os.path.join(self.data_dir, "tempest.log"))
self._configure_option("oslo_concurrency", "lock_path",
os.path.join(self.data_dir, "lock_files"))
self._configure_img_options()
self._configure_option("compute", "image_ref",
helper_method=self._discover_or_create_image)
self._configure_option("compute", "image_ref_alt",
helper_method=self._discover_or_create_image)
self._configure_option("compute", "flavor_ref",
helper_method=self._discover_or_create_flavor,
flv_ram=conf.CONF.openstack.flavor_ref_ram,
flv_disk=conf.CONF.openstack.flavor_ref_disk)
self._configure_option("compute", "flavor_ref_alt",
helper_method=self._discover_or_create_flavor,
flv_ram=conf.CONF.openstack.flavor_ref_alt_ram,
flv_disk=conf.CONF.openstack.flavor_ref_alt_disk
)
if "neutron" in self.available_services:
neutronclient = self.clients.neutron()
if neutronclient.list_networks(shared=True)["networks"]:
# If the OpenStack cloud has some shared networks, we will
# create our own shared network and specify its name in the
# Tempest config file. Such approach will allow us to avoid
# failures of Tempest tests with error "Multiple possible
# networks found". Otherwise the default behavior defined in
# Tempest will be used and Tempest itself will manage network
# resources.
LOG.debug("Shared networks found. "
"'fixed_network_name' option should be configured.")
self._configure_option(
"compute", "fixed_network_name",
helper_method=self._create_network_resources)
if "heat" in self.available_services:
self._configure_option(
"orchestration", "instance_type",
helper_method=self._discover_or_create_flavor,
flv_ram=conf.CONF.openstack.heat_instance_type_ram,
flv_disk=conf.CONF.openstack.heat_instance_type_disk)
with open(self.conf_path, "w") as configfile:
self.conf.write(configfile)
def cleanup(self):
# Tempest tests may take more than 1 hour and we should remove all
# cached clients sessions to avoid tokens expiration when deleting
# Tempest resources.
self.clients.clear()
self._cleanup_tempest_roles()
self._cleanup_images()
self._cleanup_flavors()
if "neutron" in self.available_services:
self._cleanup_network_resources()
with open(self.conf_path, "w") as configfile:
self.conf.write(configfile)
def _create_tempest_roles(self):
keystoneclient = self.clients.verified_keystone()
roles = [conf.CONF.openstack.swift_operator_role,
conf.CONF.openstack.swift_reseller_admin_role,
conf.CONF.openstack.heat_stack_owner_role,
conf.CONF.openstack.heat_stack_user_role]
existing_roles = set(role.name.lower()
for role in keystoneclient.roles.list())
for role in roles:
if role.lower() not in existing_roles:
LOG.debug("Creating role '%s'." % role)
self._created_roles.append(keystoneclient.roles.create(role))
def _configure_option(self, section, option, value=None,
helper_method=None, *args, **kwargs):
option_value = self.conf.get(section, option)
if not option_value:
LOG.debug("Option '%s' from '%s' section is not configured."
% (option, section))
if helper_method:
res = helper_method(*args, **kwargs)
if res:
value = res["network"]["name"] if ("network" in
option) else res.id
LOG.debug("Setting value '%s' to option '%s'." % (value, option))
self.conf.set(section, option, value)
LOG.debug("Option '{opt}' is configured. "
"{opt} = {value}".format(opt=option, value=value))
else:
LOG.debug("Option '{opt}' is already configured "
"in Tempest config file. {opt} = {opt_val}"
.format(opt=option, opt_val=option_value))
def _discover_image(self):
LOG.debug("Trying to discover a public image with name matching "
"regular expression '%s'. Note that case insensitive "
"matching is performed."
% conf.CONF.openstack.img_name_regex)
image_service = image.Image(self.clients)
images = image_service.list_images(status="active",
visibility="public")
for image_obj in images:
if image_obj.name and re.match(conf.CONF.openstack.img_name_regex,
image_obj.name, re.IGNORECASE):
LOG.debug("The following public image discovered: '%s'."
% image_obj.name)
return image_obj
LOG.debug("There is no public image with name matching regular "
"expression '%s'." % conf.CONF.openstack.img_name_regex)
def _download_image_from_source(self, target_path, image=None):
if image:
LOG.debug("Downloading image '%s' from Glance to %s."
% (image.name, target_path))
with open(target_path, "wb") as image_file:
for chunk in self.clients.glance().images.data(image.id):
image_file.write(chunk)
else:
LOG.debug("Downloading image from %s to %s."
% (conf.CONF.openstack.img_url, target_path))
try:
response = requests.get(conf.CONF.openstack.img_url,
stream=True)
except requests.ConnectionError as err:
msg = ("Failed to download image. Possibly there is no "
"connection to Internet. Error: %s."
% (str(err) or "unknown"))
raise exceptions.RallyException(msg)
if response.status_code == 200:
with open(target_path, "wb") as image_file:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
image_file.write(chunk)
image_file.flush()
else:
if response.status_code == 404:
msg = "Failed to download image. Image was not found."
else:
msg = ("Failed to download image. HTTP error code %d."
% response.status_code)
raise exceptions.RallyException(msg)
LOG.debug("The image has been successfully downloaded!")
def _download_image(self):
image_path = os.path.join(self.data_dir, self.image_name)
if os.path.isfile(image_path):
LOG.debug("Image is already downloaded to %s." % image_path)
return
if conf.CONF.openstack.img_name_regex:
image = self._discover_image()
if image:
return self._download_image_from_source(image_path, image)
self._download_image_from_source(image_path)
def _discover_or_create_image(self):
if conf.CONF.openstack.img_name_regex:
image_obj = self._discover_image()
if image_obj:
LOG.debug("Using image '%s' (ID = %s) for the tests."
% (image_obj.name, image_obj.id))
return image_obj
params = {
"image_name": self.generate_random_name(),
"disk_format": conf.CONF.openstack.img_disk_format,
"container_format": conf.CONF.openstack.img_container_format,
"image_location": os.path.join(self.data_dir, self.image_name),
"visibility": "public"
}
LOG.debug("Creating image '%s'." % params["image_name"])
image_service = image.Image(self.clients)
image_obj = image_service.create_image(**params)
LOG.debug("Image '%s' (ID = %s) has been successfully created!"
% (image_obj.name, image_obj.id))
self._created_images.append(image_obj)
return image_obj
def _discover_or_create_flavor(self, flv_ram, flv_disk):
novaclient = self.clients.nova()
LOG.debug("Trying to discover a flavor with the following properties: "
"RAM = %(ram)dMB, VCPUs = 1, disk >= %(disk)dGiB." %
{"ram": flv_ram, "disk": flv_disk})
for flavor in novaclient.flavors.list():
if (flavor.ram == flv_ram
and flavor.vcpus == 1 and flavor.disk >= flv_disk):
LOG.debug("The following flavor discovered: '{0}'. "
"Using flavor '{0}' (ID = {1}) for the tests."
.format(flavor.name, flavor.id))
return flavor
LOG.debug("There is no flavor with the mentioned properties.")
params = {
"name": self.generate_random_name(),
"ram": flv_ram,
"vcpus": 1,
"disk": flv_disk
}
LOG.debug("Creating flavor '%s' with the following properties: RAM "
"= %dMB, VCPUs = 1, disk = %dGB." %
(params["name"], flv_ram, flv_disk))
flavor = novaclient.flavors.create(**params)
LOG.debug("Flavor '%s' (ID = %s) has been successfully created!"
% (flavor.name, flavor.id))
self._created_flavors.append(flavor)
return flavor
def _create_network_resources(self):
client = neutron.NeutronService(
clients=self.clients,
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions()
)
tenant_id = self.clients.keystone.auth_ref.project_id
router_create_args = {"project_id": tenant_id}
public_net = None
if self.conf.has_section("network"):
public_net = self.conf.get("network", "public_network_id")
if public_net:
external_gateway_info = {
"network_id": public_net
}
if client.supports_extension("ext-gw-mode", silent=True):
external_gateway_info["enable_snat"] = True
router_create_args["external_gateway_info"] = external_gateway_info
LOG.debug("Creating network resources: network, subnet, router.")
net = client.create_network_topology(
subnets_count=1,
router_create_args=router_create_args,
subnet_create_args={"project_id": tenant_id},
network_create_args={"shared": True, "project_id": tenant_id})
LOG.debug("Network resources have been successfully created!")
self._created_networks.append(net)
return net
def _cleanup_tempest_roles(self):
keystoneclient = self.clients.keystone()
for role in self._created_roles:
LOG.debug("Deleting role '%s'." % role.name)
keystoneclient.roles.delete(role.id)
LOG.debug("Role '%s' has been deleted." % role.name)
def _cleanup_images(self):
image_service = image.Image(self.clients)
for image_obj in self._created_images:
LOG.debug("Deleting image '%s'." % image_obj.name)
self.clients.glance().images.delete(image_obj.id)
task_utils.wait_for_status(
image_obj, ["deleted", "pending_delete"],
check_deletion=True,
update_resource=image_service.get_image,
timeout=conf.CONF.openstack.glance_image_delete_timeout,
check_interval=conf.CONF.openstack.
glance_image_delete_poll_interval)
LOG.debug("Image '%s' has been deleted." % image_obj.name)
self._remove_opt_value_from_config("compute", image_obj.id)
def _cleanup_flavors(self):
novaclient = self.clients.nova()
for flavor in self._created_flavors:
LOG.debug("Deleting flavor '%s'." % flavor.name)
novaclient.flavors.delete(flavor.id)
LOG.debug("Flavor '%s' has been deleted." % flavor.name)
self._remove_opt_value_from_config("compute", flavor.id)
self._remove_opt_value_from_config("orchestration", flavor.id)
def _cleanup_network_resources(self):
client = neutron.NeutronService(
clients=self.clients,
name_generator=self.generate_random_name,
atomic_inst=self.atomic_actions()
)
for topo in self._created_networks:
LOG.debug("Deleting network resources: router, subnet, network.")
client.delete_network_topology(topo)
self._remove_opt_value_from_config("compute",
topo["network"]["name"])
LOG.debug("Network resources have been deleted.")
def _remove_opt_value_from_config(self, section, opt_value):
for option, value in self.conf.items(section):
if opt_value == value:
LOG.debug("Removing value '%s' of option '%s' "
"from Tempest config file." % (opt_value, option))
self.conf.set(section, option, "")
LOG.debug("Value '%s' has been removed." % opt_value)
|
