1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
|
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2025 Arm Limited
"""Artifact module.
This module provides the :class:`Artifact` class, which represents a file artifact that can be or is
stored on a remote node or locally.
Example usage of the :class:`Artifact` class:
.. code:: python
from api.artifact import Artifact
# Create an artifact on a remote node
artifact = Artifact(node="sut", file_name="example.txt")
# Open the artifact file in write mode
with artifact.open("w") as f:
f.write("Hello, World!")
# Pull the artifact to the local output directory
artifact.save_locally() # This is also done automatically on object deletion
# if save_local_copy is set to True
# Check if the artifact exists
if artifact.exists():
print(f"Artifact exists at {artifact.path}")
# Delete the artifact
artifact.delete()
# Create an artifact from a local file
local_artifact = Artifact.create_from(
original_file="local_file.txt",
node="sut",
file_name="copied_file.txt",
)
# Copy the content of the local artifact to another artifact
another_artifact = Artifact("sut", "another_file.txt")
another_artifact.copy_contents_from(local_artifact)
"""
import shutil
import uuid
from collections.abc import Iterable
from io import SEEK_SET, RawIOBase, TextIOWrapper
from pathlib import Path, PurePath
from typing import BinaryIO, ClassVar, Literal, TypeAlias, Union, cast, overload
from paramiko import SFTPClient, SFTPFile
from typing_extensions import Buffer
from framework.exception import InternalError
from framework.logger import DTSLogger, get_dts_logger
from framework.settings import SETTINGS
from framework.testbed_model.node import Node, NodeIdentifier, get_node
TextMode: TypeAlias = (
Literal["r", "r+", "w", "w+", "a", "a+", "x", "x+"]
| Literal["rt", "r+t", "wt", "w+t", "at", "a+t", "xt", "x+t"]
)
"""Open text mode for artifacts."""
BinaryMode: TypeAlias = Literal["rb", "r+b", "wb", "w+b", "ab", "a+b", "xb", "x+b"]
"""Open binary mode for artifacts."""
OpenMode: TypeAlias = TextMode | BinaryMode
"""Open mode for artifacts, can be either text or binary mode."""
@overload
def make_file_path(node: Node, file_name: str, custom_path: PurePath | None = None) -> PurePath: ...
@overload
def make_file_path(node: None, file_name: str, custom_path: PurePath | None = None) -> Path: ...
def make_file_path(
node: Node | None, file_name: str, custom_path: PurePath | None = None
) -> Path | PurePath:
"""Make a file path for the artifact."""
if node is None:
path: Path | PurePath = Path(SETTINGS.output_dir).resolve()
else:
path = node.tmp_dir
if custom_path is not None:
if custom_path.is_absolute():
return custom_path / file_name
path /= custom_path
else:
from framework.context import get_ctx
try:
ctx = get_ctx()
if ctx.local.current_test_suite is not None:
path /= ctx.local.current_test_suite.name
if ctx.local.current_test_case is not None:
path /= ctx.local.current_test_case.name
except InternalError:
# If the context is not available, use the root path.
pass
return path / file_name
def make_unique_file_name() -> str:
"""Generate a unique filename for the artifact."""
return f"{uuid.uuid4().hex}.dat"
class Artifact:
"""Artifact class.
Represents a file artifact that can be or is stored on a remote node or locally. It provides
methods to open, read, write, and manage the artifact file, in the same familiar Python API. It
also provides functionality to save a local copy of the artifact if it is remote – saved in the
test run output directory. It can be used to manage files that are part of the test run, such as
logs, reports, or any other files that need to be stored for later analysis.
By default, the artifact is created in the temporary directory of the node, following the tree
directory structure defined by :class:`DirectoryTree` and managed by the test run states.
The artifact file is not created automatically upon instantiation. The methods :meth:`open` –
with either `w` or `a` modes – and :meth:`touch` can be used to create it.
If `save_local_copy` is :data:`True` and there already exist a local file with the same name, it
will be overwritten. If this is undesired, make sure to give distinct names to the artifacts.
Attributes:
save_local_copy: If :data:`True`, a local copy of the artifact will be saved at the end of
the lifetime of the object automatically.
"""
DIRECTORY_PERMISSIONS: ClassVar[int] = 0o755
"""Permission mode for directories created by the artifact."""
TEXT_MODE_ENCODING: ClassVar[str] = "utf-8"
"""Encoding used for text mode artifacts."""
TEXT_MODE_NEWLINE: ClassVar[str] = "\n"
"""Newline character used for text mode artifacts."""
_logger: DTSLogger
_node: Node | None = None
_sftp: Union[SFTPClient, None] = None
_fd: Union["ArtifactFile", None] = None
_file_path: PurePath
_local_path: Path
_file_was_saved: bool = False
_directories_created: bool = False
save_local_copy: bool
def __init__(
self,
node: NodeIdentifier,
file_name: str = "",
save_local_copy: bool = True,
custom_path: PurePath | None = None,
):
"""Constructor for an artifact.
Args:
node: The node identifier on which the file is.
file_name: The name of the file. If not provided, a unique filename will be generated.
save_local_copy: If :data:`True`, makes a local copy of the artifact in the output
directory. Applies only to remote artifacts.
custom_path: A custom path to save the artifact. If :data:`None`, the default path will
be used based on the node identifier. If a relative path is provided, it will be
relative to the remote temporary directory (for remote artifacts) and local output
directory (for local artifacts and copies).
"""
self._logger = get_dts_logger(f"{node}_artifact")
self._node = get_node(node)
self.save_local_copy = save_local_copy
if not file_name:
file_name = make_unique_file_name()
if self._node is not None:
self._sftp = self._node.main_session.remote_session.session.sftp()
if custom_path is not None and not custom_path.is_absolute():
relative_custom_path = custom_path
else:
relative_custom_path = None
self._file_path = make_file_path(self._node, file_name, custom_path)
self._local_path = make_file_path(
node=None, file_name=file_name, custom_path=relative_custom_path
)
else:
self._local_path = self._file_path = make_file_path(self._node, file_name, custom_path)
@overload
def open(
self,
file_mode: BinaryMode = "rb",
buffering: int = -1,
) -> "ArtifactFile": ...
@overload
def open(
self,
file_mode: TextMode = "r",
buffering: int = -1,
) -> TextIOWrapper: ...
def open(
self, file_mode: BinaryMode | TextMode = "rb", buffering: int = -1
) -> Union["ArtifactFile", TextIOWrapper]:
"""Open the artifact file.
Args:
file_mode: The mode of file opening.
buffering: The size of the buffer to use. If -1, the default buffer size is used.
Returns:
An instance of :class:`ArtifactFile` or :class:`TextIOWrapper`.
"""
if self._fd is not None and not self._fd.closed:
self._logger.warning(
f"Artifact {self.path} is already open. Closing the previous file descriptor."
)
self._fd.close()
elif not self._directories_created:
self.mkdir()
# SFTPFile does not support text mode, therefore everything needs to be handled as binary.
if "t" in file_mode:
actual_mode = cast(BinaryMode, cast(str, file_mode).replace("t", "") + "b")
elif "b" not in file_mode:
actual_mode = cast(BinaryMode, file_mode + "b")
else:
actual_mode = cast(BinaryMode, file_mode)
self._logger.debug(f"Opening file at {self.path} with mode {file_mode}.")
if self._sftp is None:
fd: BinaryIO | SFTPFile = open(str(self.path), mode=actual_mode, buffering=buffering)
else:
fd = self._sftp.open(str(self.path), mode=actual_mode, bufsize=buffering)
self._fd = ArtifactFile(fd, self.path, file_mode)
if "b" in file_mode:
return self._fd
else:
return TextIOWrapper(
self._fd,
encoding=self.TEXT_MODE_ENCODING,
newline=self.TEXT_MODE_NEWLINE,
write_through=True,
)
@classmethod
def create_from(
cls,
original_file: Union[Path, "Artifact"],
node: NodeIdentifier,
/,
new_file_name: str = "",
save_local_copy: bool = False,
custom_path: PurePath | None = None,
) -> "Artifact":
"""Create a new artifact from a local file or another artifact.
Args:
node: The node identifier on which the file is.
original_file: The local file or artifact to copy.
new_file_name: The name of the new file. If not provided, the name of the original file
will be used.
save_local_copy: Makes a local copy of the artifact if :data:`True`. Applies only to
remote files.
custom_path: A custom path to save the artifact. If :data:`None`, the default path will
be used based on the node identifier.
Returns:
An instance of :class:`Artifact`.
"""
if not new_file_name:
if isinstance(original_file, Artifact):
new_file_name = original_file.local_path.name
else:
new_file_name = original_file.name
artifact = cls(node, new_file_name, save_local_copy, custom_path)
artifact.copy_contents_from(original_file)
return artifact
def copy_contents_from(self, original_file: Union[Path, "Artifact"]) -> None:
"""Copy the content of another file or artifact into this artifact.
This action will close the file descriptor associated with `self` or `original_file` if
open.
Args:
original_file: The local file or artifact to copy.
Raises:
InternalError: If the provided `original_file` does not exist.
"""
if isinstance(original_file, Path) and not original_file.exists():
raise InternalError(f"The provided file '{original_file}' does not exist.")
self.open("wb").close() # Close any prior fd and truncate file.
self._logger.debug(f"Copying content from {original_file} to {self}.")
match (original_file, self._sftp):
case (Path(), None): # local file to local artifact
# Use syscalls to copy
shutil.copyfile(original_file, self.path)
case (Artifact(_sftp=None), None): # local artifact to local artifact
# Use syscalls to copy
shutil.copyfile(original_file.local_path, self.path)
case (Artifact(), None): # remote artifact to local artifact
# Use built-in chunked transfer copy
with original_file.open("rb") as original_fd:
with self.open("wb") as copy_fd:
shutil.copyfileobj(original_fd, copy_fd)
case (_, SFTPClient()): # remote artifact to remote artifact
# Use SFTPClient's buffered file copy
with original_file.open("rb") as original_fd:
self._sftp.putfo(original_fd, str(self.path))
@property
def path(self) -> PurePath:
"""Return the actual path of the artifact."""
return self._file_path
@property
def local_path(self) -> Path:
"""Return the local path of the artifact."""
return self._local_path
def save_locally(self) -> None:
"""Copy remote artifact file and save it locally. Does nothing on local artifacts.
If there already exist a local file with the same name, it will be overwritten. If this is
undesired, make sure to give distinct names to the artifacts.
"""
if self._sftp is not None:
if not self.exists():
self._logger.debug(f"File {self.path} was never created, skipping save.")
return
self._logger.debug(f"Pulling artifact {self.path} to {self.local_path}.")
if not self._file_was_saved and self.local_path.exists():
self._logger.warning(
f"While saving a remote artifact: local file {self.local_path} already exists, "
"overwriting it. Please use distinct file names."
)
self._file_was_saved = True
self._sftp.get(str(self.path), str(self.local_path))
def delete(self, remove_local_copy: bool = True) -> None:
"""Delete the artifact file. It also prevents a local copy from being saved.
Args:
remove_local_copy: If :data:`True`, the local copy of the artifact will be deleted if
it already exists.
"""
self._logger.debug(f"Deleting artifact {self.path}.")
if self._fd is not None and not self._fd.closed:
self._fd.close()
self._fd = None
if self._sftp is not None:
self._sftp.remove(str(self._file_path))
if self._sftp is None or remove_local_copy:
self.local_path.unlink(missing_ok=True)
def touch(self, mode: int = 0o644) -> None:
"""Touch the artifact file, creating it if it does not exist.
Args:
mode: The permission mode to set for the artifact file, if just created.
"""
if not self._directories_created:
self.mkdir()
self._logger.debug(f"Touching artifact {self.path} with mode {oct(mode)}.")
if self._sftp is not None:
file_path = str(self._file_path)
try:
self._sftp.stat(file_path)
except FileNotFoundError:
self._sftp.open(file_path, "w").close()
self._sftp.chmod(file_path, mode)
else:
Path(self._file_path).touch(mode=mode)
def chmod(self, mode: int = 0o644) -> None:
"""Change the permissions of the artifact file.
Args:
mode: The permission mode to set for the artifact file.
"""
self._logger.debug(f"Changing permissions of {self.path} to {oct(mode)}.")
if self._sftp is not None:
self._sftp.chmod(str(self._file_path), mode)
else:
Path(self._file_path).chmod(mode)
def exists(self) -> bool:
"""Check if the artifact file exists.
Returns:
:data:`True` if the artifact file exists, :data:`False` otherwise.
"""
if self._sftp is not None:
try:
self._sftp.stat(str(self._file_path))
return True
except FileNotFoundError:
return False
else:
return self._local_path.exists()
def mkdir(self) -> None:
"""Create all the intermediate file path directories."""
if self._sftp is not None:
parts = self._file_path.parts[:-1]
paths = (PurePath(*parts[:tree_depth]) for tree_depth in range(1, len(parts) + 1))
for path in paths:
try:
self._sftp.stat(str(path))
except FileNotFoundError:
self._logger.debug(f"Creating directories {path}.")
self._sftp.mkdir(str(path), mode=self.DIRECTORY_PERMISSIONS)
if self._sftp is None or self.save_local_copy:
self._logger.debug(f"Creating directories {self.local_path.parent} locally.")
self.local_path.parent.mkdir(
mode=self.DIRECTORY_PERMISSIONS, parents=True, exist_ok=True
)
self._directories_created = True
def __del__(self):
"""Close the file descriptor if it is open and save it if requested."""
if self._fd is not None and not self._fd.closed:
self._fd.close()
self._fd = None
if self.save_local_copy:
self.save_locally()
def __str__(self):
"""Return path of the artifact."""
return str(self.path)
class ArtifactFile(RawIOBase, BinaryIO):
"""Artifact file wrapper class.
Provides a single interface for either local or remote files.
This class implements the :class:`~io.RawIOBase` interface, allowing it to be used
interchangeably with standard file objects.
"""
_fd: Union[BinaryIO, SFTPFile]
_path: PurePath
_mode: OpenMode
def __init__(self, fd: Union[BinaryIO, SFTPFile], path: PurePath, mode: OpenMode):
"""Initialize the artifact file wrapper.
Args:
fd: The file descriptor of the artifact.
path: The path of the artifact file.
mode: The mode in which the artifact file was opened.
"""
super().__init__()
self._fd = fd
self._path = path
self._mode = mode
def close(self) -> None:
"""Close artifact file.
This method implements :meth:`~io.RawIOBase.close()`.
"""
self._fd.close()
def read(self, size: int | None = -1) -> bytes:
"""Read bytes from the artifact file.
This method implements :meth:`~io.RawIOBase.read()`.
"""
return self._fd.read(size if size is not None else -1)
def readline(self, size: int | None = -1) -> bytes:
"""Read line from the artifact file.
This method implements :meth:`~io.RawIOBase.readline()`.
"""
if size is None:
size = -1 # Turning None to -1 due to abstract type mismatch.
return self._fd.readline(size)
def readlines(self, hint: int = -1) -> list[bytes]:
"""Read lines from the artifact file.
This method implements :meth:`~io.RawIOBase.readlines()`.
"""
return self._fd.readlines(hint)
def write(self, data: Buffer) -> int:
"""Write bytes to the artifact file.
Returns the number of bytes written if available, otherwise -1.
This method implements :meth:`~io.RawIOBase.write()`.
"""
return self._fd.write(data) or -1
def writelines(self, lines: Iterable[Buffer]):
"""Write lines to the artifact file.
This method implements :meth:`~io.RawIOBase.writelines()`.
"""
return self._fd.writelines(lines)
def flush(self) -> None:
"""Flush the write buffers to the artifact file if applicable.
This method implements :meth:`~io.RawIOBase.flush()`.
"""
self._fd.flush()
def seek(self, offset: int, whence: int = SEEK_SET) -> int:
"""Change the file position to the given byte offset.
This method implements :meth:`~io.RawIOBase.seek()`.
"""
pos = self._fd.seek(offset, whence)
if pos is None:
return self._fd.tell()
return pos
def tell(self) -> int:
"""Return the current absolute file position.
This method implements :meth:`~io.RawIOBase.tell()`.
"""
return self._fd.tell()
def truncate(self, size: int | None = None) -> int:
"""Change the size of the file to `size` or to the current position.
This method implements :meth:`~io.RawIOBase.truncate()`.
"""
if size is None:
size = self._fd.tell()
new_size = self._fd.truncate(size)
if new_size is None:
return size
return new_size
@property
def name(self) -> str:
"""Return the name of the artifact file.
This method implements :meth:`~io.RawIOBase.name()`.
"""
return str(self._path)
@property
def mode(self) -> str:
"""Return the mode in which the artifact file was opened.
This method implements :meth:`~io.RawIOBase.mode()`.
"""
return self._mode
@property
def closed(self) -> bool:
""":data:`True` if the file is closed."""
return self._fd.closed
def fileno(self) -> int:
"""Return the underlying file descriptor.
This method implements :meth:`~io.RawIOBase.fileno()`.
"""
return self._fd.fileno() if hasattr(self._fd, "fileno") else -1
def isatty(self) -> bool:
"""Return :data:`True` if the file is connected to a terminal device.
This method implements :meth:`~io.RawIOBase.isatty()`.
"""
return self._fd.isatty() if hasattr(self._fd, "isatty") else False
def readable(self) -> bool:
"""Return :data:`True` if the file is readable.
This method implements :meth:`~io.RawIOBase.readable()`.
"""
return self._fd.readable()
def writable(self) -> bool:
"""Return :data:`True` if the file is writable.
This method implements :meth:`~io.RawIOBase.writable()`.
"""
return self._fd.writable()
def seekable(self) -> bool:
"""Return :data:`True` if the file is seekable.
This method implements :meth:`~io.RawIOBase.seekable()`.
"""
return self._fd.seekable()
def __enter__(self) -> "ArtifactFile":
"""Enter the runtime context related to this object.
This method implements :meth:`~io.RawIOBase.__enter__()`.
"""
return self
def __exit__(self, *args) -> None:
"""Exit the runtime context related to this object.
This method implements :meth:`~io.RawIOBase.__exit__()`.
"""
self.close()
|