File: earl.py

package info (click to toggle)
rdflib 6.1.1-1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 38,248 kB
  • sloc: python: 39,216; sh: 153; makefile: 110
file content (287 lines) | stat: -rw-r--r-- 10,844 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
"""
PYTEST_DONT_REWRITE
"""
import enum
import logging
from datetime import datetime
from pathlib import Path
from test.manifest import RDFT
from typing import TYPE_CHECKING, Generator, Optional, Tuple, cast

import pytest

from pytest import Item

from rdflib import RDF, BNode, Graph, Literal, Namespace, URIRef
from rdflib.namespace import DC, DOAP, FOAF, DefinedNamespace
from rdflib.term import Node

if TYPE_CHECKING:
    from _pytest.main import Session
    from _pytest.python import CallSpec2
    from _pytest.reports import TestReport
    from _pytest.runner import CallInfo
    from pluggy._result import _Result


class EARL(DefinedNamespace):
    _fail = True
    _NS = Namespace("http://www.w3.org/ns/earl#")

    assertedBy: URIRef  # assertor of an assertion
    Assertion: URIRef  # a statement that embodies the results of a test
    Assertor: URIRef  # an entity such as a person, a software tool, an organization, or any other grouping that carries out a test collectively
    automatic: URIRef  # where the test was carried out automatically by the software tool and without any human intervention
    CannotTell: URIRef  # the class of outcomes to denote an undetermined outcome
    cantTell: URIRef  # it is unclear if the subject passed or failed the test
    failed: URIRef  # the subject failed the test
    Fail: URIRef  # the class of outcomes to denote failing a test
    inapplicable: URIRef  # the test is not applicable to the subject
    info: URIRef  # additional warnings or error messages in a human-readable form
    mainAssertor: URIRef  # assertor that is primarily responsible for performing the test
    manual: URIRef  # where the test was carried out by human evaluators
    mode: URIRef  # mode in which the test was performed
    NotApplicable: URIRef  # the class of outcomes to denote the test is not applicable
    NotTested: URIRef  # the class of outcomes to denote the test has not been carried out
    outcome: URIRef  # outcome of performing the test
    OutcomeValue: URIRef  # a discrete value that describes a resulting condition from carrying out the test
    passed: URIRef  # the subject passed the test
    Pass: URIRef  # the class of outcomes to denote passing a test
    pointer: URIRef  # location within a test subject that are most relevant to a test result
    result: URIRef  # result of an assertion
    semiAuto: URIRef  # where the test was partially carried out by software tools, but where human input or judgment was still required to decide or help decide the outcome of the test
    Software: URIRef  # any piece of software such as an authoring tool, browser, or evaluation tool
    subject: URIRef  # test subject of an assertion
    TestCase: URIRef  # an atomic test, usually one that is a partial test for a requirement
    TestCriterion: URIRef  # a testable statement, usually one that can be passed or failed
    TestMode: URIRef  # describes how a test was carried out
    TestRequirement: URIRef  # a higher-level requirement that is tested by executing one or more sub-tests
    TestResult: URIRef  # the actual result of performing the test
    TestSubject: URIRef  # the class of things that have been tested against some test criterion
    test: URIRef  # test criterion of an assertion
    undisclosed: URIRef  # where the exact testing process is undisclosed
    unknownMode: URIRef  # where the testing process is unknown or undetermined
    untested: URIRef  # the test has not been carried out


class EarlReport:
    """
    This is a helper class for building an EARL report graph.
    """

    def __init__(
        self,
        asserter_uri: Optional[str] = None,
        asserter_homepage: Optional[str] = None,
        asserter_name: Optional[str] = None,
    ) -> None:
        self.graph = graph = Graph()
        graph.bind("foaf", FOAF)
        graph.bind("earl", EARL)
        graph.bind("doap", DOAP)
        graph.bind("dc", DC)

        self.asserter: Node
        asserter: Node
        if asserter_uri is not None or asserter_homepage is not None:
            self.asserter = asserter = URIRef(
                asserter_homepage if asserter_uri is None else asserter_uri
            )
            graph.add((asserter, RDF.type, FOAF.Person))
        else:
            self.asserter = asserter = BNode()
            graph.add((asserter, RDF.type, FOAF.Person))
        if asserter_name:
            graph.add((asserter, FOAF.name, Literal(asserter_name)))
        if asserter_homepage:
            graph.add((asserter, FOAF.homepage, URIRef(asserter_homepage)))

        self.project = project = URIRef("https://github.com/RDFLib/rdflib")

        graph.add((project, DOAP.homepage, project))
        graph.add((project, DOAP.name, Literal("RDFLib")))
        graph.add((project, RDF.type, DOAP.Project))
        graph.add((project, DOAP["programming-language"], Literal("Python")))
        graph.add(
            (
                project,
                DOAP.description,
                Literal(
                    (
                        "RDFLib is a Python library for working with RDF, "
                        "a simple yet powerful language for representing information."
                    ),
                    lang="en",
                ),
            )
        )

        self.now = Literal(datetime.now())

    def add_test_outcome(
        self, test_id: URIRef, outcome: URIRef, info: Optional[Literal] = None
    ) -> Tuple[Node, Node]:
        graph = self.graph
        assertion = BNode()
        graph.add((assertion, RDF.type, EARL.Assertion))
        graph.add((assertion, EARL.test, test_id))
        graph.add((assertion, EARL.subject, self.project))
        graph.add((assertion, EARL.mode, EARL.automatic))
        if self.asserter:
            graph.add((assertion, EARL.assertedBy, self.asserter))

        result = BNode()
        graph.add((assertion, EARL.result, result))
        graph.add((result, RDF.type, EARL.TestResult))
        graph.add((result, DC.date, self.now))
        graph.add((result, EARL.outcome, outcome))
        if info:
            graph.add((result, EARL.info, info))

        return graph, result


def pytest_addoption(parser):
    group = parser.getgroup("terminal reporting")
    group.addoption(
        "--earl-report",
        action="store",
        dest="earl_path",
        metavar="path",
        default=None,
        help="create EARL report file at given path.",
    )

    group.addoption(
        "--earl-asserter-uri",
        action="store",
        dest="earl_asserter_uri",
        metavar="uri",
        default=None,
        help="Set the EARL asserter URI, defaults to the asserter homepage if not set.",
    )

    group.addoption(
        "--earl-asserter-homepage",
        action="store",
        dest="earl_asserter_homepage",
        metavar="URL",
        default=None,
        help="Set the EARL asserter homepage.",
    )

    group.addoption(
        "--earl-asserter-name",
        action="store",
        dest="earl_asserter_name",
        metavar="name",
        default=None,
        help="Set the EARL asserter name.",
    )


def pytest_configure(config):
    earl_path = config.option.earl_path
    if earl_path:
        config._earl = EarlReporter(
            Path(earl_path),
            EarlReport(
                asserter_uri=config.option.earl_asserter_uri,
                asserter_name=config.option.earl_asserter_name,
                asserter_homepage=config.option.earl_asserter_homepage,
            ),
        )
        config.pluginmanager.register(config._earl)


def pytest_unconfigure(config):
    earl = getattr(config, "_excel", None)
    if earl:
        del config._earl
        config.pluginmanager.unregister(earl)


# https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_runtest_protocol


class TestResult(enum.Enum):
    PASS = enum.auto()
    FAIL = enum.auto()
    ERROR = enum.auto()
    SKIP = enum.auto()


class TestReportHelper:
    @classmethod
    def get_rdf_test_uri(cls, report: "TestReport") -> Optional[URIRef]:
        return next(
            (
                cast(URIRef, item[1])
                for item in report.user_properties
                if item[0] == RDFT.Test
            ),
            None,
        )


class EarlReporter:
    """
    This class is a pytest plugin that will write a EARL report with results for
    every pytest which has a rdf_test_uri parameter that is a string or an
    URIRef.
    """

    def __init__(self, output_path: Path, report: Optional[EarlReport] = None) -> None:
        self.report = report if report is not None else EarlReport()
        self.output_path = output_path

    @pytest.hookimpl(hookwrapper=True)
    def pytest_runtest_makereport(
        self, item: Item, call: "CallInfo[None]"
    ) -> Generator[None, "_Result", None]:
        result = yield

        report: "TestReport" = result.get_result()

        if not hasattr(item, "callspec"):
            return
        callspec: "CallSpec2" = getattr(item, "callspec")
        rdf_test_uri = callspec.params.get("rdf_test_uri")
        if rdf_test_uri is None:
            return
        if not isinstance(rdf_test_uri, URIRef) and not isinstance(rdf_test_uri, str):
            logging.warning("rdf_test_uri parameter is not a URIRef or a str")
            return
        if not isinstance(rdf_test_uri, URIRef):
            rdf_test_uri = URIRef(rdf_test_uri)

        report.user_properties.append((RDFT.Test, rdf_test_uri))

    def append_result(self, report: "TestReport", test_result: TestResult) -> None:
        rdf_test_uri = TestReportHelper.get_rdf_test_uri(report)
        if rdf_test_uri is None:
            # No RDF test
            return
        if test_result is TestResult.PASS:
            self.report.add_test_outcome(rdf_test_uri, EARL.passed)
        elif test_result is TestResult.FAIL:
            self.report.add_test_outcome(rdf_test_uri, EARL.failed)
        elif (test_result) is TestResult.SKIP:
            self.report.add_test_outcome(rdf_test_uri, EARL.untested)
        else:
            self.report.add_test_outcome(rdf_test_uri, EARL.cantTell)

    def pytest_runtest_logreport(self, report: "TestReport") -> None:
        if report.passed:
            if report.when == "call":  # ignore setup/teardown
                self.append_result(report, TestResult.PASS)
        elif report.failed:
            if report.when == "call":  # ignore setup/teardown
                self.append_result(report, TestResult.FAIL)
            else:
                self.append_result(report, TestResult.ERROR)
        elif report.skipped:
            self.append_result(report, TestResult.SKIP)

    def pytest_sessionfinish(self, session: "Session"):
        self.report.graph.serialize(format="turtle", destination=self.output_path)