File: sarif_test.py

package info (click to toggle)
cppcheck 2.19.0-3
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 26,688 kB
  • sloc: cpp: 272,455; python: 22,408; ansic: 8,088; sh: 1,059; makefile: 1,041; xml: 987; cs: 291
file content (622 lines) | stat: -rw-r--r-- 18,111 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
# python -m pytest sarif_test.py

import os
import json
import tempfile

import pytest

from testutils import cppcheck

__script_dir = os.path.dirname(os.path.abspath(__file__))

# Test code with various error types
TEST_CODE = """
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <vector>
#include <memory>

class TestClass {
public:
    TestClass() : value(0) {}
    ~TestClass() { delete ptr; }
    
    void setValue(int v) { value = v; }
    int getValue() const { return value; }
    
private:
    int value;
    int* ptr = nullptr;
};

void testSecurityViolations() {
    // Null pointer dereference
    int* ptr = nullptr;
    *ptr = 5;
    
    // Array bounds violation  
    int array[5];
    array[10] = 1;
    
    // Memory leak
    int* mem = (int*)malloc(sizeof(int) * 10);
    // forgot to free mem
    
    // Uninitialized variable
    int x;
    printf("%d", x);
    
    // Double free
    int* p = (int*)malloc(sizeof(int));
    free(p);
    free(p);
    
    // Buffer overflow with strcpy
    char buffer[10];
    char source[20] = "This is too long";
    strcpy(buffer, source);
    
    // Use after free
    int* freed = (int*)malloc(sizeof(int));
    free(freed);
    *freed = 42;
}

void testStyleAndPortabilityIssues() {
    // Redundant assignment
    int redundant = 5;
    redundant = redundant;
    
    // Unused variable
    int unused = 42;
    
    // Variable scope reduction
    int i;
    for (i = 0; i < 10; i++) {
        // i could be declared in for loop
    }
}

int main() {
    testSecurityViolations();
    testStyleAndPortabilityIssues();
    return 0;
}
"""

BASIC_TEST_CODE = """
int main() {
    int* p = nullptr;
    *p = 5; // null pointer dereference
    return 0;
}
"""


def create_test_file(tmp_path, filename="test.cpp", content=TEST_CODE):
    """Create a temporary test file with the given content."""
    filepath = tmp_path / filename
    filepath.write_text(content)
    return str(filepath)


def run_sarif_check(code, extra_args=None):
    """Run cppcheck with SARIF output on the given code."""
    with tempfile.TemporaryDirectory() as tmpdir:
        tmp_path = os.path.join(tmpdir, "test.cpp")
        with open(tmp_path, "w") as f:
            f.write(code)

        args = ["--output-format=sarif", "--enable=all", tmp_path]

        if extra_args:
            args.extend(extra_args)

        _, _, stderr = cppcheck(args)

        # SARIF output is in stderr
        try:
            sarif_data = json.loads(stderr)
            return sarif_data
        except json.JSONDecodeError as e:
            pytest.fail(f"Failed to parse SARIF JSON: {e}\nOutput: {stderr}")
            return None


def test_sarif_basic_structure():
    """Test that SARIF output has the correct basic structure."""
    sarif = run_sarif_check(BASIC_TEST_CODE)

    # Check required SARIF fields
    assert sarif["version"] == "2.1.0"
    assert "$schema" in sarif
    assert "sarif-schema" in sarif["$schema"]
    assert "runs" in sarif
    assert len(sarif["runs"]) == 1

    run = sarif["runs"][0]
    assert "tool" in run
    assert "results" in run

    tool = run["tool"]
    assert "driver" in tool

    driver = tool["driver"]
    assert driver["name"] == "Cppcheck"
    assert "rules" in driver
    assert "semanticVersion" in driver


def test_sarif_null_pointer():
    """Test SARIF output for null pointer dereference."""
    sarif = run_sarif_check(BASIC_TEST_CODE)

    run = sarif["runs"][0]
    results = run["results"]

    # Should have at least one result
    assert len(results) > 0

    # Find null pointer result
    null_pointer_results = [r for r in results if r["ruleId"] == "nullPointer"]
    assert len(null_pointer_results) > 0

    result = null_pointer_results[0]
    assert result["level"] == "error"
    assert "message" in result
    assert "text" in result["message"]
    assert (
        "null" in result["message"]["text"].lower()
        or "nullptr" in result["message"]["text"].lower()
    )

    # Check location information
    assert "locations" in result
    assert len(result["locations"]) > 0
    location = result["locations"][0]
    assert "physicalLocation" in location
    assert "artifactLocation" in location["physicalLocation"]
    assert "region" in location["physicalLocation"]

    region = location["physicalLocation"]["region"]
    assert "startLine" in region
    assert region["startLine"] > 0
    assert "startColumn" in region
    assert region["startColumn"] > 0


def test_sarif_security_rules():
    """Test that security-related rules have proper security properties."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    # Check for security-related rules
    security_rule_ids = [
        "nullPointer",
        "arrayIndexOutOfBounds",
        "memleak",
        "uninitvar",
        "doubleFree",
    ]

    for rule_id in security_rule_ids:
        matching_rules = [r for r in rules if r["id"] == rule_id]
        if matching_rules:
            rule = matching_rules[0]
            props = rule.get("properties", {})

            # Security rules should have security-severity
            if "tags" in props and "security" in props["tags"]:
                assert "security-severity" in props
                assert float(props["security-severity"]) > 0

            # Should have problem.severity
            assert "problem.severity" in props

            # Should have precision
            assert "precision" in props


def test_sarif_rule_descriptions():
    """Test that rule descriptions are properly formatted."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    assert len(rules) > 0

    for rule in rules:
        # Each rule should have required fields
        assert "id" in rule
        assert "name" in rule
        assert "shortDescription" in rule
        assert "fullDescription" in rule

        # Descriptions should be empty (allowing GitHub to use instance messages)
        assert rule["name"] == ""
        assert rule["shortDescription"]["text"] == ""
        assert rule["fullDescription"]["text"] == ""

        # Should have properties
        assert "properties" in rule


def test_sarif_cwe_tags():
    """Test that CWE tags are properly formatted."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    # Find rules with CWE tags
    rules_with_cwe = []
    for rule in rules:
        props = rule.get("properties", {})
        tags = props.get("tags", [])
        cwe_tags = [t for t in tags if t.startswith("external/cwe/cwe-")]
        if cwe_tags:
            rules_with_cwe.append((rule["id"], cwe_tags))

    # Should have at least some rules with CWE tags
    assert len(rules_with_cwe) > 0

    # Validate CWE tag format
    for _, cwe_tags in rules_with_cwe:
        for tag in cwe_tags:
            assert tag.startswith("external/cwe/cwe-")
            cwe_num = tag[17:]  # After 'external/cwe/cwe-'
            assert cwe_num.isdigit()
            assert int(cwe_num) > 0


def test_sarif_severity_levels():
    """Test that different severity levels are properly mapped."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    results = run["results"]

    # Collect severity levels
    levels = set()
    for result in results:
        levels.add(result["level"])

    # Should have at least error level
    assert "error" in levels

    # Valid SARIF levels
    valid_levels = {"error", "warning", "note", "none"}
    for level in levels:
        assert level in valid_levels


def test_sarif_instance_specific_messages():
    """Test that result messages contain instance-specific information."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    results = run["results"]

    # Check that messages are instance-specific
    found_specific_messages = False

    for result in results:
        message_text = result["message"]["text"]
        rule_id = result["ruleId"]

        # Skip system include warnings
        if rule_id == "missingIncludeSystem":
            continue

        # Messages should not be empty
        assert len(message_text) > 0

        # Check for specific variable names or values from our test code
        if rule_id == "nullPointer" and "ptr" in message_text:
            found_specific_messages = True
        elif rule_id == "arrayIndexOutOfBounds" and (
            "array" in message_text or "10" in message_text
        ):
            found_specific_messages = True
        elif rule_id == "uninitvar" and "x" in message_text:
            found_specific_messages = True
        elif rule_id == "memleak" and "mem" in message_text:
            found_specific_messages = True
        elif rule_id == "doubleFree" and "p" in message_text:
            found_specific_messages = True

    assert (
        found_specific_messages
    ), "Should find at least some instance-specific messages"


def test_sarif_location_info():
    """Test that location information is correct."""
    sarif = run_sarif_check(BASIC_TEST_CODE)

    run = sarif["runs"][0]
    results = run["results"]

    for result in results:
        assert "locations" in result
        locations = result["locations"]
        assert len(locations) > 0

        for location in locations:
            assert "physicalLocation" in location
            phys_loc = location["physicalLocation"]

            assert "artifactLocation" in phys_loc
            assert "uri" in phys_loc["artifactLocation"]

            assert "region" in phys_loc
            region = phys_loc["region"]

            # SARIF requires line and column numbers >= 1
            assert "startLine" in region
            assert region["startLine"] >= 1

            assert "startColumn" in region
            assert region["startColumn"] >= 1


def test_sarif_with_multiple_files(tmp_path):
    """Test SARIF output with multiple source files."""
    # Create two test files
    file1_content = """
    void test1() {
        int* p = nullptr;
        *p = 1;
    }
    """

    file2_content = """
    void test2() {
        int arr[5];
        arr[10] = 2;
    }
    """

    file1 = tmp_path / "file1.cpp"
    file2 = tmp_path / "file2.cpp"
    file1.write_text(file1_content)
    file2.write_text(file2_content)

    args = ["--output-format=sarif", "--enable=all", str(tmp_path)]

    _, _, stderr = cppcheck(args)
    sarif = json.loads(stderr)

    run = sarif["runs"][0]
    results = run["results"]

    # Should have results from both files
    files = set()
    for result in results:
        if "locations" in result:
            for location in result["locations"]:
                uri = location["physicalLocation"]["artifactLocation"]["uri"]
                files.add(os.path.basename(uri))

    assert "file1.cpp" in files or "file2.cpp" in files


def test_sarif_default_configuration():
    """Test that rules have default configuration with level."""
    sarif = run_sarif_check(BASIC_TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    for rule in rules:
        # Check for defaultConfiguration.level (#13885)
        assert "defaultConfiguration" in rule
        assert "level" in rule["defaultConfiguration"]

        level = rule["defaultConfiguration"]["level"]
        assert level in ["error", "warning", "note", "none"]


def test_sarif_rule_properties():
    """Test that rules have required properties."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    for rule in rules:
        assert "properties" in rule
        props = rule["properties"]

        # Required properties
        assert "precision" in props
        assert props["precision"] in ["very-high", "high", "medium", "low"]

        assert "problem.severity" in props
        assert props["problem.severity"] in [
            "error",
            "warning",
            "style",
            "performance",
            "portability",
            "information",
            "note",
        ]


def test_sarif_rule_coverage():
    """Test that a variety of rules are triggered by comprehensive test code."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    # Collect all rule IDs
    rule_ids = set(rule["id"] for rule in rules)

    # Should have at least 5 different rules triggered
    assert (
        len(rule_ids) >= 5
    ), f"Expected at least 5 rules, found {len(rule_ids)}: {rule_ids}"

    # Check for some specific expected rules from different categories
    expected_rules = [
        "nullPointer",  # Security
        "arrayIndexOutOfBounds",  # Security
        "memleak",  # Security
        "uninitvar",  # Security
        "unusedVariable",  # Style
        "redundantAssignment",  # Style
        "unusedFunction",  # Style (if enabled)
        "constParameter",  # Style/Performance
        "cstyleCast",  # Style
        "variableScope",  # Style
    ]

    found_expected_rules = sum(1 for rule in expected_rules if rule in rule_ids)

    # Should find at least 3 of our expected rules
    assert (
        found_expected_rules >= 3
    ), f"Expected at least 3 known rules, found {found_expected_rules}"


def test_sarif_generic_descriptions():
    """Test that ALL rule descriptions are empty for GitHub integration."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    assert len(rules) > 0, "Should have at least one rule"

    # Verify that ALL rule descriptions are empty so GitHub uses instance-specific messages
    for rule in rules:
        rule_id = rule["id"]

        # All rules must have these fields
        assert "name" in rule, f"Rule {rule_id} missing 'name'"
        assert "shortDescription" in rule, f"Rule {rule_id} missing 'shortDescription'"
        assert "fullDescription" in rule, f"Rule {rule_id} missing 'fullDescription'"

        # The key test: ALL descriptions should be empty
        assert (
            rule["name"] == ""
        ), f"Rule {rule_id} name should be empty, got: {rule['name']}"
        assert (
            rule["shortDescription"]["text"] == ""
        ), f"Rule {rule_id} shortDescription should be empty, got: {rule['shortDescription']['text']}"
        assert (
            rule["fullDescription"]["text"] == ""
        ), f"Rule {rule_id} fullDescription should be empty, got: {rule['fullDescription']['text']}"


def test_sarif_security_rules_classification():
    """Test that security classification is correctly based on CWE IDs."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]

    found_rule_with_cwe = False
    found_rule_without_cwe = False

    for rule in rules:
        rule_id = rule["id"]
        props = rule.get("properties", {})
        tags = props.get("tags", [])

        # Check if rule has CWE tag
        cwe_tags = [t for t in tags if t.startswith("external/cwe/")]
        has_cwe = len(cwe_tags) > 0

        if has_cwe:
            found_rule_with_cwe = True

            # Rules with CWE should have security-severity and security tag
            assert (
                "security-severity" in props
            ), f"Rule {rule_id} with CWE should have security-severity"
            assert (
                float(props["security-severity"]) > 0
            ), f"Rule {rule_id} security-severity should be positive"

            # Check for security tag
            assert (
                "security" in tags
            ), f"Rule {rule_id} with CWE should have 'security' tag"
        else:
            found_rule_without_cwe = True

            # Rules without CWE should NOT have security-severity or security tag
            assert (
                "security-severity" not in props
            ), f"Rule {rule_id} without CWE should not have security-severity"
            assert (
                "security" not in tags
            ), f"Rule {rule_id} without CWE should not have 'security' tag"

        # All rules should still have basic properties
        assert "precision" in props, f"Rule {rule_id} missing 'precision'"
        assert "problem.severity" in props, f"Rule {rule_id} missing 'problem.severity'"

    # Should find at least some rules in test data
    assert (
        found_rule_with_cwe or found_rule_without_cwe
    ), "Should find at least some rules with or without CWE"


def test_sarif_results_consistency():
    """Test consistency between rule definitions and results."""
    sarif = run_sarif_check(TEST_CODE)

    run = sarif["runs"][0]
    driver = run["tool"]["driver"]
    rules = driver["rules"]
    results = run["results"]

    # Collect rule IDs from both rules and results
    rule_ids_in_rules = set(rule["id"] for rule in rules)
    rule_ids_in_results = set(result["ruleId"] for result in results)

    # Every rule ID in results should have a corresponding rule definition
    for result_rule_id in rule_ids_in_results:
        assert (
            result_rule_id in rule_ids_in_rules
        ), f"Result references undefined rule: {result_rule_id}"

    # Verify severity level consistency
    severity_levels = set()
    for result in results:
        level = result["level"]
        severity_levels.add(level)

        # Valid SARIF levels
        assert level in [
            "error",
            "warning",
            "note",
            "none",
        ], f"Invalid severity level: {level}"

    # Should have at least error level
    assert "error" in severity_levels, "Should have at least one error"

    # Should have multiple severity levels for comprehensive test
    assert (
        len(severity_levels) >= 2
    ), f"Expected multiple severity levels, found: {severity_levels}"