1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
from .base import Converter
from ...helpers import iterate_files
try:
# python 3.x: Import ABC from collections.abc
from collections.abc import MutableMapping
except ImportError:
# Python 2.x: Import ABC from collections
from collections import MutableMapping
from csv import writer as csv_writer, excel
class CsvConverter(Converter):
fields = [
'data',
'hostnames',
'ip',
'ip_str',
'ipv6',
'org',
'isp',
'location.country_code',
'location.city',
'location.country_name',
'location.latitude',
'location.longitude',
'os',
'asn',
'port',
'tags',
'timestamp',
'transport',
'product',
'version',
'vulns',
'ssl.cipher.version',
'ssl.cipher.bits',
'ssl.cipher.name',
'ssl.alpn',
'ssl.versions',
'ssl.cert.serial',
'ssl.cert.fingerprint.sha1',
'ssl.cert.fingerprint.sha256',
'html',
'title',
]
def process(self, files):
writer = csv_writer(self.fout, dialect=excel, lineterminator='\n')
# Write the header
writer.writerow(self.fields)
for banner in iterate_files(files):
# The "vulns" property can't be nicely flattened as-is so we turn
# it into a list before processing the banner.
if 'vulns' in banner:
banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list
try:
row = []
for field in self.fields:
value = self.banner_field(banner, field)
row.append(value)
writer.writerow(row)
except Exception:
pass
def banner_field(self, banner, flat_field):
# The provided field is a collapsed form of the actual field
fields = flat_field.split('.')
try:
current_obj = banner
for field in fields:
current_obj = current_obj[field]
# Convert a list into a concatenated string
if isinstance(current_obj, list):
current_obj = ','.join([str(i) for i in current_obj])
return current_obj
except Exception:
pass
return ''
def flatten(self, d, parent_key='', sep='.'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, MutableMapping):
items.extend(self.flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
|
