1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
use strict;
use warnings;
use autodie;
use v5.16;
use Getopt::Long;
use JSON::XS;
use Math::Int128 qw( uint128 );
use MaxMind::DB::Writer::Tree;
use Memory::Stats;
use Net::Works::Network;
use Number::Format qw( format_bytes );
my $ipv6_offset = uint128('0xA000_0000_0000_0000_0000_0000_0000_0000');
sub main {
my $inserts = 10_000;
my $source;
GetOptions(
'inserts:i' => \$inserts,
'source:s' => \$source,
);
my $stats = Memory::Stats->new();
$stats->start();
my $tree = MaxMind::DB::Writer::Tree->new(
ip_version => 6,
database_type => 'Test',
description => { en => 'Test' },
languages => ['en'],
map_key_type_callback => sub { 'utf8_string' },
record_size => 24,
alias_ipv6_to_ipv4 => 1,
);
if ($source) {
_insert_data_from_json( $tree, $inserts, $source );
}
else {
_insert_generated_data( $tree, $inserts );
}
my $used;
$stats->stop();
say 'Memory used before write: ',
format_bytes( $used = $stats->get_memory_usage() );
$stats->start();
open my $fh, '>', '/dev/null';
$tree->write_tree($fh);
close $fh;
$stats->stop();
say 'Memory used after write: ',
format_bytes( $used + $stats->get_memory_usage() );
}
sub _insert_data_from_json {
my $tree = shift;
my $inserts = shift;
my $source = shift;
open my $fh, '<', $source;
my $json = JSON::XS->new()->utf8();
while (<$fh>) {
last if $. > $inserts;
my ( $network, $record ) = @{ $json->decode($_) };
$tree->insert_network(
Net::Works::Network->new_from_string( string => $network ),
$record
);
}
}
sub _insert_generated_data {
my $tree = shift;
my $inserts = shift;
for my $int ( 1 .. $inserts ) {
my $ipv4 = Net::Works::Network->new_from_integer(
integer => $int,
prefix_length => 128,
version => 6,
);
my %records;
for my $i ( 1 .. 5 ) {
$records{ipv4}{ 'thing' . $i } = {
'ipv4_key1-' . $i => $int,
'ipv4_key2-' . $i => { nested => "Foo $int-$i ipv4" },
};
$records{ipv6}{ 'thing' . $i } = {
'ipv6_key1-' . $i => $int,
'ipv6_key2-' . $i => { nested => "Foo $int-$i ipv6" },
};
}
$tree->insert_network( $ipv4 => $records{ipv4} );
my $ipv6 = Net::Works::Network->new_from_integer(
integer => $int + $ipv6_offset,
mask_length => 128,
);
$tree->insert_network( $ipv6 => $records{ipv6} );
}
}
main();
|
