1#!/usr/bin/env perl
2#
3# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
4#
5# This Source Code Form is subject to the terms of the Mozilla Public
6# License, v. 2.0. If a copy of the MPL was not distributed with this
7# file, you can obtain one at https://mozilla.org/MPL/2.0/.
8#
9# See the COPYRIGHT file distributed with this work for additional
10# information regarding copyright ownership.
11
12use strict;
13use warnings;
14use autodie;
15use utf8;
16
17use Carp qw( croak );
18use Cwd qw( abs_path );
19use File::Basename qw( dirname );
20use File::Slurper qw( read_binary write_binary );
21use Cpanel::JSON::XS qw( decode_json );
22use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 );
23use MaxMind::DB::Writer::Serializer 0.100004;
24use MaxMind::DB::Writer::Tree 0.100004;
25use MaxMind::DB::Writer::Util qw( key_for_data );
26use Net::Works::Network;
27use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
28
29my $Dir = dirname( abs_path($0) );
30
31sub main {
32    write_geoip2_dbs();
33}
34
35sub write_geoip2_dbs {
36    _write_geoip2_db( @{$_}[ 0, 1 ], 'Test' )
37        for (
38        ['GeoIP2-City'],
39        ['GeoIP2-Country'],
40        ['GeoIP2-Domain'],
41        ['GeoIP2-ISP'],
42        ['GeoLite2-ASN'],
43        );
44}
45
46sub _universal_map_key_type_callback {
47    my $map = {
48
49        # languages
50        de      => 'utf8_string',
51        en      => 'utf8_string',
52        es      => 'utf8_string',
53        fr      => 'utf8_string',
54        ja      => 'utf8_string',
55        'pt-BR' => 'utf8_string',
56        ru      => 'utf8_string',
57        'zh-CN' => 'utf8_string',
58
59        # production
60        accuracy_radius                => 'uint16',
61        autonomous_system_number       => 'uint32',
62        autonomous_system_organization => 'utf8_string',
63        average_income                 => 'uint32',
64        city                           => 'map',
65        code                           => 'utf8_string',
66        confidence                     => 'uint16',
67        connection_type                => 'utf8_string',
68        continent                      => 'map',
69        country                        => 'map',
70        domain                         => 'utf8_string',
71        geoname_id                     => 'uint32',
72        ipv4_24                        => 'uint32',
73        ipv4_32                        => 'uint32',
74        ipv6_32                        => 'uint32',
75        ipv6_48                        => 'uint32',
76        ipv6_64                        => 'uint32',
77        is_anonymous                   => 'boolean',
78        is_anonymous_proxy             => 'boolean',
79        is_anonymous_vpn               => 'boolean',
80        is_hosting_provider            => 'boolean',
81        is_in_european_union           => 'boolean',
82        is_legitimate_proxy            => 'boolean',
83        is_public_proxy                => 'boolean',
84        is_satellite_provider          => 'boolean',
85        is_tor_exit_node               => 'boolean',
86        iso_code                       => 'utf8_string',
87        isp                            => 'utf8_string',
88        latitude                       => 'double',
89        location                       => 'map',
90        longitude                      => 'double',
91        metro_code                     => 'uint16',
92        names                          => 'map',
93        organization                   => 'utf8_string',
94        population_density             => 'uint32',
95        postal                         => 'map',
96        registered_country             => 'map',
97        represented_country            => 'map',
98        subdivisions                   => [ 'array', 'map' ],
99        time_zone                      => 'utf8_string',
100        traits                         => 'map',
101        traits                         => 'map',
102        type                           => 'utf8_string',
103        user_type                      => 'utf8_string',
104
105        # for testing only
106        foo       => 'utf8_string',
107        bar       => 'utf8_string',
108        buzz      => 'utf8_string',
109        our_value => 'utf8_string',
110    };
111
112    my $callback = sub {
113        my $key = shift;
114
115        return $map->{$key} || die <<"ERROR";
116Unknown tree key '$key'.
117
118The universal_map_key_type_callback doesn't know what type to use for the passed
119key.  If you are adding a new key that will be used in a frozen tree / mmdb then
120you should update the mapping in both our internal code and here.
121ERROR
122    };
123
124    return $callback;
125}
126
127sub _write_geoip2_db {
128    my $type                            = shift;
129    my $populate_all_networks_with_data = shift;
130    my $description                     = shift;
131
132    my $writer = MaxMind::DB::Writer::Tree->new(
133        ip_version    => 6,
134        record_size   => 28,
135        ip_version    => 6,
136        database_type => $type,
137        languages     => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
138        description   => {
139            en => ( $type =~ s/-/ /gr )
140                . " $description Database (fake GeoIP2 data, for example purposes only)",
141            $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
142        },
143        alias_ipv6_to_ipv4    => 1,
144        map_key_type_callback => _universal_map_key_type_callback(),
145        remove_reserved_networks => 0,
146    );
147
148    _populate_all_networks( $writer, $populate_all_networks_with_data )
149        if $populate_all_networks_with_data;
150
151    my $value = shift;
152    my $nodes
153        = decode_json( read_binary("$Dir/$type.json") );
154
155    for my $node (@$nodes) {
156        for my $network ( keys %$node ) {
157            $writer->insert_network(
158                Net::Works::Network->new_from_string( string => $network ),
159                $node->{$network}
160            );
161        }
162    }
163
164    open my $output_fh, '>', "$Dir/$type.mmdb";
165    $writer->write_tree($output_fh);
166    close $output_fh;
167
168    return;
169}
170
171sub _populate_all_networks {
172    my $writer = shift;
173    my $data   = shift;
174
175    my $max_uint128 = uint128(0) - 1;
176    my @networks    = Net::Works::Network->range_as_subnets(
177        Net::Works::Address->new_from_integer(
178            integer => 0,
179            version => 6,
180        ),
181        Net::Works::Address->new_from_integer(
182            integer => $max_uint128,
183            version => 6,
184        ),
185    );
186
187    for my $network (@networks) {
188        $writer->insert_network( $network => $data );
189    }
190}
191
192main();
193