1#!/usr/bin/env perl 2# 3# Copyright (C) Internet Systems Consortium, Inc. ("ISC") 4# 5# This Source Code Form is subject to the terms of the Mozilla Public 6# License, v. 2.0. If a copy of the MPL was not distributed with this 7# file, you can obtain one at https://mozilla.org/MPL/2.0/. 8# 9# See the COPYRIGHT file distributed with this work for additional 10# information regarding copyright ownership. 11 12use strict; 13use warnings; 14use autodie; 15use utf8; 16 17use Carp qw( croak ); 18use Cwd qw( abs_path ); 19use File::Basename qw( dirname ); 20use File::Slurper qw( read_binary write_binary ); 21use Cpanel::JSON::XS qw( decode_json ); 22use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 ); 23use MaxMind::DB::Writer::Serializer 0.100004; 24use MaxMind::DB::Writer::Tree 0.100004; 25use MaxMind::DB::Writer::Util qw( key_for_data ); 26use Net::Works::Network; 27use Test::MaxMind::DB::Common::Util qw( standard_test_metadata ); 28 29my $Dir = dirname( abs_path($0) ); 30 31sub main { 32 write_geoip2_dbs(); 33} 34 35sub write_geoip2_dbs { 36 _write_geoip2_db( @{$_}[ 0, 1 ], 'Test' ) 37 for ( 38 ['GeoIP2-City'], 39 ['GeoIP2-Country'], 40 ['GeoIP2-Domain'], 41 ['GeoIP2-ISP'], 42 ['GeoLite2-ASN'], 43 ); 44} 45 46sub _universal_map_key_type_callback { 47 my $map = { 48 49 # languages 50 de => 'utf8_string', 51 en => 'utf8_string', 52 es => 'utf8_string', 53 fr => 'utf8_string', 54 ja => 'utf8_string', 55 'pt-BR' => 'utf8_string', 56 ru => 'utf8_string', 57 'zh-CN' => 'utf8_string', 58 59 # production 60 accuracy_radius => 'uint16', 61 autonomous_system_number => 'uint32', 62 autonomous_system_organization => 'utf8_string', 63 average_income => 'uint32', 64 city => 'map', 65 code => 'utf8_string', 66 confidence => 'uint16', 67 connection_type => 'utf8_string', 68 continent => 'map', 69 country => 'map', 70 domain => 'utf8_string', 71 geoname_id => 'uint32', 72 ipv4_24 => 'uint32', 73 ipv4_32 => 'uint32', 74 ipv6_32 => 'uint32', 75 ipv6_48 => 'uint32', 76 ipv6_64 => 'uint32', 77 is_anonymous => 'boolean', 78 is_anonymous_proxy => 'boolean', 79 is_anonymous_vpn => 'boolean', 80 is_hosting_provider => 'boolean', 81 is_in_european_union => 'boolean', 82 is_legitimate_proxy => 'boolean', 83 is_public_proxy => 'boolean', 84 is_satellite_provider => 'boolean', 85 is_tor_exit_node => 'boolean', 86 iso_code => 'utf8_string', 87 isp => 'utf8_string', 88 latitude => 'double', 89 location => 'map', 90 longitude => 'double', 91 metro_code => 'uint16', 92 names => 'map', 93 organization => 'utf8_string', 94 population_density => 'uint32', 95 postal => 'map', 96 registered_country => 'map', 97 represented_country => 'map', 98 subdivisions => [ 'array', 'map' ], 99 time_zone => 'utf8_string', 100 traits => 'map', 101 traits => 'map', 102 type => 'utf8_string', 103 user_type => 'utf8_string', 104 105 # for testing only 106 foo => 'utf8_string', 107 bar => 'utf8_string', 108 buzz => 'utf8_string', 109 our_value => 'utf8_string', 110 }; 111 112 my $callback = sub { 113 my $key = shift; 114 115 return $map->{$key} || die <<"ERROR"; 116Unknown tree key '$key'. 117 118The universal_map_key_type_callback doesn't know what type to use for the passed 119key. If you are adding a new key that will be used in a frozen tree / mmdb then 120you should update the mapping in both our internal code and here. 121ERROR 122 }; 123 124 return $callback; 125} 126 127sub _write_geoip2_db { 128 my $type = shift; 129 my $populate_all_networks_with_data = shift; 130 my $description = shift; 131 132 my $writer = MaxMind::DB::Writer::Tree->new( 133 ip_version => 6, 134 record_size => 28, 135 ip_version => 6, 136 database_type => $type, 137 languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ], 138 description => { 139 en => ( $type =~ s/-/ /gr ) 140 . " $description Database (fake GeoIP2 data, for example purposes only)", 141 $type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (), 142 }, 143 alias_ipv6_to_ipv4 => 1, 144 map_key_type_callback => _universal_map_key_type_callback(), 145 remove_reserved_networks => 0, 146 ); 147 148 _populate_all_networks( $writer, $populate_all_networks_with_data ) 149 if $populate_all_networks_with_data; 150 151 my $value = shift; 152 my $nodes 153 = decode_json( read_binary("$Dir/$type.json") ); 154 155 for my $node (@$nodes) { 156 for my $network ( keys %$node ) { 157 $writer->insert_network( 158 Net::Works::Network->new_from_string( string => $network ), 159 $node->{$network} 160 ); 161 } 162 } 163 164 open my $output_fh, '>', "$Dir/$type.mmdb"; 165 $writer->write_tree($output_fh); 166 close $output_fh; 167 168 return; 169} 170 171sub _populate_all_networks { 172 my $writer = shift; 173 my $data = shift; 174 175 my $max_uint128 = uint128(0) - 1; 176 my @networks = Net::Works::Network->range_as_subnets( 177 Net::Works::Address->new_from_integer( 178 integer => 0, 179 version => 6, 180 ), 181 Net::Works::Address->new_from_integer( 182 integer => $max_uint128, 183 version => 6, 184 ), 185 ); 186 187 for my $network (@networks) { 188 $writer->insert_network( $network => $data ); 189 } 190} 191 192main(); 193