1#!/usr/bin/env perl 2# 3 4use warnings; 5use strict; 6use autodie; 7 8use File::Basename; 9use File::Fetch; 10use Getopt::Long; 11use Pod::Usage; 12 13use FindBin; 14use lib "$FindBin::Bin/extlib/lib/perl5"; 15 16use URI; 17 18my %config = ( 19 asn_sources => [ 20 'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest', 21 'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest', 22 'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest', 23 'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest', 24 'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest' 25 ], 26 bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz'] 27); 28 29my $download_asn = 0; 30my $download_bgp = 0; 31my $download_target = "./"; 32my $help = 0; 33my $man = 0; 34my $v4 = 1; 35my $v6 = 1; 36my $parse = 1; 37my $v4_zone = "asn.rspamd.com"; 38my $v6_zone = "asn6.rspamd.com"; 39my $v4_file = "asn.zone"; 40my $v6_file = "asn6.zone"; 41my $ns_servers = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ]; 42my $unknown_placeholder = "--"; 43 44GetOptions( 45 "download-asn" => \$download_asn, 46 "download-bgp" => \$download_bgp, 47 "4!" => \$v4, 48 "6!" => \$v6, 49 "parse!" => \$parse, 50 "target=s" => \$download_target, 51 "zone-v4=s" => \$v4_zone, 52 "zone-v6=s" => \$v6_zone, 53 "file-v4=s" => \$v4_file, 54 "file-v6=s" => \$v6_file, 55 "ns-server=s@" => \$ns_servers, 56 "help|?" => \$help, 57 "man" => \$man, 58 "unknown-placeholder" => \$unknown_placeholder, 59) or 60 pod2usage(2); 61 62pod2usage(1) if $help; 63pod2usage(-exitval => 0, -verbose => 2) if $man; 64 65if ($download_asn) { 66 foreach my $u (@{ $config{'asn_sources'} }) { 67 download_file($u); 68 } 69} 70 71if ($download_bgp) { 72 foreach my $u (@{ $config{'bgp_sources'} }) { 73 download_file($u); 74 } 75} 76 77if (!$parse) { 78 exit 0; 79} 80 81# Prefix to ASN map 82my $networks = { 4 => {}, 6 => {} }; 83 84foreach my $u (@{ $config{'bgp_sources'} }) { 85 my $parsed = URI->new($u); 86 my $fname = $download_target . '/' . basename($parsed->path); 87 88 use constant { 89 F_MARKER => 0, 90 F_TIMESTAMP => 1, 91 F_PEER_IP => 3, 92 F_PEER_AS => 4, 93 F_PREFIX => 5, 94 F_AS_PATH => 6, 95 F_ORIGIN => 7, 96 }; 97 98 open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!"; 99 100 while (<$bgpd>) { 101 chomp; 102 my @e = split /\|/; 103 if ($e[F_MARKER] ne 'TABLE_DUMP2') { 104 warn "bad line: $_\n"; 105 next; 106 } 107 108 my $origin_as; 109 my $prefix = $e[F_PREFIX]; 110 my $ip_ver = 6; 111 112 if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) { 113 $ip_ver = 4; 114 } 115 116 if ($e[F_AS_PATH]) { 117 118 # not empty AS_PATH 119 my @as_path = split /\s/, $e[F_AS_PATH]; 120 $origin_as = pop @as_path; 121 122 if (substr($origin_as, 0, 1) eq '{') { 123 124 # route is aggregated 125 if ($origin_as =~ /^{(\d+)}$/) { 126 127 # single AS aggregated, just remove { } around 128 $origin_as = $1; 129 } else { 130 131 # use previous AS from AS_PATH 132 $origin_as = pop @as_path; 133 } 134 } 135 136 # strip bogus AS 137 while (is_bougus_asn($origin_as)) { 138 $origin_as = pop @as_path; 139 last if scalar @as_path == 0; 140 } 141 } 142 143 # empty AS_PATH or all AS_PATH elements was stripped as bogus - use 144 # PEER_AS as origin AS 145 $origin_as //= $e[F_PEER_AS]; 146 147 $networks->{$ip_ver}{$prefix} = int($origin_as); 148 } 149} 150 151# Remove default routes 152delete $networks->{4}{'0.0.0.0/0'}; 153delete $networks->{6}{'::/0'}; 154 155# Now roughly detect countries 156my $as_info = {}; 157 158# RIR statistics exchange format 159# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format 160# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf 161# first 7 fields for this two formats are same 162use constant { 163 F_REGISTRY => 0, # {afrinic,apnic,arin,iana,lacnic,ripencc} 164 F_CC => 1, # ISO 3166 2-letter contry code 165 F_TYPE => 2, # {asn,ipv4,ipv6} 166 F_START => 3, 167 F_VALUE => 4, 168 F_DATE => 5, 169 F_STATUS => 6, 170}; 171 172foreach my $u (@{ $config{'asn_sources'} }) { 173 my $parsed = URI->new($u); 174 my $fname = $download_target . '/' . basename($parsed->path); 175 open(my $fh, "<", $fname) or die "Cannot open $fname: $!"; 176 177 while (<$fh>) { 178 next if /^\#/; 179 chomp; 180 my @elts = split /\|/; 181 182 if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') { 183 my $as_start = int($elts[F_START]); 184 my $as_end = $as_start + int($elts[F_VALUE]) - 1; 185 186 for my $as ($as_start .. $as_end) { 187 $as_info->{$as}{'country'} = $elts[F_CC]; 188 $as_info->{$as}{'rir'} = $elts[F_REGISTRY]; 189 } 190 } 191 } 192} 193 194# Write zone files 195my $ns_list = join ' ', @{$ns_servers}; 196my $zone_header = << "EOH"; 197\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300 198\$NS 43200 $ns_list 199EOH 200 201if ($v4) { 202 # create temp file in the same dir so we can be sure that mv is atomic 203 my $out_dir = dirname($v4_file); 204 my $out_file = basename($v4_file); 205 my $temp_file = "$out_dir/.$out_file.tmp"; 206 open my $v4_fh, '>', $temp_file; 207 print $v4_fh $zone_header; 208 209 while (my ($net, $asn) = each %{ $networks->{4} }) { 210 my $country = $as_info->{$asn}{'country'} || $unknown_placeholder; 211 my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder; 212 213 # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8 214 printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir; 215 } 216 217 close $v4_fh; 218 rename $temp_file, $v4_file; 219} 220 221if ($v6) { 222 my $out_dir = dirname($v6_file); 223 my $out_file = basename($v6_file); 224 my $temp_file = "$out_dir/.$out_file.tmp"; 225 open my $v6_fh, '>', $temp_file; 226 print $v6_fh $zone_header; 227 228 while (my ($net, $asn) = each %{ $networks->{6} }) { 229 my $country = $as_info->{$asn}{'country'} || $unknown_placeholder; 230 my $rir = $as_info->{$asn}{'rir'} || $unknown_placeholder; 231 232 # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111 233 printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir; 234 } 235 236 close $v6_fh; 237 rename $temp_file, $v6_file; 238} 239 240exit 0; 241 242######################################################################## 243 244sub download_file { 245 my ($url) = @_; 246 247 local $File::Fetch::WARN = 0; 248 local $File::Fetch::TIMEOUT = 180; # connectivity to ftp.lacnic.net is bad 249 250 my $ff = File::Fetch->new(uri => $url); 251 my $where = $ff->fetch(to => $download_target) or 252 die "$url: ", $ff->error; 253 254 return $where; 255} 256 257# Returns true if AS number is bogus 258# e. g. a private AS. 259# List of allocated and reserved AS: 260# https://www.iana.org/assignments/as-numbers/as-numbers.txt 261sub is_bougus_asn { 262 my $as = shift; 263 264 # 64496-64511 Reserved for use in documentation and sample code 265 # 64512-65534 Designated for private use 266 # 65535 Reserved 267 # 65536-65551 Reserved for use in documentation and sample code 268 # 65552-131071 Reserved 269 return 1 if $as >= 64496 && $as <= 131071; 270 271 # Reserved (RFC6996, RFC7300, RFC7607) 272 return 1 if $as == 0 || $as >= 4200000000; 273 274 return 0; 275} 276 277__END__ 278 279=head1 NAME 280 281asn.pl - download and parse ASN data for Rspamd 282 283=head1 SYNOPSIS 284 285asn.pl [options] 286 287 Options: 288 --download-asn Download ASN data from RIRs 289 --download-bgp Download BGP full view dump from RIPE RIS 290 --target Where to download files (default: current dir) 291 --zone-v4 IPv4 zone (default: asn.rspamd.com) 292 --zone-v6 IPv6 zone (default: asn6.rspamd.com) 293 --file-v4 IPv4 zone file (default: ./asn.zone) 294 --file-v6 IPv6 zone (default: ./asn6.zone) 295 --unknown-placeholder Placeholder for unknown elements (default: --) 296 --help Brief help message 297 --man Full documentation 298 299=head1 OPTIONS 300 301=over 8 302 303=item B<--download-asn> 304 305Download ASN data from RIR. 306 307=item B<--download-bgp> 308 309Download GeoIP data from Ripe 310 311=item B<--target> 312 313Specifies where to download files. 314 315=item B<--help> 316 317Print a brief help message and exits. 318 319=item B<--man> 320 321Prints the manual page and exits. 322 323=back 324 325=head1 DESCRIPTION 326 327B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone. 328 329=cut 330 331# vim: et:ts=4:sw=4