1#!/usr/bin/env perl
2#
3
4use warnings;
5use strict;
6use autodie;
7
8use File::Basename;
9use File::Fetch;
10use Getopt::Long;
11use Pod::Usage;
12
13use FindBin;
14use lib "$FindBin::Bin/extlib/lib/perl5";
15
16use URI;
17
18my %config = (
19  asn_sources => [
20    'ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest',
21    'ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest',
22    'http://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest',
23    'ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest',
24    'ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest'
25  ],
26  bgp_sources => ['http://data.ris.ripe.net/rrc00/latest-bview.gz']
27);
28
29my $download_asn        = 0;
30my $download_bgp        = 0;
31my $download_target     = "./";
32my $help                = 0;
33my $man                 = 0;
34my $v4                  = 1;
35my $v6                  = 1;
36my $parse               = 1;
37my $v4_zone             = "asn.rspamd.com";
38my $v6_zone             = "asn6.rspamd.com";
39my $v4_file             = "asn.zone";
40my $v6_file             = "asn6.zone";
41my $ns_servers          = [ "asn-ns.rspamd.com", "asn-ns2.rspamd.com" ];
42my $unknown_placeholder = "--";
43
44GetOptions(
45  "download-asn" => \$download_asn,
46  "download-bgp" => \$download_bgp,
47  "4!"           => \$v4,
48  "6!"           => \$v6,
49  "parse!"       => \$parse,
50  "target=s"     => \$download_target,
51  "zone-v4=s"    => \$v4_zone,
52  "zone-v6=s"    => \$v6_zone,
53  "file-v4=s"    => \$v4_file,
54  "file-v6=s"    => \$v6_file,
55  "ns-server=s@" => \$ns_servers,
56  "help|?"       => \$help,
57  "man"          => \$man,
58  "unknown-placeholder" => \$unknown_placeholder,
59) or
60  pod2usage(2);
61
62pod2usage(1) if $help;
63pod2usage(-exitval => 0, -verbose => 2) if $man;
64
65if ($download_asn) {
66    foreach my $u (@{ $config{'asn_sources'} }) {
67        download_file($u);
68    }
69}
70
71if ($download_bgp) {
72    foreach my $u (@{ $config{'bgp_sources'} }) {
73        download_file($u);
74    }
75}
76
77if (!$parse) {
78    exit 0;
79}
80
81# Prefix to ASN map
82my $networks = { 4 => {}, 6 => {} };
83
84foreach my $u (@{ $config{'bgp_sources'} }) {
85    my $parsed = URI->new($u);
86    my $fname  = $download_target . '/' . basename($parsed->path);
87
88    use constant {
89      F_MARKER    => 0,
90      F_TIMESTAMP => 1,
91      F_PEER_IP   => 3,
92      F_PEER_AS   => 4,
93      F_PREFIX    => 5,
94      F_AS_PATH   => 6,
95      F_ORIGIN    => 7,
96    };
97
98    open(my $bgpd, '-|', "bgpdump -v -M $fname") or die "can't start bgpdump: $!";
99
100    while (<$bgpd>) {
101        chomp;
102        my @e = split /\|/;
103        if ($e[F_MARKER] ne 'TABLE_DUMP2') {
104            warn "bad line: $_\n";
105            next;
106        }
107
108        my $origin_as;
109        my $prefix = $e[F_PREFIX];
110        my $ip_ver = 6;
111
112        if ($prefix =~ /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{1,2}$/) {
113            $ip_ver = 4;
114        }
115
116        if ($e[F_AS_PATH]) {
117
118            # not empty AS_PATH
119            my @as_path = split /\s/, $e[F_AS_PATH];
120            $origin_as = pop @as_path;
121
122            if (substr($origin_as, 0, 1) eq '{') {
123
124                # route is aggregated
125                if ($origin_as =~ /^{(\d+)}$/) {
126
127                    # single AS aggregated, just remove { } around
128                    $origin_as = $1;
129                } else {
130
131                    # use previous AS from AS_PATH
132                    $origin_as = pop @as_path;
133                }
134            }
135
136            # strip bogus AS
137            while (is_bougus_asn($origin_as)) {
138                $origin_as = pop @as_path;
139                last if scalar @as_path == 0;
140            }
141        }
142
143        # empty AS_PATH or all AS_PATH elements was stripped as bogus - use
144        # PEER_AS as origin AS
145        $origin_as //= $e[F_PEER_AS];
146
147        $networks->{$ip_ver}{$prefix} = int($origin_as);
148    }
149}
150
151# Remove default routes
152delete $networks->{4}{'0.0.0.0/0'};
153delete $networks->{6}{'::/0'};
154
155# Now roughly detect countries
156my $as_info = {};
157
158# RIR statistics exchange format
159# https://www.apnic.net/publications/media-library/documents/resource-guidelines/rir-statistics-exchange-format
160# https://www.arin.net/knowledge/statistics/nro_extended_stats_format.pdf
161# first 7 fields for this two formats are same
162use constant {
163  F_REGISTRY => 0,  # {afrinic,apnic,arin,iana,lacnic,ripencc}
164  F_CC       => 1,  # ISO 3166 2-letter contry code
165  F_TYPE     => 2,  # {asn,ipv4,ipv6}
166  F_START    => 3,
167  F_VALUE    => 4,
168  F_DATE     => 5,
169  F_STATUS   => 6,
170};
171
172foreach my $u (@{ $config{'asn_sources'} }) {
173    my $parsed = URI->new($u);
174    my $fname  = $download_target . '/' . basename($parsed->path);
175    open(my $fh, "<", $fname) or die "Cannot open $fname: $!";
176
177    while (<$fh>) {
178        next if /^\#/;
179        chomp;
180        my @elts = split /\|/;
181
182        if ($elts[F_TYPE] eq 'asn' && $elts[F_START] ne '*') {
183            my $as_start = int($elts[F_START]);
184            my $as_end   = $as_start + int($elts[F_VALUE]) - 1;
185
186            for my $as ($as_start .. $as_end) {
187                $as_info->{$as}{'country'} = $elts[F_CC];
188                $as_info->{$as}{'rir'}     = $elts[F_REGISTRY];
189            }
190        }
191    }
192}
193
194# Write zone files
195my $ns_list     = join ' ', @{$ns_servers};
196my $zone_header = << "EOH";
197\$SOA 43200 $ns_servers->[0] support.rspamd.com 0 600 300 86400 300
198\$NS  43200 $ns_list
199EOH
200
201if ($v4) {
202    # create temp file in the same dir so we can be sure that mv is atomic
203    my $out_dir = dirname($v4_file);
204    my $out_file = basename($v4_file);
205    my $temp_file = "$out_dir/.$out_file.tmp";
206    open my $v4_fh, '>', $temp_file;
207    print $v4_fh $zone_header;
208
209    while (my ($net, $asn) = each %{ $networks->{4} }) {
210        my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
211        my $rir     = $as_info->{$asn}{'rir'}     || $unknown_placeholder;
212
213        # "15169|8.8.8.0/24|US|arin|" for 8.8.8.8
214        printf $v4_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
215    }
216
217    close $v4_fh;
218    rename $temp_file, $v4_file;
219}
220
221if ($v6) {
222    my $out_dir = dirname($v6_file);
223    my $out_file = basename($v6_file);
224    my $temp_file = "$out_dir/.$out_file.tmp";
225    open my $v6_fh, '>', $temp_file;
226    print $v6_fh $zone_header;
227
228    while (my ($net, $asn) = each %{ $networks->{6} }) {
229        my $country = $as_info->{$asn}{'country'} || $unknown_placeholder;
230        my $rir     = $as_info->{$asn}{'rir'}     || $unknown_placeholder;
231
232        # "2606:4700:4700::/48 13335|2606:4700:4700::/48|US|arin|" for 2606:4700:4700::1111
233        printf $v6_fh "%s %s|%s|%s|%s|\n", $net, $asn, $net, $country, $rir;
234    }
235
236    close $v6_fh;
237    rename $temp_file, $v6_file;
238}
239
240exit 0;
241
242########################################################################
243
244sub download_file {
245    my ($url) = @_;
246
247    local $File::Fetch::WARN    = 0;
248    local $File::Fetch::TIMEOUT = 180;  # connectivity to ftp.lacnic.net is bad
249
250    my $ff    = File::Fetch->new(uri => $url);
251    my $where = $ff->fetch(to => $download_target) or
252      die "$url: ", $ff->error;
253
254    return $where;
255}
256
257# Returns true if AS number is bogus
258# e. g. a private AS.
259# List of allocated and reserved AS:
260# https://www.iana.org/assignments/as-numbers/as-numbers.txt
261sub is_bougus_asn {
262    my $as = shift;
263
264    # 64496-64511  Reserved for use in documentation and sample code
265    # 64512-65534  Designated for private use
266    # 65535        Reserved
267    # 65536-65551  Reserved for use in documentation and sample code
268    # 65552-131071 Reserved
269    return 1 if $as >= 64496 && $as <= 131071;
270
271    # Reserved (RFC6996, RFC7300, RFC7607)
272    return 1 if $as == 0 || $as >= 4200000000;
273
274    return 0;
275}
276
277__END__
278
279=head1 NAME
280
281asn.pl - download and parse ASN data for Rspamd
282
283=head1 SYNOPSIS
284
285asn.pl [options]
286
287 Options:
288   --download-asn         Download ASN data from RIRs
289   --download-bgp         Download BGP full view dump from RIPE RIS
290   --target               Where to download files (default: current dir)
291   --zone-v4              IPv4 zone (default: asn.rspamd.com)
292   --zone-v6              IPv6 zone (default: asn6.rspamd.com)
293   --file-v4              IPv4 zone file (default: ./asn.zone)
294   --file-v6              IPv6 zone (default: ./asn6.zone)
295   --unknown-placeholder  Placeholder for unknown elements (default: --)
296   --help                 Brief help message
297   --man                  Full documentation
298
299=head1 OPTIONS
300
301=over 8
302
303=item B<--download-asn>
304
305Download ASN data from RIR.
306
307=item B<--download-bgp>
308
309Download GeoIP data from Ripe
310
311=item B<--target>
312
313Specifies where to download files.
314
315=item B<--help>
316
317Print a brief help message and exits.
318
319=item B<--man>
320
321Prints the manual page and exits.
322
323=back
324
325=head1 DESCRIPTION
326
327B<asn.pl> is intended to download ASN data and GeoIP data and create a rbldnsd zone.
328
329=cut
330
331# vim: et:ts=4:sw=4