1#!/usr/bin/env python3
2# Copyright (c) 2013-2020 The Bitcoin Core developers
3# Distributed under the MIT software license, see the accompanying
4# file COPYING or http://www.opensource.org/licenses/mit-license.php.
5#
6# Generate seeds.txt from Pieter's DNS seeder
7#
8
9import re
10import sys
11import dns.resolver
12import collections
13
14NSEEDS=512
15
16MAX_SEEDS_PER_ASN=2
17
18MIN_BLOCKS = 337600
19
20# These are hosts that have been observed to be behaving strangely (e.g.
21# aggressively connecting to every node).
22with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f:
23    SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()}
24
25
26PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$")
27PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
28PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")
29PATTERN_AGENT = re.compile(
30    r"^/Satoshi:("
31    r"0.14.(0|1|2|3|99)|"
32    r"0.15.(0|1|2|99)|"
33    r"0.16.(0|1|2|3|99)|"
34    r"0.17.(0|0.1|1|2|99)|"
35    r"0.18.(0|1|99)|"
36    r"0.19.(0|1|99)|"
37    r"0.20.(0|1|99)|"
38    r"0.21.99"
39    r")")
40
41def parseline(line):
42    sline = line.split()
43    if len(sline) < 11:
44       return None
45    m = PATTERN_IPV4.match(sline[0])
46    sortkey = None
47    ip = None
48    if m is None:
49        m = PATTERN_IPV6.match(sline[0])
50        if m is None:
51            m = PATTERN_ONION.match(sline[0])
52            if m is None:
53                return None
54            else:
55                net = 'onion'
56                ipstr = sortkey = m.group(1)
57                port = int(m.group(2))
58        else:
59            net = 'ipv6'
60            if m.group(1) in ['::']: # Not interested in localhost
61                return None
62            ipstr = m.group(1)
63            sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds
64            port = int(m.group(2))
65    else:
66        # Do IPv4 sanity check
67        ip = 0
68        for i in range(0,4):
69            if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255:
70                return None
71            ip = ip + (int(m.group(i+2)) << (8*(3-i)))
72        if ip == 0:
73            return None
74        net = 'ipv4'
75        sortkey = ip
76        ipstr = m.group(1)
77        port = int(m.group(6))
78    # Skip bad results.
79    if sline[1] == 0:
80        return None
81    # Extract uptime %.
82    uptime30 = float(sline[7][:-1])
83    # Extract Unix timestamp of last success.
84    lastsuccess = int(sline[2])
85    # Extract protocol version.
86    version = int(sline[10])
87    # Extract user agent.
88    agent = sline[11][1:-1]
89    # Extract service flags.
90    service = int(sline[9], 16)
91    # Extract blocks.
92    blocks = int(sline[8])
93    # Construct result.
94    return {
95        'net': net,
96        'ip': ipstr,
97        'port': port,
98        'ipnum': ip,
99        'uptime': uptime30,
100        'lastsuccess': lastsuccess,
101        'version': version,
102        'agent': agent,
103        'service': service,
104        'blocks': blocks,
105        'sortkey': sortkey,
106    }
107
108def dedup(ips):
109    '''deduplicate by address,port'''
110    d = {}
111    for ip in ips:
112        d[ip['ip'],ip['port']] = ip
113    return list(d.values())
114
115def filtermultiport(ips):
116    '''Filter out hosts with more nodes per IP'''
117    hist = collections.defaultdict(list)
118    for ip in ips:
119        hist[ip['sortkey']].append(ip)
120    return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
121
122def lookup_asn(net, ip):
123    '''
124    Look up the asn for an IP (4 or 6) address by querying cymru.com, or None
125    if it could not be found.
126    '''
127    try:
128        if net == 'ipv4':
129            ipaddr = ip
130            prefix = '.origin'
131        else:                  # http://www.team-cymru.com/IP-ASN-mapping.html
132            res = str()                         # 2001:4860:b002:23::68
133            for nb in ip.split(':')[:4]:  # pick the first 4 nibbles
134                for c in nb.zfill(4):           # right padded with '0'
135                    res += c + '.'              # 2001 4860 b002 0023
136            ipaddr = res.rstrip('.')            # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3
137            prefix = '.origin6'
138
139        asn = int([x.to_text() for x in dns.resolver.query('.'.join(
140                   reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com',
141                   'TXT').response.answer][0].split('\"')[1].split(' ')[0])
142        return asn
143    except Exception:
144        sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n')
145        return None
146
147# Based on Greg Maxwell's seed_filter.py
148def filterbyasn(ips, max_per_asn, max_per_net):
149    # Sift out ips by type
150    ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']]
151    ips_onion = [ip for ip in ips if ip['net'] == 'onion']
152
153    # Filter IPv46 by ASN, and limit to max_per_net per network
154    result = []
155    net_count = collections.defaultdict(int)
156    asn_count = collections.defaultdict(int)
157    for ip in ips_ipv46:
158        if net_count[ip['net']] == max_per_net:
159            continue
160        asn = lookup_asn(ip['net'], ip['ip'])
161        if asn is None or asn_count[asn] == max_per_asn:
162            continue
163        asn_count[asn] += 1
164        net_count[ip['net']] += 1
165        result.append(ip)
166
167    # Add back Onions (up to max_per_net)
168    result.extend(ips_onion[0:max_per_net])
169    return result
170
171def ip_stats(ips):
172    hist = collections.defaultdict(int)
173    for ip in ips:
174        if ip is not None:
175            hist[ip['net']] += 1
176
177    return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion'])
178
179def main():
180    lines = sys.stdin.readlines()
181    ips = [parseline(line) for line in lines]
182
183    print('\x1b[7m  IPv4   IPv6  Onion Pass                                               \x1b[0m', file=sys.stderr)
184    print('%s Initial' % (ip_stats(ips)), file=sys.stderr)
185    # Skip entries with invalid address.
186    ips = [ip for ip in ips if ip is not None]
187    print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr)
188    # Skip duplicates (in case multiple seeds files were concatenated)
189    ips = dedup(ips)
190    print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr)
191    # Skip entries from suspicious hosts.
192    ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS]
193    print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr)
194    # Enforce minimal number of blocks.
195    ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS]
196    print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr)
197    # Require service bit 1.
198    ips = [ip for ip in ips if (ip['service'] & 1) == 1]
199    print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr)
200    # Require at least 50% 30-day uptime for clearnet, 10% for onion.
201    req_uptime = {
202        'ipv4': 50,
203        'ipv6': 50,
204        'onion': 10,
205    }
206    ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]]
207    print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr)
208    # Require a known and recent user agent.
209    ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])]
210    print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr)
211    # Sort by availability (and use last success as tie breaker)
212    ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True)
213    # Filter out hosts with multiple bitcoin ports, these are likely abusive
214    ips = filtermultiport(ips)
215    print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr)
216    # Look up ASNs and limit results, both per ASN and globally.
217    ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
218    print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr)
219    # Sort the results by IP address (for deterministic output).
220    ips.sort(key=lambda x: (x['net'], x['sortkey']))
221    for ip in ips:
222        if ip['net'] == 'ipv6':
223            print('[%s]:%i' % (ip['ip'], ip['port']))
224        else:
225            print('%s:%i' % (ip['ip'], ip['port']))
226
227if __name__ == '__main__':
228    main()
229