1#!/usr/bin/env python3 2# Copyright (c) 2013-2019 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5# 6# Generate seeds.txt from Pieter's DNS seeder 7# 8 9import re 10import sys 11import dns.resolver 12import collections 13 14NSEEDS=512 15 16MAX_SEEDS_PER_ASN=2 17 18MIN_BLOCKS = 337600 19 20# These are hosts that have been observed to be behaving strangely (e.g. 21# aggressively connecting to every node). 22with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: 23 SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} 24 25 26PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") 27PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") 28PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$") 29PATTERN_AGENT = re.compile( 30 r"^/Satoshi:(" 31 r"0.14.(0|1|2|3|99)|" 32 r"0.15.(0|1|2|99)|" 33 r"0.16.(0|1|2|3|99)|" 34 r"0.17.(0|0.1|1|2|99)|" 35 r"0.18.(0|1|99)|" 36 r"0.19.(0|1|99)|" 37 r"0.20.99" 38 r")") 39 40def parseline(line): 41 sline = line.split() 42 if len(sline) < 11: 43 return None 44 m = PATTERN_IPV4.match(sline[0]) 45 sortkey = None 46 ip = None 47 if m is None: 48 m = PATTERN_IPV6.match(sline[0]) 49 if m is None: 50 m = PATTERN_ONION.match(sline[0]) 51 if m is None: 52 return None 53 else: 54 net = 'onion' 55 ipstr = sortkey = m.group(1) 56 port = int(m.group(2)) 57 else: 58 net = 'ipv6' 59 if m.group(1) in ['::']: # Not interested in localhost 60 return None 61 ipstr = m.group(1) 62 sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds 63 port = int(m.group(2)) 64 else: 65 # Do IPv4 sanity check 66 ip = 0 67 for i in range(0,4): 68 if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255: 69 return None 70 ip = ip + (int(m.group(i+2)) << (8*(3-i))) 71 if ip == 0: 72 return None 73 net = 'ipv4' 74 sortkey = ip 75 ipstr = m.group(1) 76 port = int(m.group(6)) 77 # Skip bad results. 78 if sline[1] == 0: 79 return None 80 # Extract uptime %. 81 uptime30 = float(sline[7][:-1]) 82 # Extract Unix timestamp of last success. 83 lastsuccess = int(sline[2]) 84 # Extract protocol version. 85 version = int(sline[10]) 86 # Extract user agent. 87 agent = sline[11][1:-1] 88 # Extract service flags. 89 service = int(sline[9], 16) 90 # Extract blocks. 91 blocks = int(sline[8]) 92 # Construct result. 93 return { 94 'net': net, 95 'ip': ipstr, 96 'port': port, 97 'ipnum': ip, 98 'uptime': uptime30, 99 'lastsuccess': lastsuccess, 100 'version': version, 101 'agent': agent, 102 'service': service, 103 'blocks': blocks, 104 'sortkey': sortkey, 105 } 106 107def dedup(ips): 108 '''deduplicate by address,port''' 109 d = {} 110 for ip in ips: 111 d[ip['ip'],ip['port']] = ip 112 return list(d.values()) 113 114def filtermultiport(ips): 115 '''Filter out hosts with more nodes per IP''' 116 hist = collections.defaultdict(list) 117 for ip in ips: 118 hist[ip['sortkey']].append(ip) 119 return [value[0] for (key,value) in list(hist.items()) if len(value)==1] 120 121def lookup_asn(net, ip): 122 ''' 123 Look up the asn for an IP (4 or 6) address by querying cymru.com, or None 124 if it could not be found. 125 ''' 126 try: 127 if net == 'ipv4': 128 ipaddr = ip 129 prefix = '.origin' 130 else: # http://www.team-cymru.com/IP-ASN-mapping.html 131 res = str() # 2001:4860:b002:23::68 132 for nb in ip.split(':')[:4]: # pick the first 4 nibbles 133 for c in nb.zfill(4): # right padded with '0' 134 res += c + '.' # 2001 4860 b002 0023 135 ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 136 prefix = '.origin6' 137 138 asn = int([x.to_text() for x in dns.resolver.query('.'.join( 139 reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', 140 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) 141 return asn 142 except Exception: 143 sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n') 144 return None 145 146# Based on Greg Maxwell's seed_filter.py 147def filterbyasn(ips, max_per_asn, max_per_net): 148 # Sift out ips by type 149 ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] 150 ips_onion = [ip for ip in ips if ip['net'] == 'onion'] 151 152 # Filter IPv46 by ASN, and limit to max_per_net per network 153 result = [] 154 net_count = collections.defaultdict(int) 155 asn_count = collections.defaultdict(int) 156 for ip in ips_ipv46: 157 if net_count[ip['net']] == max_per_net: 158 continue 159 asn = lookup_asn(ip['net'], ip['ip']) 160 if asn is None or asn_count[asn] == max_per_asn: 161 continue 162 asn_count[asn] += 1 163 net_count[ip['net']] += 1 164 result.append(ip) 165 166 # Add back Onions (up to max_per_net) 167 result.extend(ips_onion[0:max_per_net]) 168 return result 169 170def ip_stats(ips): 171 hist = collections.defaultdict(int) 172 for ip in ips: 173 if ip is not None: 174 hist[ip['net']] += 1 175 176 return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) 177 178def main(): 179 lines = sys.stdin.readlines() 180 ips = [parseline(line) for line in lines] 181 182 print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) 183 print('%s Initial' % (ip_stats(ips)), file=sys.stderr) 184 # Skip entries with invalid address. 185 ips = [ip for ip in ips if ip is not None] 186 print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) 187 # Skip duplicates (in case multiple seeds files were concatenated) 188 ips = dedup(ips) 189 print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) 190 # Skip entries from suspicious hosts. 191 ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] 192 print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) 193 # Enforce minimal number of blocks. 194 ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] 195 print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) 196 # Require service bit 1. 197 ips = [ip for ip in ips if (ip['service'] & 1) == 1] 198 print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) 199 # Require at least 50% 30-day uptime for clearnet, 10% for onion. 200 req_uptime = { 201 'ipv4': 50, 202 'ipv6': 50, 203 'onion': 10, 204 } 205 ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] 206 print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) 207 # Require a known and recent user agent. 208 ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] 209 print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) 210 # Sort by availability (and use last success as tie breaker) 211 ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) 212 # Filter out hosts with multiple bitcoin ports, these are likely abusive 213 ips = filtermultiport(ips) 214 print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) 215 # Look up ASNs and limit results, both per ASN and globally. 216 ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) 217 print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) 218 # Sort the results by IP address (for deterministic output). 219 ips.sort(key=lambda x: (x['net'], x['sortkey'])) 220 for ip in ips: 221 if ip['net'] == 'ipv6': 222 print('[%s]:%i' % (ip['ip'], ip['port'])) 223 else: 224 print('%s:%i' % (ip['ip'], ip['port'])) 225 226if __name__ == '__main__': 227 main() 228