1#!/usr/bin/env python3 2# Copyright (c) 2013-2020 The Bitcoin Core developers 3# Distributed under the MIT software license, see the accompanying 4# file COPYING or http://www.opensource.org/licenses/mit-license.php. 5# 6# Generate seeds.txt from Pieter's DNS seeder 7# 8 9import re 10import sys 11import dns.resolver 12import collections 13 14NSEEDS=512 15 16MAX_SEEDS_PER_ASN=2 17 18MIN_BLOCKS = 337600 19 20# These are hosts that have been observed to be behaving strangely (e.g. 21# aggressively connecting to every node). 22with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: 23 SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} 24 25 26PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") 27PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") 28PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$") 29PATTERN_AGENT = re.compile( 30 r"^/Satoshi:(" 31 r"0.14.(0|1|2|3|99)|" 32 r"0.15.(0|1|2|99)|" 33 r"0.16.(0|1|2|3|99)|" 34 r"0.17.(0|0.1|1|2|99)|" 35 r"0.18.(0|1|99)|" 36 r"0.19.(0|1|99)|" 37 r"0.20.(0|1|99)|" 38 r"0.21.99" 39 r")") 40 41def parseline(line): 42 sline = line.split() 43 if len(sline) < 11: 44 return None 45 m = PATTERN_IPV4.match(sline[0]) 46 sortkey = None 47 ip = None 48 if m is None: 49 m = PATTERN_IPV6.match(sline[0]) 50 if m is None: 51 m = PATTERN_ONION.match(sline[0]) 52 if m is None: 53 return None 54 else: 55 net = 'onion' 56 ipstr = sortkey = m.group(1) 57 port = int(m.group(2)) 58 else: 59 net = 'ipv6' 60 if m.group(1) in ['::']: # Not interested in localhost 61 return None 62 ipstr = m.group(1) 63 sortkey = ipstr # XXX parse IPv6 into number, could use name_to_ipv6 from generate-seeds 64 port = int(m.group(2)) 65 else: 66 # Do IPv4 sanity check 67 ip = 0 68 for i in range(0,4): 69 if int(m.group(i+2)) < 0 or int(m.group(i+2)) > 255: 70 return None 71 ip = ip + (int(m.group(i+2)) << (8*(3-i))) 72 if ip == 0: 73 return None 74 net = 'ipv4' 75 sortkey = ip 76 ipstr = m.group(1) 77 port = int(m.group(6)) 78 # Skip bad results. 79 if sline[1] == 0: 80 return None 81 # Extract uptime %. 82 uptime30 = float(sline[7][:-1]) 83 # Extract Unix timestamp of last success. 84 lastsuccess = int(sline[2]) 85 # Extract protocol version. 86 version = int(sline[10]) 87 # Extract user agent. 88 agent = sline[11][1:-1] 89 # Extract service flags. 90 service = int(sline[9], 16) 91 # Extract blocks. 92 blocks = int(sline[8]) 93 # Construct result. 94 return { 95 'net': net, 96 'ip': ipstr, 97 'port': port, 98 'ipnum': ip, 99 'uptime': uptime30, 100 'lastsuccess': lastsuccess, 101 'version': version, 102 'agent': agent, 103 'service': service, 104 'blocks': blocks, 105 'sortkey': sortkey, 106 } 107 108def dedup(ips): 109 '''deduplicate by address,port''' 110 d = {} 111 for ip in ips: 112 d[ip['ip'],ip['port']] = ip 113 return list(d.values()) 114 115def filtermultiport(ips): 116 '''Filter out hosts with more nodes per IP''' 117 hist = collections.defaultdict(list) 118 for ip in ips: 119 hist[ip['sortkey']].append(ip) 120 return [value[0] for (key,value) in list(hist.items()) if len(value)==1] 121 122def lookup_asn(net, ip): 123 ''' 124 Look up the asn for an IP (4 or 6) address by querying cymru.com, or None 125 if it could not be found. 126 ''' 127 try: 128 if net == 'ipv4': 129 ipaddr = ip 130 prefix = '.origin' 131 else: # http://www.team-cymru.com/IP-ASN-mapping.html 132 res = str() # 2001:4860:b002:23::68 133 for nb in ip.split(':')[:4]: # pick the first 4 nibbles 134 for c in nb.zfill(4): # right padded with '0' 135 res += c + '.' # 2001 4860 b002 0023 136 ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 137 prefix = '.origin6' 138 139 asn = int([x.to_text() for x in dns.resolver.resolve('.'.join( 140 reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', 141 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) 142 return asn 143 except Exception: 144 sys.stderr.write('ERR: Could not resolve ASN for "' + ip + '"\n') 145 return None 146 147# Based on Greg Maxwell's seed_filter.py 148def filterbyasn(ips, max_per_asn, max_per_net): 149 # Sift out ips by type 150 ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] 151 ips_onion = [ip for ip in ips if ip['net'] == 'onion'] 152 153 # Filter IPv46 by ASN, and limit to max_per_net per network 154 result = [] 155 net_count = collections.defaultdict(int) 156 asn_count = collections.defaultdict(int) 157 for ip in ips_ipv46: 158 if net_count[ip['net']] == max_per_net: 159 continue 160 asn = lookup_asn(ip['net'], ip['ip']) 161 if asn is None or asn_count[asn] == max_per_asn: 162 continue 163 asn_count[asn] += 1 164 net_count[ip['net']] += 1 165 result.append(ip) 166 167 # Add back Onions (up to max_per_net) 168 result.extend(ips_onion[0:max_per_net]) 169 return result 170 171def ip_stats(ips): 172 hist = collections.defaultdict(int) 173 for ip in ips: 174 if ip is not None: 175 hist[ip['net']] += 1 176 177 return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) 178 179def main(): 180 lines = sys.stdin.readlines() 181 ips = [parseline(line) for line in lines] 182 183 print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) 184 print('%s Initial' % (ip_stats(ips)), file=sys.stderr) 185 # Skip entries with invalid address. 186 ips = [ip for ip in ips if ip is not None] 187 print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) 188 # Skip duplicates (in case multiple seeds files were concatenated) 189 ips = dedup(ips) 190 print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) 191 # Skip entries from suspicious hosts. 192 ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] 193 print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) 194 # Enforce minimal number of blocks. 195 ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] 196 print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) 197 # Require service bit 1. 198 ips = [ip for ip in ips if (ip['service'] & 1) == 1] 199 print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) 200 # Require at least 50% 30-day uptime for clearnet, 10% for onion. 201 req_uptime = { 202 'ipv4': 50, 203 'ipv6': 50, 204 'onion': 10, 205 } 206 ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] 207 print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) 208 # Require a known and recent user agent. 209 ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] 210 print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) 211 # Sort by availability (and use last success as tie breaker) 212 ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) 213 # Filter out hosts with multiple bitcoin ports, these are likely abusive 214 ips = filtermultiport(ips) 215 print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) 216 # Look up ASNs and limit results, both per ASN and globally. 217 ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) 218 print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) 219 # Sort the results by IP address (for deterministic output). 220 ips.sort(key=lambda x: (x['net'], x['sortkey'])) 221 for ip in ips: 222 if ip['net'] == 'ipv6': 223 print('[%s]:%i' % (ip['ip'], ip['port'])) 224 else: 225 print('%s:%i' % (ip['ip'], ip['port'])) 226 227if __name__ == '__main__': 228 main() 229