1# 2# This file is a part of DNSViz, a tool suite for DNS/DNSSEC monitoring, 3# analysis, and visualization. 4# Created by Casey Deccio (casey@deccio.net) 5# 6# Copyright 2012-2014 Sandia Corporation. Under the terms of Contract 7# DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains 8# certain rights in this software. 9# 10# Copyright 2014-2016 VeriSign, Inc. 11# 12# Copyright 2016-2021 Casey Deccio 13# 14# DNSViz is free software; you can redistribute it and/or modify 15# it under the terms of the GNU General Public License as published by 16# the Free Software Foundation; either version 2 of the License, or 17# (at your option) any later version. 18# 19# DNSViz is distributed in the hope that it will be useful, 20# but WITHOUT ANY WARRANTY; without even the implied warranty of 21# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22# GNU General Public License for more details. 23# 24# You should have received a copy of the GNU General Public License along 25# with DNSViz. If not, see <http://www.gnu.org/licenses/>. 26# 27 28from __future__ import unicode_literals 29 30import binascii 31import datetime 32import logging 33import random 34import re 35import socket 36import sys 37import threading 38import time 39import uuid 40 41# minimal support for python2.6 42try: 43 from collections import OrderedDict 44except ImportError: 45 from ordereddict import OrderedDict 46 47import dns.flags, dns.name, dns.rdataclass, dns.rdatatype, dns.resolver 48 49import dnsviz.format as fmt 50from dnsviz.ipaddr import * 51import dnsviz.query as Q 52import dnsviz.resolver as Resolver 53from dnsviz import transport 54from dnsviz import util 55lb2s = fmt.latin1_binary_to_string 56 57_logger = logging.getLogger(__name__) 58 59DNS_RAW_VERSION = 1.2 60 61class NetworkConnectivityException(Exception): 62 pass 63 64class IPv4ConnectivityException(NetworkConnectivityException): 65 pass 66 67class IPv6ConnectivityException(NetworkConnectivityException): 68 pass 69 70class NoNameservers(NetworkConnectivityException): 71 pass 72 73ARPA_NAME = dns.name.from_text('arpa') 74IP6_ARPA_NAME = dns.name.from_text('ip6', ARPA_NAME) 75INADDR_ARPA_NAME = dns.name.from_text('in-addr', ARPA_NAME) 76E164_ARPA_NAME = dns.name.from_text('e164', ARPA_NAME) 77 78DANE_PORT_RE = re.compile(r'^_(\d+)$') 79SRV_PORT_RE = re.compile(r'^_.*[^\d].*$') 80PROTO_LABEL_RE = re.compile(r'^_(tcp|udp|sctp)$') 81 82WILDCARD_EXPLICIT_DELEGATION = dns.name.from_text('*') 83 84COOKIE_STANDIN = binascii.unhexlify('cccccccccccccccc') 85COOKIE_BAD = binascii.unhexlify('bbbbbbbbbbbbbbbb') 86 87ANALYSIS_TYPE_AUTHORITATIVE = 0 88ANALYSIS_TYPE_RECURSIVE = 1 89ANALYSIS_TYPE_CACHE = 2 90 91analysis_types = { 92 ANALYSIS_TYPE_AUTHORITATIVE: 'authoritative', 93 ANALYSIS_TYPE_RECURSIVE: 'recursive', 94 ANALYSIS_TYPE_CACHE: 'cache', 95} 96analysis_type_codes = { 97 'authoritative': ANALYSIS_TYPE_AUTHORITATIVE, 98 'recursive': ANALYSIS_TYPE_RECURSIVE, 99 'cache': ANALYSIS_TYPE_CACHE, 100} 101 102class OnlineDomainNameAnalysis(object): 103 QUERY_CLASS = Q.MultiQuery 104 105 def __init__(self, name, stub=False, analysis_type=ANALYSIS_TYPE_AUTHORITATIVE, cookie_standin=None, cookie_bad=None): 106 107 ################################################## 108 # General attributes 109 ################################################## 110 111 # The name that is the focus of the analysis (serialized). 112 self.name = name 113 self.analysis_type = analysis_type 114 self.stub = stub 115 116 # Attributes related to DNS cookie 117 if cookie_standin is None: 118 cookie_standin = COOKIE_STANDIN 119 self.cookie_standin = cookie_standin 120 if cookie_bad is None: 121 cookie_bad = COOKIE_BAD 122 self.cookie_bad = cookie_bad 123 124 # a class for constructing the queries 125 self._query_cls = self.QUERY_CLASS 126 127 # A unique identifier for the analysis 128 self.uuid = uuid.uuid4() 129 130 # Analysis start and end (serialized). 131 self.analysis_start = None 132 self.analysis_end = None 133 134 # The record types queried with the name when eliciting a referral, 135 # eliciting authority section NS records, and eliciting DNS cookies 136 # (serialized). 137 self.referral_rdtype = None 138 self.auth_rdtype = None 139 self.cookie_rdtype = None 140 141 # Whether or not the delegation was specified explicitly or learned 142 # by delegation. This is for informational purposes more than 143 # functional purposes (serialized). 144 self.explicit_delegation = False 145 146 # The queries issued to and corresponding responses received from the 147 # servers (serialized). 148 self.queries = {} 149 150 # A reference to the analysis of the parent authority (and that of the 151 # DLV parent, if any). 152 self.parent = None 153 154 self._dlv_parent = None 155 self._dlv_name = None 156 157 # A reference to the highest ancestor for which NXDOMAIN was received 158 # (serialized). 159 self.nxdomain_ancestor = None 160 161 # The clients used for queries (serialized - for convenience) 162 self.clients_ipv4 = set() 163 self.clients_ipv6 = set() 164 165 # Meta information associated with the domain name. These are 166 # set when responses are processed. 167 self.has_soa = False 168 self.has_ns = False 169 self.cname_targets = {} 170 self.ns_dependencies = {} 171 self.mx_targets = {} 172 self.external_signers = {} 173 174 ################################################## 175 # Zone-specific attributes 176 ################################################## 177 178 # The DNS names and record types queried to analyze negative responses 179 # of different types (serialized). 180 self.nxdomain_name = None 181 self.nxdomain_rdtype = None 182 self.nxrrset_name = None 183 self.nxrrset_rdtype = None 184 185 # A mapping of names of authoritative servers to IP addresses returned 186 # in authoritative responses (serialized). 187 self._auth_ns_ip_mapping = {} 188 189 # These are populated as responses are added. 190 self._glue_ip_mapping = {} 191 self._ns_names_in_child = set() 192 self._all_servers_queried = set() 193 self._all_servers_clients_queried = set() 194 self._all_servers_clients_queried_tcp = set() 195 self._responsive_servers_clients_udp = set() 196 self._responsive_servers_clients_tcp = set() 197 self._auth_servers_clients = set() 198 self._valid_servers_clients_udp = set() 199 self._valid_servers_clients_tcp = set() 200 201 # A mapping of server to server-provided DNS cookie 202 self.cookie_jar = {} 203 204 def __repr__(self): 205 return '<%s %s>' % (self.__class__.__name__, self.__str__()) 206 207 def __str__(self): 208 return fmt.humanize_name(self.name, True) 209 210 def __eq__(self, other): 211 return self.name == other.name 212 213 def __hash__(self): 214 return hash(self.name) 215 216 def parent_name(self): 217 if self.parent is not None: 218 return self.parent.name 219 return None 220 221 def dlv_parent_name(self): 222 if self.dlv_parent is not None: 223 return self.dlv_parent.name 224 return None 225 226 def nxdomain_ancestor_name(self): 227 if self.nxdomain_ancestor is not None: 228 return self.nxdomain_ancestor.name 229 return None 230 231 def _set_dlv_parent(self, dlv_parent): 232 self._dlv_parent = dlv_parent 233 if dlv_parent is None: 234 self._dlv_name = None 235 else: 236 try: 237 self._dlv_name = dns.name.Name(self.name.labels[:-1] + dlv_parent.name.labels) 238 except dns.name.NameTooLong: 239 self._dlv_parent = None 240 self._dlv_name = None 241 242 def _get_dlv_parent(self): 243 return self._dlv_parent 244 dlv_parent = property(_get_dlv_parent, _set_dlv_parent) 245 246 def _get_dlv_name(self): 247 return self._dlv_name 248 dlv_name = property(_get_dlv_name) 249 250 def is_zone(self): 251 return bool(self.has_ns or self.name == dns.name.root or self._auth_ns_ip_mapping) 252 253 def _get_zone(self): 254 if self.is_zone(): 255 return self 256 else: 257 return self.parent 258 zone = property(_get_zone) 259 260 def single_client(self, exclude_loopback=True, exclude_ipv4_mapped=True): 261 clients_ipv4 = [x for x in self.clients_ipv4 if not exclude_loopback or LOOPBACK_IPV4_RE.match(x) is None] 262 clients_ipv6 = [x for x in self.clients_ipv6 if (not exclude_loopback or x != LOOPBACK_IPV6) and (not exclude_ipv4_mapped or IPV4_MAPPED_IPV6_RE.match(x) is None)] 263 return len(clients_ipv4) <= 1 and len(clients_ipv6) <= 1 264 265 def get_name(self, name, trace=None): 266 #XXX this whole method is a hack 267 if trace is None: 268 trace = [] 269 270 if self in trace: 271 return None 272 273 if name in (self.name, self.nxdomain_name, self.nxrrset_name, self.dlv_name): 274 return self 275 for cname in self.cname_targets: 276 for target, cname_obj in self.cname_targets[cname].items(): 277 #XXX it is possible for cname_obj to be None where 278 # this name was populated with level RDTYPES_SECURE_DELEGATION. 279 # when this method is refactored appropriately, this check won't 280 # be necessary. 281 if cname_obj is None: 282 continue 283 ref = cname_obj.get_name(name, trace=trace + [self]) 284 if ref is not None: 285 return ref 286 if name in self.external_signers: 287 return self.external_signers[name] 288 if name in self.ns_dependencies and self.ns_dependencies[name] is not None: 289 return self.ns_dependencies[name] 290 if name in self.mx_targets and self.mx_targets[name] is not None: 291 return self.mx_targets[name] 292 if self.name.is_subdomain(name) and self.parent is not None: 293 return self.parent.get_name(name, trace=trace + [self]) 294 elif name == self.dlv_parent_name(): 295 return self.dlv_parent 296 elif name == self.nxdomain_ancestor_name(): 297 return self.nxdomain_ancestor 298 return None 299 300 def get_bailiwick_mapping(self): 301 if not hasattr(self, '_bailiwick_mapping') or self._bailiwick_mapping is None: 302 if self.parent is None: 303 self._bailiwick_mapping = {}, self.name 304 else: 305 self._bailiwick_mapping = dict([(s,self.parent_name()) for s in self.parent.get_auth_or_designated_servers()]), self.name 306 return self._bailiwick_mapping 307 308 def get_cookie_jar_mapping(self): 309 if not hasattr(self, '_cookie_jar_mapping') or self._cookie_jar_mapping is None: 310 if self.parent is None: 311 self._cookie_jar_mapping = {}, self.cookie_jar 312 else: 313 self._cookie_jar_mapping = dict([(s,self.parent.cookie_jar) for s in self.parent.get_auth_or_designated_servers()]), self.cookie_jar 314 return self._cookie_jar_mapping 315 316 def _add_glue_ip_mapping(self, response): 317 '''Extract a mapping of NS targets to IP addresses from A and AAAA 318 records in the additional section of a referral.''' 319 320 ip_mapping = response.ns_ip_mapping_from_additional(self.name, self.parent_name()) 321 for name, ip_set in ip_mapping.items(): 322 if name not in self._glue_ip_mapping: 323 self._glue_ip_mapping[name] = set() 324 self._glue_ip_mapping[name].update(ip_set) 325 326 # this includes both out-of-bailiwick names (because 327 # ns_ip_mapping_from_additional() is called with 328 # self.parent_name()) and those that have no IPs 329 # in the additional section. 330 if not ip_set: 331 self.ns_dependencies[name] = None 332 333 def _handle_mx_response(self, rrset): 334 '''Save the targets from an MX RRset with the name which is the 335 subject of this analysis.''' 336 337 for mx in rrset: 338 self.mx_targets[mx.exchange] = None 339 340 def _handle_cname_response(self, rrset): 341 '''Save the targets from a CNAME RRset with the name which is the 342 subject of this analysis.''' 343 344 if rrset.name not in self.cname_targets: 345 self.cname_targets[rrset.name] = {} 346 self.cname_targets[rrset.name][rrset[0].target] = None 347 348 def _handle_ns_response(self, rrset, update_ns_names): 349 '''Indicate that there exist NS records for the name which is the 350 subject of this analysis, and, if authoritative, save the NS 351 targets.''' 352 353 self.has_ns = True 354 if update_ns_names: 355 for ns in rrset: 356 self._ns_names_in_child.add(ns.target) 357 358 def set_ns_dependencies(self): 359 # the following check includes explicit delegations 360 if self.parent is None: 361 return 362 for ns in self.get_ns_names_in_child().difference(self.get_ns_names_in_parent()): 363 self.ns_dependencies[ns] = None 364 365 def _set_server_cookies(self, response, server): 366 server_cookie = response.get_server_cookie() 367 if server_cookie is not None and server not in self.cookie_jar: 368 self.cookie_jar[server] = server_cookie 369 370 def _process_response_answer_rrset(self, rrset, query, response): 371 if query.qname in (self.name, self.dlv_name): 372 if rrset.rdtype == dns.rdatatype.MX: 373 self._handle_mx_response(rrset) 374 elif rrset.rdtype == dns.rdatatype.NS: 375 self._handle_ns_response(rrset, not self.explicit_delegation) 376 377 # check whether it is signed and whether the signer matches 378 try: 379 rrsig_rrset = response.message.find_rrset(response.message.answer, query.qname, query.rdclass, dns.rdatatype.RRSIG, rrset.rdtype) 380 381 for rrsig in rrsig_rrset: 382 if rrsig_rrset.covers == dns.rdatatype.DS and self.parent is None: 383 pass 384 elif rrsig_rrset.covers == dns.rdatatype.DS and rrsig.signer == self.parent_name(): 385 pass 386 elif rrsig_rrset.covers == dns.rdatatype.DLV and rrsig.signer == self.dlv_parent_name(): 387 pass 388 elif rrsig.signer == self.zone.name: 389 pass 390 else: 391 self.external_signers[rrsig.signer] = None 392 except KeyError: 393 pass 394 395 if rrset.rdtype == dns.rdatatype.CNAME: 396 self._handle_cname_response(rrset) 397 398 def _process_response(self, response, server, client, query, bailiwick, detect_ns, detect_cookies): 399 '''Process a DNS response from a query, setting and updating instance 400 variables appropriately, and calling helper methods as necessary.''' 401 402 if response.message is None: 403 return 404 405 if detect_cookies: 406 self._set_server_cookies(response, server) 407 408 is_authoritative = response.is_authoritative() 409 410 if response.is_valid_response(): 411 if response.effective_tcp: 412 self._valid_servers_clients_tcp.add((server, client)) 413 else: 414 self._valid_servers_clients_udp.add((server, client)) 415 if is_authoritative: 416 # we're marking servers as authoritative for the zone. if the query 417 # type is DS or DLV, then query is to the parent zone, so its 418 # authoritativeness means nothing to us. 419 if query.rdtype in (dns.rdatatype.DS, dns.rdatatype.DLV): 420 pass 421 # If this response comes from a parent server, and the response was 422 # NXDOMAIN, then don't count it as authoritative. Either it was 423 # the child of a zone, in which case _auth_servers_clients doesn't 424 # matter (since _auth_servers_clients is only used for zones), or 425 # it was a zone itself, and not all servers in the parent carry the 426 # delegation 427 elif response.message.rcode() == dns.rcode.NXDOMAIN and bailiwick != self.name: 428 pass 429 else: 430 self._auth_servers_clients.add((server, client)) 431 432 if not response.is_complete_response(): 433 return 434 435 # retrieve the corresponding RRset in the answer section 436 rrset = None 437 try: 438 rrset = response.message.find_rrset(response.message.answer, query.qname, query.rdclass, query.rdtype) 439 except KeyError: 440 try: 441 rrset = response.message.find_rrset(response.message.answer, query.qname, query.rdclass, dns.rdatatype.CNAME) 442 except KeyError: 443 pass 444 445 # in the case where a corresponding RRset is found, analyze it here 446 if rrset is not None: 447 self._process_response_answer_rrset(rrset, query, response) 448 449 # look for SOA in authority section, in the case of negative responses 450 try: 451 soa_rrset = [x for x in response.message.authority if x.rdtype == dns.rdatatype.SOA and x.rdclass == query.rdclass][0] 452 if soa_rrset.name == self.name: 453 self.has_soa = True 454 except IndexError: 455 pass 456 457 if query.qname == self.name and detect_ns: 458 # if this is a referral, also grab the referral information, if it 459 # pertains to this name (could alternatively be a parent) 460 if response.is_referral(query.qname, query.rdtype, query.rdclass, bailiwick): 461 try: 462 rrset = response.message.find_rrset(response.message.authority, self.name, query.rdclass, dns.rdatatype.NS) 463 except KeyError: 464 pass 465 else: 466 self._add_glue_ip_mapping(response) 467 self._handle_ns_response(rrset, False) 468 469 # if it is an (authoritative) answer that has authority information, then add it 470 else: 471 try: 472 rrset = response.message.find_rrset(response.message.authority, query.qname, query.rdclass, dns.rdatatype.NS) 473 self._handle_ns_response(rrset, is_authoritative and not self.explicit_delegation) 474 except KeyError: 475 pass 476 477 def add_auth_ns_ip_mappings(self, *mappings): 478 '''Add one or more mappings from NS targets to IPv4 or IPv6 addresses, 479 as resolved by querying authoritative sources. Arguments are 2-tuples 480 of the form (DNS name, address).''' 481 482 for name, ip in mappings: 483 if name not in self._auth_ns_ip_mapping: 484 self._auth_ns_ip_mapping[name] = set() 485 if ip is not None: 486 self._auth_ns_ip_mapping[name].add(ip) 487 488 def add_query(self, query, detect_ns, detect_cookies): 489 '''Process a DNS query and its responses, setting and updating instance 490 variables appropriately, and calling helper methods as necessary.''' 491 492 bailiwick_map, default_bailiwick = self.get_bailiwick_mapping() 493 494 key = (query.qname, query.rdtype) 495 if key not in self.queries: 496 self.queries[key] = self._query_cls(query.qname, query.rdtype, query.rdclass) 497 self.queries[key].add_query(query, bailiwick_map, default_bailiwick) 498 499 for server in query.responses: 500 bailiwick = bailiwick_map.get(server, default_bailiwick) 501 502 # note the fact that the server was queried 503 self._all_servers_queried.add(server) 504 505 for client in query.responses[server]: 506 response = query.responses[server][client] 507 508 # note clients used 509 if client.version == 6: 510 self.clients_ipv6.add(client) 511 else: 512 self.clients_ipv4.add(client) 513 514 # note server responsiveness 515 if response.udp_attempted: 516 self._all_servers_clients_queried.add((server, client)) 517 if response.tcp_attempted: 518 self._all_servers_clients_queried_tcp.add((server, client)) 519 if response.udp_responsive: 520 self._responsive_servers_clients_udp.add((server, client)) 521 if response.tcp_responsive: 522 self._responsive_servers_clients_tcp.add((server, client)) 523 524 self._process_response(query.responses[server][client], server, client, query, bailiwick, detect_ns, detect_cookies) 525 526 def get_glue_ip_mapping(self): 527 '''Return a reference to the mapping of targets of delegation records 528 (i.e., NS records in the parent zone) and their corresponding IPv4 or 529 IPv6 glue, if any.''' 530 531 return self._glue_ip_mapping 532 533 def get_root_hint_mapping(self): 534 servers = {} 535 hints = util.get_root_hints() 536 for rdata in hints[(dns.name.root, dns.rdatatype.NS)]: 537 servers[rdata.target] = set() 538 for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA): 539 if (rdata.target, rdtype) in hints: 540 servers[rdata.target].update([IPAddr(r.address) for r in hints[(rdata.target, rdtype)]]) 541 for name, server in util.HISTORICAL_ROOT_IPS: 542 if name not in servers: 543 servers[name] = set() 544 servers[name].add(server) 545 return servers 546 547 def _get_servers_from_hints(self, name, hints): 548 servers = set() 549 server_mapping = self._get_server_ip_mapping_from_hints(name, hints) 550 for ns_name in server_mapping: 551 servers.update(server_mapping[ns_name]) 552 return servers 553 554 def get_auth_ns_ip_mapping(self): 555 '''Return a reference to the mapping of NS targets from delegation or 556 authoritative source to their authoritative IPv4 and IPv6 addresses.''' 557 558 return self._auth_ns_ip_mapping 559 560 def get_ns_names_in_parent(self): 561 '''Return the set of names corresponding to targets of delegation 562 records.''' 563 564 return set(self.get_glue_ip_mapping()) 565 566 def get_ns_names_in_child(self): 567 '''Return the set of names corresponding to targets of authoritative 568 NS records.''' 569 570 return self._ns_names_in_child 571 572 def get_ns_names(self): 573 '''Return the comprehensive set of names corresponding to NS targets.''' 574 575 return self.get_ns_names_in_parent().union(self.get_ns_names_in_child()) 576 577 def get_servers_in_parent(self): 578 '''Return the IP addresses of servers corresponding to names in the 579 delegation records. If the name is a subset of the name being queried, 580 then glue is required, and the glue is used exclusively. If the name 581 is in-bailiwick and there is glue in the referral, then the glue 582 records alone are used; otherwise the authoritative IPs are used. If 583 the name is out-of-bailiwick, then only the authoritative IPs are 584 used.''' 585 586 if not hasattr(self, '_servers_in_parent') or self._servers_in_parent is None: 587 servers = set() 588 if self.parent is None: 589 return servers 590 glue_ips = self.get_glue_ip_mapping() 591 auth_ips = self.get_auth_ns_ip_mapping() 592 for name in glue_ips: 593 in_bailiwick = name.is_subdomain(self.parent_name()) 594 glue_required = name.is_subdomain(self.name) 595 if glue_required: 596 servers.update(glue_ips[name]) 597 elif in_bailiwick: 598 if glue_ips[name]: 599 servers.update(glue_ips[name]) 600 elif name in auth_ips: 601 servers.update(auth_ips[name]) 602 elif name in auth_ips: 603 servers.update(auth_ips[name]) 604 self._servers_in_parent = servers 605 return self._servers_in_parent 606 607 def get_servers_in_child(self): 608 '''Return the authoritative IP addresses of servers corresponding to 609 names in the authoritative NS records.''' 610 611 if not hasattr(self, '_servers_in_child') or self._servers_in_child is None: 612 servers = set() 613 auth_ips = self.get_auth_ns_ip_mapping() 614 for name in self.get_ns_names_in_child(): 615 if name in auth_ips: 616 servers.update(auth_ips[name]) 617 self._servers_in_child = servers 618 return self._servers_in_child 619 620 def get_designated_servers(self, no_cache=False): 621 '''Return the set of glue or authoritative IP addresses of servers 622 corresponding to names in the delegation or authoritative NS 623 records.''' 624 625 if not hasattr(self, '_designated_servers') or self._designated_servers is None: 626 servers = set() 627 glue_ips = self.get_glue_ip_mapping() 628 auth_ips = self.get_auth_ns_ip_mapping() 629 for name in glue_ips: 630 servers.update(glue_ips[name]) 631 for name in auth_ips: 632 servers.update(auth_ips[name]) 633 if no_cache: 634 return servers 635 self._designated_servers = servers 636 return self._designated_servers 637 638 def get_valid_servers_udp(self, proto=None): 639 '''Return the set of servers that responded with a valid (rcode of 640 NOERROR or NXDOMAIN) response.''' 641 642 valid_servers = set([x[0] for x in self._valid_servers_clients_udp]) 643 if proto is not None: 644 return set([x for x in valid_servers if x.version == proto]) 645 else: 646 return valid_servers 647 648 def get_valid_servers_tcp(self, proto=None): 649 '''Return the set of servers that responded with a valid (rcode of 650 NOERROR or NXDOMAIN) response.''' 651 652 valid_servers = set([x[0] for x in self._valid_servers_clients_tcp]) 653 if proto is not None: 654 return set([x for x in valid_servers if x.version == proto]) 655 else: 656 return valid_servers 657 658 def get_responsive_servers_udp(self, proto=None): 659 '''Return the set of servers for which some type of response was 660 received from any client over UDP.''' 661 662 responsive_servers = set([x[0] for x in self._responsive_servers_clients_udp]) 663 if proto is not None: 664 return set([x for x in responsive_servers if x.version == proto]) 665 else: 666 return responsive_servers 667 668 def get_responsive_servers_tcp(self, proto=None): 669 '''Return the set of servers for which some type of response was 670 received from any client over TCP.''' 671 672 responsive_servers = set([x[0] for x in self._responsive_servers_clients_tcp]) 673 if proto is not None: 674 return set([x for x in responsive_servers if x.version == proto]) 675 else: 676 return responsive_servers 677 678 def get_auth_or_designated_servers(self, proto=None, no_cache=False): 679 '''Return the set of servers that either answered authoritatively 680 or were explicitly designated by NS and glue or authoritative IP.''' 681 682 if not hasattr(self, '_auth_or_designated_servers') or self._auth_or_designated_servers is None: 683 servers = set([x[0] for x in self._auth_servers_clients]).union(self.get_designated_servers(no_cache)) 684 if not no_cache: 685 self._auth_or_designated_servers = servers 686 else: 687 servers = self._auth_or_designated_servers 688 689 if proto is not None: 690 return set([x for x in servers if x.version == proto]) 691 else: 692 return servers 693 694 def get_responsive_auth_or_designated_servers(self, proto=None, no_cache=False): 695 '''Return the set of servers that either answered authoritatively 696 or were explicitly designated by NS and glue or authoritative IP and 697 were responsive to queries.''' 698 699 return self.get_auth_or_designated_servers(proto, no_cache).intersection(self.get_responsive_servers_udp(proto)) 700 701 def get_valid_auth_or_designated_servers(self, proto=None, no_cache=False): 702 '''Return the set of servers that either answered authoritatively 703 or were explicitly designated by NS and glue or authoritative IP and 704 returned a valid (rcode of NOERROR or NXDOMAIN) response.''' 705 706 return self.get_auth_or_designated_servers(proto, no_cache).intersection(self.get_valid_servers_udp(proto)) 707 708 def get_stealth_servers(self): 709 '''Return the set of servers that authoritatively but weren't 710 explicitly designated by NS and glue or authoritative IP.''' 711 712 if not hasattr(self, '_stealth_auth_servers') or self._stealth_auth_servers is None: 713 servers = self.get_auth_or_designated_servers().difference(self.get_designated_servers()) 714 self._stealth_auth_servers = servers 715 return self._stealth_auth_servers 716 717 def get_ip_ns_name_mapping(self): 718 '''Return a mapping of each designated server to the NS target name 719 that it resolves to. The result for each IP is a list of names in 720 which names that resolve to it authoritatively appear before names 721 that map to it in glue.''' 722 723 if not hasattr(self, '_ip_ns_name_mapping') or self._ip_ns_name_mapping is None: 724 self._ip_ns_name_mapping = {} 725 if self.name == dns.name.root: 726 glue_ips = self.get_root_hint_mapping() 727 else: 728 glue_ips = self.get_glue_ip_mapping() 729 auth_ips = self.get_auth_ns_ip_mapping() 730 if self.stub: 731 auth_names = set(auth_ips) 732 else: 733 auth_names = self.get_ns_names() 734 735 # if there are no names from glue or from authoritative responses, 736 # then use the authoritative IP. Such is the case with explicit 737 # delegation 738 if not auth_names: 739 auth_names = auth_ips 740 741 for name in auth_names: 742 if name in auth_ips: 743 for ip in auth_ips[name]: 744 if ip not in self._ip_ns_name_mapping: 745 self._ip_ns_name_mapping[ip] = [] 746 self._ip_ns_name_mapping[ip].append(name) 747 748 for name in glue_ips: 749 for ip in glue_ips[name]: 750 if ip not in self._ip_ns_name_mapping: 751 self._ip_ns_name_mapping[ip] = [name] 752 elif name not in self._ip_ns_name_mapping[ip]: 753 self._ip_ns_name_mapping[ip].append(name) 754 755 return self._ip_ns_name_mapping 756 757 def get_ns_name_for_ip(self, ip): 758 '''Return the NS target name(s) that resolve to the given IP, either 759 authoritatively or using glue.''' 760 761 ip_name_mapping = self.get_ip_ns_name_mapping() 762 try: 763 return ip_name_mapping[ip], self.name 764 except KeyError: 765 pass 766 767 if self.parent is None: 768 return [], None 769 return self.parent.get_ns_name_for_ip(ip) 770 771 def serialize(self, d=None, meta_only=False, trace=None): 772 if d is None: 773 d = OrderedDict() 774 775 if trace is None: 776 trace = [] 777 778 if self in trace: 779 return 780 781 name_str = lb2s(self.name.canonicalize().to_text()) 782 if name_str in d: 783 return 784 785 # serialize dependencies first because their version of the analysis 786 # might be the most complete (considering re-dos) 787 self._serialize_dependencies(d, meta_only, trace) 788 789 if self.parent is not None: 790 self.parent.serialize(d, meta_only, trace + [self]) 791 if self.dlv_parent is not None: 792 self.dlv_parent.serialize(d, meta_only, trace + [self]) 793 if self.nxdomain_ancestor is not None: 794 self.nxdomain_ancestor.serialize(d, meta_only, trace + [self]) 795 796 clients_ipv4 = list(self.clients_ipv4) 797 clients_ipv4.sort() 798 clients_ipv6 = list(self.clients_ipv6) 799 clients_ipv6.sort() 800 801 d[name_str] = OrderedDict() 802 d[name_str]['type'] = analysis_types[self.analysis_type] 803 d[name_str]['stub'] = self.stub 804 if self.cookie_standin is not None: 805 d[name_str]['cookie_standin'] = lb2s(binascii.hexlify(self.cookie_standin)) 806 if self.cookie_bad is not None: 807 d[name_str]['cookie_bad'] = lb2s(binascii.hexlify(self.cookie_bad)) 808 d[name_str]['analysis_start'] = fmt.datetime_to_str(self.analysis_start) 809 d[name_str]['analysis_end'] = fmt.datetime_to_str(self.analysis_end) 810 if not self.stub: 811 d[name_str]['clients_ipv4'] = clients_ipv4 812 d[name_str]['clients_ipv6'] = clients_ipv6 813 814 if self.parent is not None: 815 d[name_str]['parent'] = lb2s(self.parent_name().canonicalize().to_text()) 816 if self.dlv_parent is not None: 817 d[name_str]['dlv_parent'] = lb2s(self.dlv_parent_name().canonicalize().to_text()) 818 if self.nxdomain_ancestor is not None: 819 d[name_str]['nxdomain_ancestor'] = lb2s(self.nxdomain_ancestor_name().canonicalize().to_text()) 820 if self.referral_rdtype is not None: 821 d[name_str]['referral_rdtype'] = dns.rdatatype.to_text(self.referral_rdtype) 822 if self.auth_rdtype is not None: 823 d[name_str]['auth_rdtype'] = dns.rdatatype.to_text(self.auth_rdtype) 824 if self.cookie_rdtype is not None: 825 d[name_str]['cookie_rdtype'] = dns.rdatatype.to_text(self.cookie_rdtype) 826 d[name_str]['explicit_delegation'] = self.explicit_delegation 827 if self.nxdomain_name is not None: 828 d[name_str]['nxdomain_name'] = lb2s(self.nxdomain_name.to_text()) 829 d[name_str]['nxdomain_rdtype'] = dns.rdatatype.to_text(self.nxdomain_rdtype) 830 if self.nxrrset_name is not None: 831 d[name_str]['nxrrset_name'] = lb2s(self.nxrrset_name.to_text()) 832 d[name_str]['nxrrset_rdtype'] = dns.rdatatype.to_text(self.nxrrset_rdtype) 833 834 self._serialize_related(d[name_str], meta_only) 835 836 def _serialize_related(self, d, meta_only): 837 if self._auth_ns_ip_mapping: 838 d['auth_ns_ip_mapping'] = OrderedDict() 839 ns_names = list(self._auth_ns_ip_mapping.keys()) 840 ns_names.sort() 841 for name in ns_names: 842 addrs = list(self._auth_ns_ip_mapping[name]) 843 addrs.sort() 844 d['auth_ns_ip_mapping'][lb2s(name.canonicalize().to_text())] = addrs 845 846 if self.stub: 847 return 848 849 d['queries'] = [] 850 query_keys = list(self.queries.keys()) 851 query_keys.sort() 852 for (qname, rdtype) in query_keys: 853 for query in self.queries[(qname, rdtype)].queries.values(): 854 d['queries'].append(query.serialize(meta_only)) 855 856 def _serialize_dependencies(self, d, meta_only, trace): 857 if self.stub: 858 return 859 860 for cname in self.cname_targets: 861 for target, cname_obj in self.cname_targets[cname].items(): 862 if cname_obj is not None: 863 cname_obj.serialize(d, meta_only, trace=trace + [self]) 864 for signer, signer_obj in self.external_signers.items(): 865 if signer_obj is not None: 866 signer_obj.serialize(d, meta_only, trace=trace + [self]) 867 for target, ns_obj in self.ns_dependencies.items(): 868 if ns_obj is not None: 869 ns_obj.serialize(d, meta_only, trace=trace + [self]) 870 for target, mx_obj in self.mx_targets.items(): 871 if mx_obj is not None: 872 mx_obj.serialize(d, meta_only, trace=trace + [self]) 873 874 @classmethod 875 def deserialize(cls, name, d1, cache=None, **kwargs): 876 if cache is None: 877 cache = {} 878 879 if name in cache: 880 return cache[name] 881 882 name_str = lb2s(name.canonicalize().to_text()) 883 d = d1[name_str] 884 885 analysis_type = analysis_type_codes[d['type']] 886 stub = d['stub'] 887 888 if 'parent' in d: 889 parent_name = dns.name.from_text(d['parent']) 890 parent = cls.deserialize(parent_name, d1, cache=cache, **kwargs) 891 else: 892 parent = None 893 894 if name != dns.name.root and 'dlv_parent' in d: 895 dlv_parent_name = dns.name.from_text(d['dlv_parent']) 896 dlv_parent = cls.deserialize(dlv_parent_name, d1, cache=cache, **kwargs) 897 else: 898 dlv_parent_name = None 899 dlv_parent = None 900 901 if 'nxdomain_ancestor' in d: 902 nxdomain_ancestor_name = dns.name.from_text(d['nxdomain_ancestor']) 903 nxdomain_ancestor = cls.deserialize(nxdomain_ancestor_name, d1, cache=cache, **kwargs) 904 else: 905 nxdomain_ancestor_name = None 906 nxdomain_ancestor = None 907 908 if 'cookie_standin' in d: 909 cookie_standin = binascii.unhexlify(d['cookie_standin']) 910 else: 911 cookie_standin = None 912 if 'cookie_bad' in d: 913 cookie_bad = binascii.unhexlify(d['cookie_bad']) 914 else: 915 cookie_bad = None 916 917 _logger.info('Loading %s' % fmt.humanize_name(name)) 918 919 cache[name] = a = cls(name, stub=stub, analysis_type=analysis_type, cookie_standin=cookie_standin, cookie_bad=cookie_bad, **kwargs) 920 a.parent = parent 921 if dlv_parent is not None: 922 a.dlv_parent = dlv_parent 923 if nxdomain_ancestor is not None: 924 a.nxdomain_ancestor = nxdomain_ancestor 925 a.analysis_start = fmt.str_to_datetime(d['analysis_start']) 926 a.analysis_end = fmt.str_to_datetime(d['analysis_end']) 927 928 if not a.stub: 929 if 'referral_rdtype' in d: 930 a.referral_rdtype = dns.rdatatype.from_text(d['referral_rdtype']) 931 if 'auth_rdtype' in d: 932 a.auth_rdtype = dns.rdatatype.from_text(d['auth_rdtype']) 933 if 'cookie_rdtype' in d: 934 a.cookie_rdtype = dns.rdatatype.from_text(d['cookie_rdtype']) 935 a.explicit_delegation = d['explicit_delegation'] 936 if 'nxdomain_name' in d: 937 a.nxdomain_name = dns.name.from_text(d['nxdomain_name']) 938 a.nxdomain_rdtype = dns.rdatatype.from_text(d['nxdomain_rdtype']) 939 if 'nxrrset_name' in d: 940 a.nxrrset_name = dns.name.from_text(d['nxrrset_name']) 941 a.nxrrset_rdtype = dns.rdatatype.from_text(d['nxrrset_rdtype']) 942 943 a._deserialize_related(d) 944 a._deserialize_dependencies(d1, cache) 945 return a 946 947 def _deserialize_related(self, d): 948 if 'auth_ns_ip_mapping' in d: 949 for target in d['auth_ns_ip_mapping']: 950 self.add_auth_ns_ip_mappings((dns.name.from_text(target), None)) 951 for addr in d['auth_ns_ip_mapping'][target]: 952 self.add_auth_ns_ip_mappings((dns.name.from_text(target), IPAddr(addr))) 953 954 if self.stub: 955 return 956 957 bailiwick_map, default_bailiwick = self.get_bailiwick_mapping() 958 cookie_jar_map, default_cookie_jar = self.get_cookie_jar_mapping() 959 cookie_standin = self.cookie_standin 960 cookie_bad = self.cookie_bad 961 962 query_map = {} 963 #XXX backwards compatibility with previous version 964 if isinstance(d['queries'], list): 965 for query in d['queries']: 966 key = (dns.name.from_text(query['qname']), dns.rdatatype.from_text(query['qtype'])) 967 if key not in query_map: 968 query_map[key] = [] 969 query_map[key].append(query) 970 else: 971 for query_str in d['queries']: 972 vals = query_str.split('/') 973 qname = dns.name.from_text('/'.join(vals[:-2])) 974 rdtype = dns.rdatatype.from_text(vals[-1]) 975 key = (qname, rdtype) 976 if key not in query_map: 977 query_map[key] = [] 978 for query in d['queries'][query_str]: 979 query_map[key].append(query) 980 981 # Import the following first, in this order: 982 # - Queries used to detect delegation (NS and referral_rdtype) 983 # - Queries used to detect NS records from authority section (auth_rdtype) 984 # - Queries used to detect server cookies (cookie_rdtype) 985 delegation_types = OrderedDict(((dns.rdatatype.NS, None),)) 986 if self.referral_rdtype is not None: 987 delegation_types[self.referral_rdtype] = None 988 if self.auth_rdtype is not None: 989 delegation_types[self.auth_rdtype] = None 990 if self.cookie_rdtype is not None: 991 delegation_types[self.cookie_rdtype] = None 992 for rdtype in delegation_types: 993 # if the query has already been imported, then 994 # don't re-import 995 if (self.name, rdtype) in self.queries: 996 continue 997 key = (self.name, rdtype) 998 if key in query_map: 999 _logger.debug('Importing %s/%s...' % (fmt.humanize_name(self.name), dns.rdatatype.to_text(rdtype))) 1000 for query in query_map[key]: 1001 detect_ns = rdtype in (dns.rdatatype.NS, self.referral_rdtype, self.auth_rdtype) 1002 detect_cookies = rdtype == self.cookie_rdtype 1003 self.add_query(Q.DNSQuery.deserialize(query, bailiwick_map, default_bailiwick, cookie_jar_map, default_cookie_jar, cookie_standin, cookie_bad), detect_ns, detect_cookies) 1004 1005 # set the NS dependencies for the name 1006 if self.is_zone(): 1007 self.set_ns_dependencies() 1008 1009 for key in query_map: 1010 qname, rdtype = key 1011 # if the query has already been imported, then 1012 # don't re-import 1013 if (qname, rdtype) in self.queries: 1014 continue 1015 if qname == self.name and rdtype in delegation_types: 1016 continue 1017 if (qname, rdtype) == (self.nxdomain_name, self.nxdomain_rdtype): 1018 extra = ' (NXDOMAIN)' 1019 elif (qname, rdtype) == (self.nxrrset_name, self.nxrrset_rdtype): 1020 extra = ' (NODATA)' 1021 else: 1022 extra = '' 1023 _logger.debug('Importing %s/%s%s...' % (fmt.humanize_name(qname), dns.rdatatype.to_text(rdtype), extra)) 1024 for query in query_map[key]: 1025 self.add_query(Q.DNSQuery.deserialize(query, bailiwick_map, default_bailiwick, cookie_jar_map, default_cookie_jar, cookie_standin, cookie_bad), False, False) 1026 1027 def _deserialize_dependencies(self, d, cache): 1028 if self.stub: 1029 return 1030 1031 for cname in self.cname_targets: 1032 for target in self.cname_targets[cname]: 1033 self.cname_targets[cname][target] = self.__class__.deserialize(target, d, cache=cache) 1034 for signer in self.external_signers: 1035 self.external_signers[signer] = self.__class__.deserialize(signer, d, cache=cache) 1036 1037 # these two are optional 1038 for target in self.ns_dependencies: 1039 if lb2s(target.canonicalize().to_text()) in d: 1040 self.ns_dependencies[target] = self.__class__.deserialize(target, d, cache=cache) 1041 for target in self.mx_targets: 1042 if lb2s(target.canonicalize().to_text()) in d: 1043 self.mx_targets[target] = self.__class__.deserialize(target, d, cache=cache) 1044 1045class ActiveDomainNameAnalysis(OnlineDomainNameAnalysis): 1046 def __init__(self, *args, **kwargs): 1047 super(ActiveDomainNameAnalysis, self).__init__(*args, **kwargs) 1048 self.complete = threading.Event() 1049 1050class Analyst(object): 1051 analysis_model = ActiveDomainNameAnalysis 1052 _simple_query = Q.SimpleDNSQuery 1053 _quick_query = Q.QuickDNSSECQuery 1054 _diagnostic_query = Q.DiagnosticQuery 1055 _tcp_diagnostic_query = Q.TCPDiagnosticQuery 1056 _pmtu_diagnostic_query = Q.PMTUDiagnosticQuery 1057 _truncation_diagnostic_query = Q.TruncationDiagnosticQuery 1058 _edns_version_diagnostic_query = Q.EDNSVersionDiagnosticQuery 1059 _edns_flag_diagnostic_query = Q.EDNSFlagDiagnosticQuery 1060 _edns_opt_diagnostic_query = Q.EDNSOptDiagnosticQuery 1061 1062 default_th_factory = transport.DNSQueryTransportHandlerDNSFactory() 1063 1064 qname_only = True 1065 analysis_type = ANALYSIS_TYPE_AUTHORITATIVE 1066 1067 clone_attrnames = ['rdclass', 'dlv_domain', 'try_ipv4', 'try_ipv6', 'client_ipv4', 'client_ipv6', 'query_class_mixin', 'logger', 'ceiling', 'edns_diagnostics', 'follow_ns', 'explicit_delegations', 'stop_at_explicit', 'odd_ports', 'analysis_cache', 'cache_level', 'analysis_cache_lock', 'transport_manager', 'th_factories', 'resolver'] 1068 1069 def __init__(self, name, rdclass=dns.rdataclass.IN, dlv_domain=None, try_ipv4=True, try_ipv6=True, client_ipv4=None, client_ipv6=None, query_class_mixin=None, logger=_logger, ceiling=None, edns_diagnostics=False, 1070 follow_ns=False, follow_mx=False, trace=None, explicit_delegations=None, stop_at_explicit=None, odd_ports=None, extra_rdtypes=None, explicit_only=False, 1071 analysis_cache=None, cache_level=None, analysis_cache_lock=None, th_factories=None, transport_manager=None, resolver=None): 1072 1073 self.simple_query = self._simple_query 1074 self.quick_query = self._quick_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1075 self.diagnostic_query_no_server_cookie = self._diagnostic_query.add_mixin(query_class_mixin) 1076 self.diagnostic_query_bad_server_cookie = self._diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_BAD) 1077 self.diagnostic_query = self._diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1078 self.tcp_diagnostic_query = self._tcp_diagnostic_query.add_mixin(query_class_mixin).remove_cookie_option() 1079 self.pmtu_diagnostic_query = self._pmtu_diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1080 self.truncation_diagnostic_query = self._truncation_diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1081 self.edns_version_diagnostic_query = self._edns_version_diagnostic_query 1082 self.edns_flag_diagnostic_query = self._edns_flag_diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1083 self.edns_opt_diagnostic_query = self._edns_opt_diagnostic_query.add_mixin(query_class_mixin).add_server_cookie(COOKIE_STANDIN) 1084 1085 self.query_class_mixin = query_class_mixin 1086 1087 if transport_manager is None: 1088 self.transport_manager = transport.DNSQueryTransportManager() 1089 else: 1090 self.transport_manager = transport_manager 1091 1092 if th_factories is None: 1093 self.th_factories = (self.default_th_factory,) 1094 else: 1095 self.th_factories = th_factories 1096 self.allow_loopback_query = not bool([x for x in self.th_factories if not x.cls.allow_loopback_query]) 1097 self.allow_private_query = not bool([x for x in self.th_factories if not x.cls.allow_private_query]) 1098 1099 self.name = name 1100 self.rdclass = rdclass 1101 self.dlv_domain = dlv_domain 1102 1103 if explicit_delegations is None: 1104 self.explicit_delegations = {} 1105 else: 1106 self.explicit_delegations = explicit_delegations 1107 1108 if stop_at_explicit is None: 1109 self.stop_at_explicit = {} 1110 else: 1111 self.stop_at_explicit = stop_at_explicit 1112 1113 if odd_ports is None: 1114 self.odd_ports = {} 1115 else: 1116 self.odd_ports = odd_ports 1117 1118 if resolver is None: 1119 resolver = self._get_resolver() 1120 self.resolver = resolver 1121 1122 self.ceiling = ceiling 1123 1124 # if an ancestor of the name (not wildcard!) is explicitly delegated, 1125 # then set the ceiling to the lowest ancestor with an explicit 1126 # delegation. 1127 c = self.name 1128 try: 1129 while True: 1130 if (c, dns.rdatatype.NS) in self.explicit_delegations and self.stop_at_explicit[c]: 1131 break 1132 c = c.parent() 1133 except dns.name.NoParent: 1134 # if no ancestors are explicitly delegated, then don't modify the 1135 # ceiling 1136 pass 1137 else: 1138 # if ceiling was not specified, if there is a ceiling, but the name 1139 # is not a subdomain of the ceiling, or if the lowest explicit 1140 # delegation is a subdomain of the specified ceiling, then replace 1141 # it with the modified lowest ancestor that is explicitly 1142 # delegated. 1143 if ceiling is None or not self.name.is_subdomain(ceiling) or c.is_subdomain(ceiling): 1144 ceiling = c 1145 1146 self.local_ceiling = self._detect_ceiling(ceiling)[0] 1147 1148 self.try_ipv4 = try_ipv4 1149 self.try_ipv6 = try_ipv6 1150 self.client_ipv4 = client_ipv4 1151 self.client_ipv6 = client_ipv6 1152 1153 self.logger = logger 1154 1155 self.edns_diagnostics = edns_diagnostics 1156 1157 cookie_opt = self.diagnostic_query.get_cookie_opt() 1158 self.dns_cookies = cookie_opt is not None 1159 1160 self.follow_ns = follow_ns 1161 self.follow_mx = follow_mx 1162 1163 if trace is None: 1164 self.trace = [] 1165 else: 1166 self.trace = trace 1167 1168 assert not explicit_only or extra_rdtypes is not None or self._force_dnskey_query(name), 'If explicit_only is specified, then extra_rdtypes must be specified or force_dnskey must be true.' 1169 1170 self.extra_rdtypes = extra_rdtypes 1171 self.explicit_only = explicit_only 1172 if analysis_cache is None: 1173 self.analysis_cache = {} 1174 else: 1175 self.analysis_cache = analysis_cache 1176 self.cache_level = cache_level 1177 if analysis_cache_lock is None: 1178 self.analysis_cache_lock = threading.Lock() 1179 else: 1180 self.analysis_cache_lock = analysis_cache_lock 1181 self._detect_cname_chain() 1182 1183 def _get_resolver(self): 1184 hints = util.get_root_hints() 1185 for key in self.explicit_delegations: 1186 hints[key] = self.explicit_delegations[key] 1187 return Resolver.FullResolver(hints, query_cls=(self.quick_query, self.diagnostic_query), odd_ports=self.odd_ports, cookie_standin=COOKIE_STANDIN, transport_manager=self.transport_manager) 1188 1189 def _detect_cname_chain(self): 1190 self._cname_chain = [] 1191 1192 if self.dlv_domain == self.name: 1193 return 1194 1195 if len(self.name) < 3: 1196 return 1197 1198 try: 1199 rdtype = self._rdtypes_to_query(self.name)[0] 1200 except IndexError: 1201 rdtype = dns.rdatatype.A 1202 1203 try: 1204 ans = self.resolver.query_for_answer(self.name, rdtype, self.rdclass, allow_noanswer=True) 1205 except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN, dns.exception.DNSException): 1206 return 1207 1208 cname = self.name 1209 for i in range(Resolver.MAX_CNAME_REDIRECTION): 1210 try: 1211 cname = ans.response.find_rrset(ans.response.answer, cname, self.rdclass, dns.rdatatype.CNAME)[0].target 1212 self._cname_chain.append(cname) 1213 except KeyError: 1214 return 1215 1216 def _detect_ceiling(self, ceiling): 1217 if ceiling == dns.name.root or ceiling is None: 1218 1219 # make sure we can communicate with a resolver; we must be able to 1220 # resolve the root 1221 server, response = self.resolver.query(dns.name.root, dns.rdatatype.NS) 1222 if server is None: 1223 raise NetworkConnectivityException('No network connectivity available!') 1224 1225 return ceiling, None 1226 1227 # if there is a ceiling, but the name is not a subdomain 1228 # of the ceiling, then use the name itself as a base 1229 if not self.name.is_subdomain(ceiling): 1230 ceiling = self.name 1231 1232 try: 1233 ans = self.resolver.query_for_answer(ceiling, dns.rdatatype.NS, self.rdclass) 1234 try: 1235 ans.response.find_rrset(ans.response.answer, ceiling, self.rdclass, dns.rdatatype.NS) 1236 return ceiling, False 1237 except KeyError: 1238 pass 1239 except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): 1240 pass 1241 except dns.exception.DNSException as e: 1242 parent_ceiling, fail = self._detect_ceiling(ceiling.parent()) 1243 if fail: 1244 return parent_ceiling, True 1245 else: 1246 return ceiling, True 1247 return self._detect_ceiling(ceiling.parent()) 1248 1249 def _get_servers_from_hints(self, name, hints): 1250 servers = set() 1251 for rdata in hints[(name, dns.rdatatype.NS)]: 1252 for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA): 1253 if (rdata.target, rdtype) in hints: 1254 servers.update([IPAddr(r.address) for r in hints[(rdata.target, rdtype)]]) 1255 return servers 1256 1257 def _root_servers(self, proto=None): 1258 key = None 1259 if (dns.name.root, dns.rdatatype.NS) in self.explicit_delegations: 1260 key = dns.name.root 1261 elif (WILDCARD_EXPLICIT_DELEGATION, dns.rdatatype.NS) in self.explicit_delegations: 1262 key = WILDCARD_EXPLICIT_DELEGATION 1263 if key is not None: 1264 servers = self._get_servers_from_hints(key, self.explicit_delegations) 1265 else: 1266 servers = self._get_servers_from_hints(dns.name.root, util.get_root_hints()) 1267 if proto == 4: 1268 servers = set([x for x in servers if x.version == 4]) 1269 elif proto == 6: 1270 servers = set([x for x in servers if x.version == 6]) 1271 return servers 1272 1273 def _is_referral_of_type(self, rdtype): 1274 '''Return True if analysis of this name was invoked as a dependency (specified by 1275 rdtype) for another name; False otherwise. Examples are CNAME, NS, MX.''' 1276 1277 try: 1278 return self.trace[-1][1] == rdtype 1279 except IndexError: 1280 return False 1281 1282 def _original_alias_of_cname(self): 1283 name = self.name 1284 for i in range(len(self.trace) - 1, -1, -1): 1285 if self.trace[i][1] != dns.rdatatype.CNAME: 1286 return name 1287 name = self.trace[i][0].name 1288 return name 1289 1290 def _force_dnskey_query(self, name): 1291 return name == self.name and self._is_referral_of_type(dns.rdatatype.RRSIG) 1292 1293 def _rdtypes_to_query(self, name): 1294 orig_name = self._original_alias_of_cname() 1295 1296 rdtypes = [] 1297 if self.explicit_only: 1298 if self.name == name: 1299 if self.extra_rdtypes is not None: 1300 rdtypes.extend(self.extra_rdtypes) 1301 else: 1302 if name in self._cname_chain: 1303 rdtypes.extend(self._rdtypes_to_query(self.name)) 1304 else: 1305 rdtypes.extend(self._rdtypes_to_query_for_name(name)) 1306 if self.name == name: 1307 if orig_name != name: 1308 rdtypes.extend(self._rdtypes_to_query_for_name(orig_name)) 1309 1310 if self.extra_rdtypes is not None: 1311 rdtypes.extend(self.extra_rdtypes) 1312 1313 else: 1314 if name in self._cname_chain: 1315 rdtypes.extend(self._rdtypes_to_query(self.name)) 1316 1317 if self._ask_tlsa_queries(self.name) and len(name) == len(self.name) - 2: 1318 rdtypes.extend([dns.rdatatype.A, dns.rdatatype.AAAA]) 1319 1320 # remove duplicates 1321 rdtypes = list(OrderedDict.fromkeys(rdtypes)) 1322 1323 return rdtypes 1324 1325 def _rdtypes_to_query_for_name(self, name): 1326 rdtypes = [] 1327 1328 if self._ask_ptr_queries(name): 1329 rdtypes.append(dns.rdatatype.PTR) 1330 elif self._ask_naptr_queries(name): 1331 rdtypes.append(dns.rdatatype.NAPTR) 1332 elif self._ask_tlsa_queries(name): 1333 rdtypes.append(dns.rdatatype.TLSA) 1334 elif self._ask_srv_queries(name): 1335 rdtypes.append(dns.rdatatype.SRV) 1336 elif self._is_dkim(name): 1337 rdtypes.append(dns.rdatatype.TXT) 1338 elif name.is_subdomain(ARPA_NAME): 1339 pass 1340 elif PROTO_LABEL_RE.search(lb2s(name[0])): 1341 pass 1342 elif self._is_sld_or_lower(name): 1343 rdtypes.extend([dns.rdatatype.A, dns.rdatatype.AAAA]) 1344 1345 return rdtypes 1346 1347 def _ask_ptr_queries(self, name): 1348 '''Return True if PTR queries should be asked for this name, as guessed 1349 by the nature of the name, based on its length and its presence in the 1350 in-addr.arpa or ip6.arpa trees.''' 1351 1352 # if this name is in the ip6.arpa tree and is the length of a full 1353 # reverse IPv6 address, then return True 1354 if name.is_subdomain(IP6_ARPA_NAME) and len(name) == 35: 1355 return True 1356 1357 # if this name is in the in-addr.arpa tree and is the length of a full 1358 # reverse IPv4 address, then return True 1359 if name.is_subdomain(INADDR_ARPA_NAME) and len(name) == 7: 1360 return True 1361 1362 return False 1363 1364 def _ask_naptr_queries(self, name): 1365 '''Return True if NAPTR queries should be asked for this name, as guessed by 1366 the nature of the name, based on its presence in the e164.arpa tree.''' 1367 1368 if name.is_subdomain(E164_ARPA_NAME) and name != E164_ARPA_NAME: 1369 return True 1370 1371 return False 1372 1373 def _ask_tlsa_queries(self, name): 1374 '''Return True if TLSA queries should be asked for this name, which is 1375 determined by examining the structure of the name for _<port>._<proto> 1376 format.''' 1377 1378 if len(name) > 2 and DANE_PORT_RE.search(lb2s(name[0])) is not None and PROTO_LABEL_RE.search(lb2s(name[1])) is not None: 1379 return True 1380 1381 return False 1382 1383 def _ask_srv_queries(self, name): 1384 '''Return True if SRV queries should be asked for this name, which is 1385 determined by examining the structure of the name for common 1386 service-related names.''' 1387 1388 if len(name) > 2 and SRV_PORT_RE.search(lb2s(name[0])) is not None and PROTO_LABEL_RE.search(lb2s(name[1])) is not None: 1389 return True 1390 1391 return False 1392 1393 def _is_dkim(self, name): 1394 '''Return True if the name is a DKIM name.''' 1395 1396 return '_domainkey' in name 1397 1398 def _is_sld_or_lower(self, name): 1399 '''Return True if the name is an SLD or lower.''' 1400 1401 return len(name) >= 3 1402 1403 def _ask_non_delegation_queries(self, name): 1404 '''Return True if non-delegation-related queries should be asked for 1405 name.''' 1406 1407 if self.qname_only and not \ 1408 (name == self.name or \ 1409 name in self._cname_chain or \ 1410 (self._ask_tlsa_queries(self.name) and len(name) == len(self.name) - 2)): 1411 return False 1412 if self.dlv_domain == self.name: 1413 return False 1414 return True 1415 1416 def _add_query(self, name_obj, query, detect_ns, detect_cookies, iterative=False): 1417 # if this query is empty (i.e., nothing was actually asked, e.g., due 1418 # to client-side connectivity failure), then raise a connectivity 1419 # failure 1420 if not query.responses and not iterative: 1421 self._raise_connectivity_error_local(query.servers) 1422 1423 name_obj.add_query(query, detect_ns, detect_cookies) 1424 1425 def _filter_servers_network(self, servers): 1426 if not self.try_ipv6: 1427 servers = [x for x in servers if x.version != 6] 1428 if not self.try_ipv4: 1429 servers = [x for x in servers if x.version != 4] 1430 return servers 1431 1432 def _filter_servers_locality(self, servers): 1433 if not self.allow_loopback_query: 1434 servers = [x for x in servers if not LOOPBACK_IPV4_RE.match(x) and not x == LOOPBACK_IPV6] 1435 if not self.allow_private_query: 1436 servers = [x for x in servers if not RFC_1918_RE.match(x) and not LINK_LOCAL_RE.match(x) and not UNIQ_LOCAL_RE.match(x)] 1437 return [x for x in servers if ZERO_SLASH8_RE.search(x) is None] 1438 1439 def _filter_servers(self, servers, no_raise=False): 1440 filtered_servers = self._filter_servers_network(servers) 1441 if servers and not filtered_servers and not no_raise: 1442 self._raise_connectivity_error_remote() 1443 return self._filter_servers_locality(filtered_servers) 1444 1445 def _get_name_for_analysis(self, name, stub=False, lock=True): 1446 with self.analysis_cache_lock: 1447 try: 1448 name_obj = self.analysis_cache[name] 1449 except KeyError: 1450 if lock: 1451 name_obj = self.analysis_cache[name] = self.analysis_model(name, stub=stub, analysis_type=self.analysis_type, cookie_standin=COOKIE_STANDIN) 1452 return name_obj 1453 # if not locking, then return None 1454 else: 1455 return None 1456 1457 # if there is a complete event, then wait on it 1458 if hasattr(name_obj, 'complete'): 1459 name_obj.complete.wait() 1460 # loop and wait for analysis to be completed 1461 while name_obj.analysis_end is None: 1462 time.sleep(1) 1463 name_obj = self.analysis_cache[name] 1464 1465 # check if this analysis needs to be re-done 1466 if self.name == name: 1467 redo_analysis = False 1468 # re-do analysis if force_dnskey is True and dnskey hasn't been queried 1469 if self._force_dnskey_query(self.name) and (self.name, dns.rdatatype.DNSKEY) not in name_obj.queries: 1470 redo_analysis = True 1471 1472 # re-do analysis if there were no queries (previously an 1473 # "empty" non-terminal) but now it is the name in question 1474 if not name_obj.queries: 1475 redo_analysis = True 1476 1477 # re-do analysis if this name is referenced by an alias 1478 # and previously the necessary queries weren't asked 1479 if self._is_referral_of_type(dns.rdatatype.CNAME): 1480 rdtypes_to_query = set(self._rdtypes_to_query(name)) 1481 rdtypes_queried = set([r for n,r in name_obj.queries if n == name_obj.name]) 1482 if rdtypes_to_query.difference(rdtypes_queried): 1483 redo_analysis = True 1484 1485 # if the previous analysis was a stub, but now we want the 1486 # whole analysis 1487 if name_obj.stub and not stub: 1488 redo_analysis = True 1489 1490 if redo_analysis: 1491 with self.analysis_cache_lock: 1492 if name_obj.uuid == self.analysis_cache[name].uuid: 1493 del self.analysis_cache[name] 1494 return self._get_name_for_analysis(name, stub, lock) 1495 1496 return name_obj 1497 1498 def analyze_async(self, callback=None, exc_callback=None): 1499 def _analyze(): 1500 try: 1501 result = self.analyze() 1502 if callback is not None: 1503 callback(result) 1504 except: 1505 if exc_callback is not None: 1506 exc_callback(sys.exc_info()) 1507 t = threading.Thread(target=_analyze) 1508 t.start() 1509 return t 1510 1511 def analyze(self): 1512 self._analyze_dlv() 1513 return self._analyze(self.name) 1514 1515 def _analyze_dlv(self): 1516 if self.dlv_domain is not None and self.dlv_domain != self.name and self.dlv_domain not in self.analysis_cache: 1517 kwargs = dict([(n, getattr(self, n)) for n in self.clone_attrnames]) 1518 kwargs['ceiling'] = self.dlv_domain 1519 a = self.__class__(self.dlv_domain, **kwargs) 1520 a.analyze() 1521 1522 def _finalize_analysis_proper(self, name_obj): 1523 pass 1524 1525 def _finalize_analysis_all(self, name_obj): 1526 pass 1527 1528 def _cleanup_analysis_proper(self, name_obj): 1529 if hasattr(name_obj, 'complete'): 1530 name_obj.complete.set() 1531 1532 def _cleanup_analysis_all(self, name_obj): 1533 if self.cache_level is not None and len(name_obj.name) > self.cache_level: 1534 del self.analysis_cache[name_obj.name] 1535 1536 def _handle_explicit_delegations(self, name_obj): 1537 key = None 1538 if (name_obj.name, dns.rdatatype.NS) in self.explicit_delegations: 1539 key = name_obj.name 1540 elif (WILDCARD_EXPLICIT_DELEGATION, dns.rdatatype.NS) in self.explicit_delegations: 1541 key = WILDCARD_EXPLICIT_DELEGATION 1542 if key is not None: 1543 for ns_rdata in self.explicit_delegations[(key, dns.rdatatype.NS)]: 1544 for a_rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA): 1545 if (ns_rdata.target, a_rdtype) in self.explicit_delegations: 1546 name_obj.add_auth_ns_ip_mappings(*[(ns_rdata.target, IPAddr(r.address)) for r in self.explicit_delegations[(ns_rdata.target, a_rdtype)]]) 1547 name_obj.explicit_delegation = True 1548 1549 def _analyze_stub(self, name): 1550 name_obj = self._get_name_for_analysis(name, stub=True) 1551 if name_obj.analysis_end is not None: 1552 return name_obj 1553 1554 try: 1555 self.logger.info('Analyzing %s (stub)' % fmt.humanize_name(name)) 1556 1557 name_obj.analysis_start = datetime.datetime.now(fmt.utc).replace(microsecond=0) 1558 1559 self._handle_explicit_delegations(name_obj) 1560 if not name_obj.explicit_delegation: 1561 try: 1562 ans = self.resolver.query_for_answer(name, dns.rdatatype.NS, self.rdclass) 1563 1564 # resolve every name in the NS RRset 1565 query_tuples = [] 1566 for rr in ans.rrset: 1567 query_tuples.extend([(rr.target, dns.rdatatype.A, self.rdclass), (rr.target, dns.rdatatype.AAAA, self.rdclass)]) 1568 answer_map = self.resolver.query_multiple_for_answer(*query_tuples) 1569 for query_tuple in answer_map: 1570 a = answer_map[query_tuple] 1571 if isinstance(a, Resolver.DNSAnswer): 1572 for a_rr in a.rrset: 1573 name_obj.add_auth_ns_ip_mappings((query_tuple[0], IPAddr(a_rr.to_text()))) 1574 except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): 1575 name_obj.parent = self._analyze_stub(name.parent()).zone 1576 except dns.exception.DNSException: 1577 if name == dns.name.root: 1578 raise NetworkConnectivityException('DNS resolver is unresponsive!') 1579 name_obj.parent = self._analyze_stub(name.parent()).zone 1580 1581 name_obj.analysis_end = datetime.datetime.now(fmt.utc).replace(microsecond=0) 1582 1583 self._finalize_analysis_proper(name_obj) 1584 self._finalize_analysis_all(name_obj) 1585 finally: 1586 self._cleanup_analysis_proper(name_obj) 1587 self._cleanup_analysis_all(name_obj) 1588 1589 return name_obj 1590 1591 def _analyze_ancestry(self, name): 1592 # only analyze the parent if the name is not root and if there is no 1593 # ceiling or the name is a subdomain of the ceiling 1594 if name == dns.name.root: 1595 parent_obj = None 1596 elif (name, dns.rdatatype.NS) in self.explicit_delegations and self.stop_at_explicit[name]: 1597 parent_obj = None 1598 elif self.local_ceiling is not None and self.local_ceiling.is_subdomain(name): 1599 parent_obj = self._analyze_stub(name.parent()) 1600 else: 1601 parent_obj = self._analyze(name.parent()) 1602 1603 if parent_obj is not None: 1604 nxdomain_ancestor = parent_obj.nxdomain_ancestor 1605 if nxdomain_ancestor is None and \ 1606 parent_obj.referral_rdtype is not None and \ 1607 parent_obj.queries[(parent_obj.name, parent_obj.referral_rdtype)].is_nxdomain_all(): 1608 nxdomain_ancestor = parent_obj 1609 1610 # for zones other than the root assign parent_obj to the zone apex, 1611 # rather than the simply the domain formed by dropping its lower 1612 # leftmost label 1613 parent_obj = parent_obj.zone 1614 1615 else: 1616 nxdomain_ancestor = None 1617 1618 # retrieve the dlv 1619 if self.dlv_domain is not None and self.name != self.dlv_domain: 1620 dlv_parent_obj = self.analysis_cache[self.dlv_domain] 1621 else: 1622 dlv_parent_obj = None 1623 1624 return parent_obj, dlv_parent_obj, nxdomain_ancestor 1625 1626 def _analyze(self, name): 1627 '''Analyze a DNS name to learn about its health using introspective 1628 queries.''' 1629 1630 # determine immediately if we need to do anything 1631 name_obj = self._get_name_for_analysis(name, lock=False) 1632 if name_obj is not None and name_obj.analysis_end is not None: 1633 return name_obj 1634 1635 parent_obj, dlv_parent_obj, nxdomain_ancestor = \ 1636 self._analyze_ancestry(name) 1637 1638 # get or create the name 1639 name_obj = self._get_name_for_analysis(name) 1640 if name_obj.analysis_end is not None: 1641 return name_obj 1642 1643 try: 1644 try: 1645 name_obj.parent = parent_obj 1646 name_obj.dlv_parent = dlv_parent_obj 1647 name_obj.nxdomain_ancestor = nxdomain_ancestor 1648 1649 name_obj.analysis_start = datetime.datetime.now(fmt.utc).replace(microsecond=0) 1650 1651 # perform the actual analysis on this name 1652 self._analyze_name(name_obj) 1653 1654 # set analysis_end 1655 name_obj.analysis_end = datetime.datetime.now(fmt.utc).replace(microsecond=0) 1656 1657 # remove dlv_parent if there are no DLV queries associated with it 1658 if name_obj.dlv_parent is not None and \ 1659 (name_obj.dlv_name, dns.rdatatype.DLV) not in name_obj.queries: 1660 name_obj.dlv_parent = None 1661 1662 # sanity check - if we weren't able to get responses from any 1663 # servers, check that we actually have connectivity 1664 self._check_connectivity(name_obj) 1665 1666 self._finalize_analysis_proper(name_obj) 1667 finally: 1668 self._cleanup_analysis_proper(name_obj) 1669 1670 # analyze dependencies 1671 self._analyze_dependencies(name_obj) 1672 1673 self._finalize_analysis_all(name_obj) 1674 finally: 1675 self._cleanup_analysis_all(name_obj) 1676 1677 return name_obj 1678 1679 def _analyze_name(self, name_obj): 1680 self.logger.info('Analyzing %s' % fmt.humanize_name(name_obj.name)) 1681 1682 self._handle_explicit_delegations(name_obj) 1683 if not name_obj.explicit_delegation: 1684 # analyze delegation, and return if name doesn't exist, unless 1685 # explicit_only was specified 1686 yxdomain = self._analyze_delegation(name_obj) 1687 if not yxdomain and not self.explicit_only: 1688 return 1689 1690 # set the NS dependencies for the name 1691 if name_obj.is_zone(): 1692 name_obj.set_ns_dependencies() 1693 1694 self._analyze_queries(name_obj) 1695 1696 def _analyze_queries(self, name_obj): 1697 bailiwick = name_obj.zone.name 1698 1699 servers = name_obj.zone.get_auth_or_designated_servers() 1700 # if we haven't queried any of the designated servers, then query them 1701 # all 1702 if not name_obj.zone._all_servers_queried.intersection(servers): 1703 pass 1704 # otherwise, just query the ones that were responsive 1705 else: 1706 servers = name_obj.zone.get_responsive_auth_or_designated_servers() 1707 1708 odd_ports = dict([(s, self.odd_ports[(n, s)]) for n, s in self.odd_ports if n in (name_obj.zone.name, WILDCARD_EXPLICIT_DELEGATION)]) 1709 cookie_jar = name_obj.zone.cookie_jar 1710 1711 servers = self._filter_servers(servers) 1712 exclude_no_answer = set() 1713 queries = {} 1714 1715 # if there are responsive servers to query... 1716 if servers: 1717 1718 # If 1) this is a zone, 2) DNS cookies are supported, and 1719 # 3) cookies have not yet been elicited, then issue queries now to 1720 # elicit DNS cookies. 1721 if name_obj.is_zone() and self.dns_cookies and name_obj.cookie_rdtype is None: 1722 self.logger.debug('Querying for DNS server cookies %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(dns.rdatatype.SOA))) 1723 query = self.diagnostic_query_no_server_cookie(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports) 1724 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 1725 self._add_query(name_obj, query, False, True) 1726 1727 name_obj.cookie_rdtype = dns.rdatatype.SOA 1728 1729 # queries specific to zones for which non-delegation-related 1730 # queries are being issued 1731 if name_obj.is_zone() and self._ask_non_delegation_queries(name_obj.name) and not self.explicit_only: 1732 1733 # EDNS diagnostic queries 1734 if self.edns_diagnostics: 1735 self.logger.debug('Preparing EDNS diagnostic queries %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(dns.rdatatype.SOA))) 1736 queries[(name_obj.name, -(dns.rdatatype.SOA+100))] = self.edns_version_diagnostic_query(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports) 1737 queries[(name_obj.name, -(dns.rdatatype.SOA+101))] = self.edns_opt_diagnostic_query(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1738 queries[(name_obj.name, -(dns.rdatatype.SOA+102))] = self.edns_flag_diagnostic_query(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1739 1740 # Query with a mixed-case name for 0x20, if possible 1741 mixed_case_name = self._mix_case(name_obj.name) 1742 if mixed_case_name is not None: 1743 self.logger.debug('Preparing 0x20 query %s/%s...' % (fmt.humanize_name(mixed_case_name, canonicalize=False), dns.rdatatype.to_text(dns.rdatatype.SOA))) 1744 queries[(name_obj.name, -(dns.rdatatype.SOA+103))] = self.diagnostic_query(mixed_case_name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1745 1746 # DNS cookies diagnostic queries 1747 if self.dns_cookies: 1748 self.logger.debug('Preparing DNS cookie diagnostic query %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(dns.rdatatype.SOA))) 1749 queries[(name_obj.name, -(dns.rdatatype.SOA+104))] = self.diagnostic_query_bad_server_cookie(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_bad=COOKIE_BAD) 1750 1751 # negative queries for all zones 1752 self._set_negative_queries(name_obj) 1753 if name_obj.nxdomain_name is not None: 1754 self.logger.debug('Preparing query %s/%s (NXDOMAIN)...' % (fmt.humanize_name(name_obj.nxdomain_name), dns.rdatatype.to_text(name_obj.nxdomain_rdtype))) 1755 queries[(name_obj.nxdomain_name, name_obj.nxdomain_rdtype)] = self.diagnostic_query(name_obj.nxdomain_name, name_obj.nxdomain_rdtype, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1756 if name_obj.nxrrset_name is not None: 1757 self.logger.debug('Preparing query %s/%s (NODATA)...' % (fmt.humanize_name(name_obj.nxrrset_name), dns.rdatatype.to_text(name_obj.nxrrset_rdtype))) 1758 queries[(name_obj.nxrrset_name, name_obj.nxrrset_rdtype)] = self.diagnostic_query(name_obj.nxrrset_name, name_obj.nxrrset_rdtype, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1759 1760 # if the name is SLD or lower, then ask MX and TXT 1761 if self._is_sld_or_lower(name_obj.name): 1762 self.logger.debug('Preparing query %s/MX...' % fmt.humanize_name(name_obj.name)) 1763 # note that we use a PMTU diagnostic query here, to simultaneously test PMTU 1764 queries[(name_obj.name, dns.rdatatype.MX)] = self.pmtu_diagnostic_query(name_obj.name, dns.rdatatype.MX, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1765 # we also do a query with small UDP payload to elicit and test a truncated response 1766 queries[(name_obj.name, -dns.rdatatype.MX)] = self.truncation_diagnostic_query(name_obj.name, dns.rdatatype.MX, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1767 1768 self.logger.debug('Preparing query %s/TXT...' % fmt.humanize_name(name_obj.name)) 1769 queries[(name_obj.name, dns.rdatatype.TXT)] = self.diagnostic_query(name_obj.name, dns.rdatatype.TXT, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1770 1771 # for zones and for (non-zone) names which have DNSKEYs referenced 1772 if name_obj.is_zone() or self._force_dnskey_query(name_obj.name): 1773 1774 # if there are responsive servers to query... 1775 if servers: 1776 if self._ask_non_delegation_queries(name_obj.name) and not self.explicit_only: 1777 self.logger.debug('Preparing query %s/SOA...' % fmt.humanize_name(name_obj.name)) 1778 queries[(name_obj.name, dns.rdatatype.SOA)] = self.diagnostic_query(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1779 1780 if name_obj.is_zone(): 1781 # for zones we also use a TCP diagnostic query here, to simultaneously test TCP connectivity 1782 queries[(name_obj.name, -dns.rdatatype.SOA)] = self.tcp_diagnostic_query(name_obj.name, dns.rdatatype.SOA, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1783 else: 1784 # for non-zones we don't need to keep the (UDP) SOA query, if there is no positive response 1785 exclude_no_answer.add((name_obj.name, dns.rdatatype.SOA)) 1786 1787 self.logger.debug('Preparing query %s/DNSKEY...' % fmt.humanize_name(name_obj.name)) 1788 # note that we use a PMTU diagnostic query here, to simultaneously test PMTU 1789 queries[(name_obj.name, dns.rdatatype.DNSKEY)] = self.pmtu_diagnostic_query(name_obj.name, dns.rdatatype.DNSKEY, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1790 1791 # we also do a query with small UDP payload to elicit and test a truncated response 1792 queries[(name_obj.name, -dns.rdatatype.DNSKEY)] = self.truncation_diagnostic_query(name_obj.name, dns.rdatatype.DNSKEY, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1793 1794 # query for DS/DLV 1795 if name_obj.parent is not None: 1796 parent_servers = name_obj.zone.parent.get_auth_or_designated_servers() 1797 # if we haven't queried any of the servers designated for the 1798 # parent, then query them all 1799 if not name_obj.parent._all_servers_queried.intersection(parent_servers): 1800 pass 1801 # otherwise, just query the ones that were responsive 1802 else: 1803 parent_servers = name_obj.zone.parent.get_responsive_auth_or_designated_servers() 1804 if not parent_servers: 1805 # while the parent servers might not be responsive for the parent name, 1806 # they must be responsive for the current name, or else we wouldn't be here. 1807 parent_servers = name_obj.zone.parent.get_auth_or_designated_servers() 1808 parent_servers = self._filter_servers(parent_servers) 1809 1810 parent_odd_ports = dict([(s, self.odd_ports[(n, s)]) for n, s in self.odd_ports if n in (name_obj.zone.parent.name, WILDCARD_EXPLICIT_DELEGATION)]) 1811 parent_cookie_jar = name_obj.zone.parent.cookie_jar 1812 1813 self.logger.debug('Preparing query %s/DS...' % fmt.humanize_name(name_obj.name)) 1814 queries[(name_obj.name, dns.rdatatype.DS)] = self.diagnostic_query(name_obj.name, dns.rdatatype.DS, self.rdclass, parent_servers, name_obj.parent_name(), self.client_ipv4, self.client_ipv6, odd_ports=parent_odd_ports, cookie_jar=parent_cookie_jar, cookie_standin=COOKIE_STANDIN) 1815 1816 if name_obj.dlv_parent is not None and self.dlv_domain != self.name: 1817 dlv_servers = name_obj.dlv_parent.get_responsive_auth_or_designated_servers() 1818 dlv_servers = self._filter_servers(dlv_servers) 1819 dlv_name = name_obj.dlv_name 1820 if dlv_servers: 1821 dlv_odd_ports = dict([(s, self.odd_ports[(n, s)]) for n, s in self.odd_ports if n in (name_obj.dlv_parent.name, WILDCARD_EXPLICIT_DELEGATION)]) 1822 dlv_cookie_jar = name_obj.dlv_parent.cookie_jar 1823 1824 self.logger.debug('Preparing query %s/DLV...' % fmt.humanize_name(dlv_name)) 1825 queries[(dlv_name, dns.rdatatype.DLV)] = self.diagnostic_query(dlv_name, dns.rdatatype.DLV, self.rdclass, dlv_servers, name_obj.dlv_parent_name(), self.client_ipv4, self.client_ipv6, odd_ports=dlv_odd_ports, cookie_jar=dlv_cookie_jar, cookie_standin=COOKIE_STANDIN) 1826 exclude_no_answer.add((dlv_name, dns.rdatatype.DLV)) 1827 1828 # get rid of any queries already asked 1829 for name, rdtype in set(name_obj.queries).intersection(set(queries)): 1830 del queries[(name, rdtype)] 1831 1832 # finally, query any additional rdtypes 1833 if servers and self._ask_non_delegation_queries(name_obj.name): 1834 all_queries = set(name_obj.queries).union(set(queries)) 1835 for rdtype in self._rdtypes_to_query(name_obj.name): 1836 if (name_obj.name, rdtype) not in all_queries: 1837 self.logger.debug('Preparing query %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(rdtype))) 1838 queries[(name_obj.name, rdtype)] = self.diagnostic_query(name_obj.name, rdtype, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1839 1840 # if no default queries were identified (e.g., empty non-terminal in 1841 # in-addr.arpa space), then add a backup. 1842 if not (queries or name_obj.queries): 1843 rdtype = dns.rdatatype.A 1844 self.logger.debug('Preparing query %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(rdtype))) 1845 queries[(name_obj.name, rdtype)] = self.diagnostic_query(name_obj.name, rdtype, self.rdclass, servers, bailiwick, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1846 1847 # actually execute the queries, then store the results 1848 self.logger.debug('Executing queries...') 1849 Q.ExecutableDNSQuery.execute_queries(*list(queries.values()), tm=self.transport_manager, th_factories=self.th_factories) 1850 for key, query in queries.items(): 1851 if query.is_answer_any() or key not in exclude_no_answer: 1852 self._add_query(name_obj, query, False, False) 1853 1854 def _analyze_delegation(self, name_obj): 1855 if name_obj.parent is None: 1856 parent_auth_servers = self._root_servers() 1857 elif name_obj.parent.stub: 1858 parent_auth_servers = name_obj.parent.get_auth_or_designated_servers() 1859 else: 1860 parent_auth_servers = name_obj.parent.get_responsive_auth_or_designated_servers() 1861 # even if no servers are responsive, use all designated servers if this 1862 # is the name in question, for completeness 1863 if not parent_auth_servers and name_obj.name == self.name: 1864 parent_auth_servers = name_obj.parent.get_auth_or_designated_servers() 1865 parent_auth_servers = set(self._filter_servers(parent_auth_servers)) 1866 1867 odd_ports = dict([(s, self.odd_ports[(n, s)]) for n, s in self.odd_ports if n in (name_obj.zone.name, WILDCARD_EXPLICIT_DELEGATION)]) 1868 cookie_jar = name_obj.zone.cookie_jar 1869 1870 if not parent_auth_servers: 1871 return False 1872 1873 servers_queried = OrderedDict(((dns.rdatatype.NS, set()),)) 1874 referral_queries = {} 1875 1876 try: 1877 secondary_rdtype = self._rdtypes_to_query(name_obj.name)[0] 1878 except IndexError: 1879 secondary_rdtype = None 1880 else: 1881 if secondary_rdtype in (dns.rdatatype.DS, dns.rdatatype.DLV, dns.rdatatype.NS): 1882 secondary_rdtype = None 1883 else: 1884 servers_queried[secondary_rdtype] = set() 1885 1886 # elicit a referral from parent servers by querying first for NS, then 1887 # a secondary type as a fallback 1888 for rdtype in servers_queried: 1889 servers_queried[rdtype].update(parent_auth_servers) 1890 1891 name_obj.referral_rdtype = rdtype 1892 1893 self.logger.debug('Querying %s/%s (referral)...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(rdtype))) 1894 query = self.diagnostic_query(name_obj.name, rdtype, self.rdclass, parent_auth_servers, name_obj.parent_name(), self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 1895 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 1896 referral_queries[rdtype] = query 1897 1898 # if NXDOMAIN was received, then double-check with the secondary 1899 # type, as some servers (mostly load balancers) don't respond well 1900 # to NS queries 1901 if query.is_nxdomain_all(): 1902 continue 1903 1904 # otherwise, if we received at least one valid response, then break out 1905 if query.is_valid_complete_response_any(): 1906 break 1907 1908 # we only go a second time through the loop, querying the secondary 1909 # rdtype query if 1) there was NXDOMAIN or 2) there were no valid 1910 # responses. In either case the secondary record type becomes the 1911 # referral rdtype. 1912 1913 # if the name is not a delegation, or if we received no valid and 1914 # complete response, then move along 1915 if query.is_not_delegation_all() or not query.is_valid_complete_response_any(): 1916 # We only keep the referral response if: 1917 # 1) there was an error getting a referral response; 1918 # 2) there was a discrepancy between NXDOMAIN and YXDOMAIN; or 1919 # 3) this is the name in question and the response was NXDOMAIN, 1920 # in which case we use this to show the NXDOMAIN (empty answers 1921 # will be asked later by better queries) 1922 # And in the case of a referral using the secondary rdtype, we only 1923 # keep the NS referral if there was a discrepancy between NXDOMAIN 1924 # and YXDOMAIN. 1925 1926 is_nxdomain = query.is_nxdomain_all() 1927 is_valid = query.is_valid_complete_response_any() 1928 1929 # (referral type is NS) 1930 if name_obj.referral_rdtype == dns.rdatatype.NS: 1931 # If there was a secondary type, the fact that only NS was queried 1932 # for indicates that there was no error and no NXDOMAIN response. 1933 # In this case, there is no need to save the referral. Delete it. 1934 if secondary_rdtype is not None: 1935 name_obj.referral_rdtype = None 1936 del referral_queries[dns.rdatatype.NS] 1937 1938 # If there was no secondary type, we need to evaluate the responses 1939 # to see if they're worth saving. Save the referral if there 1940 # was an error or NXDOMAIN and there is no nxdomain_ancestor or 1941 # this is the name in question. 1942 else: 1943 if not is_valid or (is_nxdomain and (name_obj.name == self.name or name_obj.nxdomain_ancestor is None)): 1944 pass 1945 else: 1946 name_obj.referral_rdtype = None 1947 del referral_queries[dns.rdatatype.NS] 1948 1949 # (referral type is secondary type) 1950 else: 1951 # don't remove either record if there's an NXDOMAIN/YXDOMAIN mismatch 1952 if referral_queries[dns.rdatatype.NS].is_nxdomain_all() and \ 1953 is_valid and not is_nxdomain: 1954 pass 1955 else: 1956 # if no mismatch, then always delete the NS record 1957 del referral_queries[dns.rdatatype.NS] 1958 # Save the referral if there was an error or NXDOMAIN and 1959 # there is no nxdomain_ancestor or this is the name in 1960 # question. 1961 if not is_valid or (is_nxdomain and (name_obj.name == self.name or name_obj.nxdomain_ancestor is None)): 1962 pass 1963 else: 1964 name_obj.referral_rdtype = None 1965 del referral_queries[secondary_rdtype] 1966 1967 # add remaining queries 1968 for query in referral_queries.values(): 1969 self._add_query(name_obj, query, True, False) 1970 1971 # return a positive response only if not nxdomain 1972 return not is_nxdomain 1973 1974 if self.dns_cookies: 1975 # An NS query to authoritative servers will always be used to 1976 # elicit a server query. 1977 name_obj.cookie_rdtype = dns.rdatatype.NS 1978 cookie_jar = name_obj.cookie_jar 1979 cookie_str = ', detecting cookies' 1980 else: 1981 name_obj.cookie_rdtype = None 1982 cookie_jar = None 1983 cookie_str = '' 1984 1985 1986 # Add any queries made. At this point, at least one of the queries is 1987 # for type NS. If there is a second, it is because the first resulted 1988 # in NXDOMAIN, and the type for the second query is secondary_rdtype. 1989 for query in referral_queries.values(): 1990 detect_cookies = query.rdtype == name_obj.cookie_rdtype 1991 self._add_query(name_obj, query, True, detect_cookies) 1992 1993 # Identify auth_rdtype, the rdtype used to query the authoritative 1994 # servers to retrieve NS records in the authority section. 1995 name_obj.auth_rdtype = secondary_rdtype 1996 1997 # Now identify the authoritative NS RRset from all servers, both by 1998 # querying the authoritative servers for NS and by querying them for 1999 # another type and looking for NS in the authority section. Resolve 2000 # all names referred to in the NS RRset(s), and query each 2001 # corresponding server, until all names have been resolved and all 2002 # corresponding addresses queried.. 2003 names_resolved = set() 2004 names_not_resolved = name_obj.get_ns_names() 2005 while names_not_resolved: 2006 # resolve every name in the NS RRset 2007 query_tuples = [] 2008 for name in names_not_resolved: 2009 query_tuples.extend([(name, dns.rdatatype.A, self.rdclass), (name, dns.rdatatype.AAAA, self.rdclass)]) 2010 answer_map = self.resolver.query_multiple_for_answer(*query_tuples) 2011 for query_tuple in answer_map: 2012 name = query_tuple[0] 2013 a = answer_map[query_tuple] 2014 if isinstance(a, Resolver.DNSAnswer): 2015 for a_rr in a.rrset: 2016 name_obj.add_auth_ns_ip_mappings((name, IPAddr(a_rr.to_text()))) 2017 # negative responses 2018 elif isinstance(a, (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer)): 2019 name_obj.add_auth_ns_ip_mappings((name, None)) 2020 # error responses 2021 elif isinstance(a, (dns.exception.Timeout, dns.resolver.NoNameservers)): 2022 pass 2023 names_resolved.add(name) 2024 2025 queries = [] 2026 auth_servers = name_obj.get_auth_or_designated_servers(no_cache=True) 2027 2028 # NS query 2029 servers = auth_servers.difference(servers_queried[dns.rdatatype.NS]) 2030 servers_queried[dns.rdatatype.NS].update(servers) 2031 servers = self._filter_servers(servers, no_raise=True) 2032 if servers: 2033 self.logger.debug('Querying %s/NS (auth%s)...' % (fmt.humanize_name(name_obj.name), cookie_str)) 2034 query = self.diagnostic_query_no_server_cookie(name_obj.name, dns.rdatatype.NS, self.rdclass, servers, name_obj.name, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports) 2035 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 2036 self._add_query(name_obj, query, True, True, True) 2037 2038 # secondary query 2039 if secondary_rdtype is not None and self._ask_non_delegation_queries(name_obj.name): 2040 servers = auth_servers.difference(servers_queried[secondary_rdtype]) 2041 servers_queried[secondary_rdtype].update(servers) 2042 servers = self._filter_servers(servers, no_raise=True) 2043 if servers: 2044 self.logger.debug('Querying %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(secondary_rdtype))) 2045 query = self.diagnostic_query(name_obj.name, secondary_rdtype, self.rdclass, servers, name_obj.name, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 2046 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 2047 self._add_query(name_obj, query, True, False, True) 2048 2049 names_not_resolved = name_obj.get_ns_names().difference(names_resolved) 2050 2051 #TODO now go back and look at servers authoritative for both parent and 2052 #child that have authoritative referrals and re-classify them as 2053 #non-referrals (do this in deserialize (and dnsvizwww retrieve also) 2054 2055 return True 2056 2057 def _analyze_dependency(self, analyst, result_map, result_key, errors): 2058 try: 2059 result_map[result_key] = analyst.analyze() 2060 except: 2061 errors.append((result_key, sys.exc_info())) 2062 2063 def _analyze_dependencies(self, name_obj): 2064 threads = [] 2065 errors = [] 2066 2067 kwargs = dict([(n, getattr(self, n)) for n in self.clone_attrnames]) 2068 for cname in name_obj.cname_targets: 2069 for target in name_obj.cname_targets[cname]: 2070 a = self.__class__(target, trace=self.trace + [(name_obj, dns.rdatatype.CNAME)], explicit_only=self.explicit_only, extra_rdtypes=self.extra_rdtypes, **kwargs) 2071 t = threading.Thread(target=self._analyze_dependency, args=(a, name_obj.cname_targets[cname], target, errors)) 2072 t.start() 2073 threads.append(t) 2074 2075 for signer in name_obj.external_signers: 2076 a = self.__class__(signer, trace=self.trace + [(name_obj, dns.rdatatype.RRSIG)], **kwargs) 2077 t = threading.Thread(target=self._analyze_dependency, args=(a, name_obj.external_signers, signer, errors)) 2078 t.start() 2079 threads.append(t) 2080 2081 if self.follow_ns: 2082 for ns in name_obj.ns_dependencies: 2083 a = self.__class__(ns, trace=self.trace + [(name_obj, dns.rdatatype.NS)], **kwargs) 2084 t = threading.Thread(target=self._analyze_dependency, args=(a, name_obj.ns_dependencies, ns, errors)) 2085 t.start() 2086 threads.append(t) 2087 2088 if self.follow_mx: 2089 for target in name_obj.mx_targets: 2090 a = self.__class__(target, trace=self.trace + [(name_obj, dns.rdatatype.MX)], explicit_only=True, extra_rdtypes=[dns.rdatatype.A, dns.rdatatype.AAAA], **kwargs) 2091 t = threading.Thread(target=self._analyze_dependency, args=(a, name_obj.mx_targets, target, errors)) 2092 t.start() 2093 threads.append(t) 2094 2095 for t in threads: 2096 t.join() 2097 if errors: 2098 # raise only the first exception, but log all the ones beyond 2099 for name, exc_info in errors[1:]: 2100 self.logger.error('Error analyzing %s' % name, exc_info=exc_info) 2101 # python3/python2 dual compatibility 2102 if hasattr(errors[0][1][0], 'with_traceback'): 2103 raise errors[0][1][1].with_traceback(errors[0][1][2]) 2104 else: 2105 # lesser python2 functionality 2106 exec('raise errors[0][1][1], None, errors[0][1][2]') 2107 2108 def _mix_case(self, name): 2109 name = name.to_text().lower() 2110 name_len = len(name) 2111 rnd = random.getrandbits((name_len + 8) - (name_len % 8)) 2112 new_name = '' 2113 changed = False 2114 for i, c in enumerate(name): 2115 # If the character is a lower case letter, mix it up randomly. 2116 # Always make the first letter upper case, to ensure that it isn't 2117 # completely lower case. 2118 if ord('a') <= ord(c) <= ord('z') and ((rnd & (1 << i)) or not changed): 2119 new_name += chr(ord(c) - 32) 2120 changed = True 2121 else: 2122 new_name += c 2123 if not changed: 2124 return None 2125 return dns.name.from_text(new_name) 2126 2127 def _set_negative_queries(self, name_obj): 2128 random_label = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz1234567890', 10)) 2129 try: 2130 name_obj.nxdomain_name = dns.name.from_text(random_label, name_obj.name) 2131 name_obj.nxdomain_rdtype = dns.rdatatype.A 2132 except dns.name.NameTooLong: 2133 pass 2134 2135 name_obj.nxrrset_name = name_obj.name 2136 name_obj.nxrrset_rdtype = dns.rdatatype.CNAME 2137 2138 def _require_connectivity_ipv4(self, name_obj): 2139 return bool([x for x in name_obj.clients_ipv4 if LOOPBACK_IPV4_RE.match(x) is None]) 2140 2141 def _require_connectivity_ipv6(self, name_obj): 2142 return bool([x for x in name_obj.clients_ipv6 if x != LOOPBACK_IPV6]) 2143 2144 def _check_connectivity(self, name_obj): 2145 if self.local_ceiling is not None and (self.local_ceiling, dns.rdatatype.NS) in self.explicit_delegations: 2146 # this check is only useful if not the descendant of an explicit 2147 # delegation 2148 return 2149 if name_obj.get_auth_or_designated_servers(4) and self._require_connectivity_ipv4(name_obj) and not name_obj.get_responsive_servers_udp(4): 2150 if self._root_responsive(4) is False: 2151 raise IPv4ConnectivityException('IPv4 network is unreachable!') 2152 if name_obj.get_auth_or_designated_servers(6) and self._require_connectivity_ipv6(name_obj) and not name_obj.get_responsive_servers_udp(6): 2153 if self._root_responsive(6) is False: 2154 raise IPv6ConnectivityException('IPv6 network is unreachable!') 2155 2156 def _raise_connectivity_error_remote(self): 2157 if not (self.try_ipv4 or self.try_ipv6): 2158 raise NetworkConnectivityException('No servers to query!') 2159 elif self.try_ipv4: 2160 raise IPv4ConnectivityException('No IPv4 servers to query!') 2161 else: # self.try_ipv6 2162 raise IPv6ConnectivityException('No IPv6 servers to query!') 2163 2164 def _raise_connectivity_error_local(self, servers): 2165 '''Raise a network connectivity exception. The class of exception and 2166 message associated with the exception are determined based on: whether 2167 IPv4 or IPv6 is specified to be attempted; whether there were IPv4 or 2168 IPv6 servers to query.''' 2169 2170 if self.try_ipv4 and self.try_ipv6: 2171 # if we are configured to try both IPv4 and IPv6, then use servers 2172 # to determine which one is missing 2173 has_v4 = bool([x for x in servers if x.version == 4]) 2174 has_v6 = bool([x for x in servers if x.version == 6]) 2175 if has_v4 and has_v6: 2176 # if both IPv4 and IPv6 servers were attempted, then there was 2177 # no network connectivity for either protocol 2178 raise NetworkConnectivityException('No network connectivity available!') 2179 elif has_v4: 2180 raise IPv4ConnectivityException('No IPv4 network connectivity available!') 2181 else: # has_v6 2182 raise IPv6ConnectivityException('No IPv6 network connectivity available!') 2183 elif self.try_ipv4: 2184 raise IPv4ConnectivityException('No IPv4 network connectivity available!') 2185 elif self.try_ipv6: 2186 raise IPv6ConnectivityException('No IPv6 network connectivity available!') 2187 else: 2188 raise NetworkConnectivityException('No network connectivity available!') 2189 2190 def _root_responsive(self, proto): 2191 servers = list(self._root_servers(proto)) 2192 checker = Resolver.Resolver(servers, self.simple_query, max_attempts=1, shuffle=True, transport_manager=self.transport_manager) 2193 try: 2194 checker.query_for_answer(dns.name.root, dns.rdatatype.NS, self.rdclass) 2195 return True 2196 except dns.resolver.NoNameservers: 2197 return None 2198 except dns.exception.Timeout: 2199 pass 2200 return False 2201 2202class PrivateAnalyst(Analyst): 2203 default_th_factory = transport.DNSQueryTransportHandlerDNSPrivateFactory() 2204 2205class RecursiveAnalyst(Analyst): 2206 _simple_query = Q.RecursiveDNSQuery 2207 _diagnostic_query = Q.RecursiveDiagnosticQuery 2208 _tcp_diagnostic_query = Q.RecursiveTCPDiagnosticQuery 2209 _pmtu_diagnostic_query = Q.RecursivePMTUDiagnosticQuery 2210 _truncation_diagnostic_query = Q.RecursiveTruncationDiagnosticQuery 2211 _edns_version_diagnostic_query = Q.RecursiveEDNSVersionDiagnosticQuery 2212 _edns_flag_diagnostic_query = Q.RecursiveEDNSFlagDiagnosticQuery 2213 _edns_opt_diagnostic_query = Q.RecursiveEDNSOptDiagnosticQuery 2214 2215 analysis_type = ANALYSIS_TYPE_RECURSIVE 2216 2217 def _get_resolver(self): 2218 servers = set() 2219 for rdata in self.explicit_delegations[(WILDCARD_EXPLICIT_DELEGATION, dns.rdatatype.NS)]: 2220 for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA): 2221 if (rdata.target, rdtype) in self.explicit_delegations: 2222 servers.update([IPAddr(r.address) for r in self.explicit_delegations[(rdata.target, rdtype)]]) 2223 return Resolver.Resolver(list(servers), Q.StandardRecursiveQueryCD, transport_manager=self.transport_manager) 2224 2225 def _detect_ceiling(self, ceiling): 2226 # if there is a ceiling, but the name is not a subdomain 2227 # of the ceiling, then use the name itself as a base 2228 if ceiling is not None and not self.name.is_subdomain(ceiling): 2229 ceiling = self.name 2230 2231 return ceiling, None 2232 2233 def _finalize_analysis_proper(self, name_obj): 2234 '''Since we initially queried the full set of queries before we knew 2235 which were appropriate for the name in question, we now identify all queries 2236 that were pertinent and remove all other.''' 2237 2238 # if it's a stub, then no need to do anything 2239 if name_obj.stub: 2240 return 2241 2242 # if there are not NS records, then it's not a zone, so clear auth NS 2243 # IP mapping 2244 if not name_obj.has_ns: 2245 name_obj._auth_ns_ip_mapping = {} 2246 2247 queries = set() 2248 if name_obj.is_zone(): 2249 queries.add((name_obj.name, dns.rdatatype.NS)) 2250 if self._ask_non_delegation_queries(name_obj.name) and not self.explicit_only: 2251 queries.add((name_obj.nxdomain_name, name_obj.nxdomain_rdtype)) 2252 queries.add((name_obj.nxrrset_name, name_obj.nxrrset_rdtype)) 2253 if self._is_sld_or_lower(name_obj.name): 2254 queries.add((name_obj.name, dns.rdatatype.MX)) 2255 queries.add((name_obj.name, dns.rdatatype.TXT)) 2256 if name_obj.is_zone() or self._force_dnskey_query(name_obj.name): 2257 if self._ask_non_delegation_queries(name_obj.name) and not self.explicit_only: 2258 queries.add((name_obj.name, dns.rdatatype.SOA)) 2259 queries.add((name_obj.name, dns.rdatatype.DNSKEY)) 2260 2261 if name_obj.parent is not None: 2262 queries.add((name_obj.name, dns.rdatatype.DS)) 2263 if name_obj.dlv_parent is not None: 2264 queries.add((name_obj.dlv_name, dns.rdatatype.DLV)) 2265 2266 if self._ask_non_delegation_queries(name_obj.name): 2267 for rdtype in self._rdtypes_to_query(name_obj.name): 2268 queries.add((name_obj.name, rdtype)) 2269 2270 if not queries: 2271 # for TLD and higher, add NS 2272 if len(name_obj.name) <= 2: 2273 rdtype = dns.rdatatype.NS 2274 # for SLD and lower, add A 2275 else: 2276 rdtype = dns.rdatatype.A 2277 queries.add((name_obj.name, rdtype)) 2278 2279 for name, rdtype in set(name_obj.queries).difference(queries): 2280 del name_obj.queries[(name, rdtype)] 2281 2282 if (name_obj.nxdomain_name, name_obj.nxdomain_rdtype) not in queries: 2283 name_obj.nxdomain_name = None 2284 name_obj.nxdomain_rdtype = None 2285 if (name_obj.nxrrset_name, name_obj.nxrrset_rdtype) not in queries: 2286 name_obj.nxrrset_name = None 2287 name_obj.nxrrset_rdtype = None 2288 2289 def _analyze_stub(self, name): 2290 name_obj = self._get_name_for_analysis(name, stub=True) 2291 if name_obj.analysis_end is not None: 2292 return name_obj 2293 2294 try: 2295 self.logger.info('Analyzing %s (stub)' % fmt.humanize_name(name)) 2296 2297 name_obj.analysis_start = datetime.datetime.now(fmt.utc).replace(microsecond=0) 2298 2299 self._handle_explicit_delegations(name_obj) 2300 servers = name_obj.zone.get_auth_or_designated_servers() 2301 servers = self._filter_servers(servers) 2302 resolver = Resolver.Resolver(list(servers), Q.StandardRecursiveQueryCD, transport_manager=self.transport_manager) 2303 2304 try: 2305 ans = resolver.query_for_answer(name, dns.rdatatype.NS, self.rdclass) 2306 except (dns.resolver.NoAnswer, dns.resolver.NXDOMAIN): 2307 name_obj.parent = self._analyze_stub(name.parent()).zone 2308 except dns.exception.DNSException: 2309 pass 2310 2311 name_obj.analysis_end = datetime.datetime.now(fmt.utc).replace(microsecond=0) 2312 2313 self._finalize_analysis_proper(name_obj) 2314 self._finalize_analysis_all(name_obj) 2315 finally: 2316 self._cleanup_analysis_proper(name_obj) 2317 self._cleanup_analysis_all(name_obj) 2318 2319 return name_obj 2320 2321 def _analyze_ancestry(self, name, is_zone): 2322 # only analyze the parent if the name is not root and if there is no 2323 # ceiling or the name is a subdomain of the ceiling 2324 if name == dns.name.root: 2325 parent_obj = None 2326 elif self.local_ceiling is not None and self.local_ceiling.is_subdomain(name) and is_zone: 2327 parent_obj = self._analyze_stub(name.parent()) 2328 else: 2329 parent_obj = self._analyze(name.parent()) 2330 2331 if parent_obj is not None: 2332 nxdomain_ancestor = parent_obj.nxdomain_ancestor 2333 if nxdomain_ancestor is None and not parent_obj.stub: 2334 rdtype = [x for x in parent_obj.queries.keys() if x[0] == parent_obj.name][0][1] 2335 if parent_obj.queries[(parent_obj.name, rdtype)].is_nxdomain_all(): 2336 nxdomain_ancestor = parent_obj 2337 2338 # for zones other than the root assign parent_obj to the zone apex, 2339 # rather than the simply the domain formed by dropping its lower 2340 # leftmost label 2341 parent_obj = parent_obj.zone 2342 2343 else: 2344 nxdomain_ancestor = None 2345 2346 # retrieve the dlv 2347 if self.dlv_domain is not None and self.name != self.dlv_domain: 2348 dlv_parent_obj = self.analysis_cache[self.dlv_domain] 2349 else: 2350 dlv_parent_obj = None 2351 2352 return parent_obj, dlv_parent_obj, nxdomain_ancestor 2353 2354 def _analyze(self, name): 2355 '''Analyze a DNS name to learn about its health using introspective 2356 queries.''' 2357 2358 # determine immediately if we need to do anything 2359 name_obj = self._get_name_for_analysis(name, lock=False) 2360 if name_obj is not None and name_obj.analysis_end is not None: 2361 return name_obj 2362 2363 # get or create the name 2364 name_obj = self._get_name_for_analysis(name) 2365 if name_obj.analysis_end is not None: 2366 return name_obj 2367 2368 try: 2369 try: 2370 name_obj.analysis_start = datetime.datetime.now(fmt.utc).replace(microsecond=0) 2371 2372 # perform the actual analysis on this name 2373 self._analyze_name(name_obj) 2374 2375 # set analysis_end 2376 name_obj.analysis_end = datetime.datetime.now(fmt.utc).replace(microsecond=0) 2377 2378 # sanity check - if we weren't able to get responses from any 2379 # servers, check that we actually have connectivity 2380 self._check_connectivity(name_obj) 2381 2382 # analyze ancestry 2383 parent_obj, dlv_parent_obj, nxdomain_ancestor = \ 2384 self._analyze_ancestry(name, name_obj.has_ns) 2385 2386 name_obj.parent = parent_obj 2387 name_obj.dlv_parent = dlv_parent_obj 2388 name_obj.nxdomain_ancestor = nxdomain_ancestor 2389 2390 self._finalize_analysis_proper(name_obj) 2391 finally: 2392 self._cleanup_analysis_proper(name_obj) 2393 2394 # analyze dependencies 2395 self._analyze_dependencies(name_obj) 2396 2397 self._finalize_analysis_all(name_obj) 2398 finally: 2399 self._cleanup_analysis_all(name_obj) 2400 2401 return name_obj 2402 2403 def _analyze_name(self, name_obj): 2404 self.logger.info('Analyzing %s' % fmt.humanize_name(name_obj.name)) 2405 2406 self._handle_explicit_delegations(name_obj) 2407 2408 servers = name_obj.zone.get_auth_or_designated_servers() 2409 if not servers: 2410 raise NoNameservers('No resolvers specified to query!') 2411 servers = self._filter_servers(servers) 2412 if not servers: 2413 raise NoNameservers('No resolvers available to query!') 2414 2415 odd_ports = dict([(s, self.odd_ports[(n, s)]) for n, s in self.odd_ports if n in (name_obj.zone.name, WILDCARD_EXPLICIT_DELEGATION)]) 2416 cookie_jar = name_obj.zone.cookie_jar 2417 2418 # make common query first to prime the cache 2419 2420 # for root and TLD, use type NS 2421 if len(name_obj.name) <= 2: 2422 rdtype = dns.rdatatype.NS 2423 # for SLDs and below detect an appropriate type 2424 # and use A as a fallback. 2425 else: 2426 try: 2427 rdtype = self._rdtypes_to_query(name_obj.name)[0] 2428 except IndexError: 2429 rdtype = dns.rdatatype.A 2430 else: 2431 if rdtype in (dns.rdatatype.DS, dns.rdatatype.NS): 2432 rdtype = dns.rdatatype.A 2433 2434 self.logger.debug('Querying %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(rdtype))) 2435 query = self.diagnostic_query_no_server_cookie(name_obj.name, rdtype, self.rdclass, servers, None, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports) 2436 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 2437 self._add_query(name_obj, query, True, True) 2438 2439 if self.dns_cookies: 2440 name_obj.cookie_rdtype = rdtype 2441 else: 2442 name_obj.cookie_rdtype = None 2443 2444 # if there were no valid responses, then exit out early 2445 if not query.is_valid_complete_response_any() and not self.explicit_only: 2446 return name_obj 2447 2448 # if there was an NXDOMAIN for the first query, then don't ask the 2449 # others, unless explicit was called 2450 if query.is_nxdomain_all() and not self.explicit_only: 2451 return name_obj 2452 2453 # now query most other queries 2454 self._analyze_queries(name_obj) 2455 2456 if name_obj.name != dns.name.root: 2457 # ensure these weren't already queried for (e.g., as part of extra_rdtypes) 2458 if (name_obj.name, dns.rdatatype.DS) not in name_obj.queries: 2459 # make DS queries (these won't be included in the above mix 2460 # because there is no parent on the name_obj) 2461 self.logger.debug('Querying %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(dns.rdatatype.DS))) 2462 query = self.diagnostic_query(name_obj.name, dns.rdatatype.DS, self.rdclass, servers, None, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 2463 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 2464 self._add_query(name_obj, query, False, False) 2465 2466 # for non-TLDs make NS queries after all others 2467 if len(name_obj.name) > 2: 2468 # ensure these weren't already queried for (e.g., as part of extra_rdtypes) 2469 if (name_obj.name, dns.rdatatype.NS) not in name_obj.queries: 2470 self.logger.debug('Querying %s/%s...' % (fmt.humanize_name(name_obj.name), dns.rdatatype.to_text(dns.rdatatype.NS))) 2471 query = self.diagnostic_query(name_obj.name, dns.rdatatype.NS, self.rdclass, servers, None, self.client_ipv4, self.client_ipv6, odd_ports=odd_ports, cookie_jar=cookie_jar, cookie_standin=COOKIE_STANDIN) 2472 query.execute(tm=self.transport_manager, th_factories=self.th_factories) 2473 self._add_query(name_obj, query, True, False) 2474 2475 return name_obj 2476 2477 def _require_connectivity_ipv4(self, name_obj): 2478 return bool(name_obj.clients_ipv4) 2479 2480 def _require_connectivity_ipv6(self, name_obj): 2481 return bool(name_obj.clients_ipv6) 2482 2483 def _check_connectivity(self, name_obj): 2484 if self._require_connectivity_ipv4(name_obj) and not name_obj.get_responsive_servers_udp(4): 2485 if self._root_responsive(4) is False: 2486 raise IPv4ConnectivityException('IPv4 resolvers are not responsive!') 2487 if self._require_connectivity_ipv6(name_obj) and not name_obj.get_responsive_servers_udp(6): 2488 if self._root_responsive(6) is False: 2489 raise IPv6ConnectivityException('IPv6 resolvers are not responsive!') 2490 2491class PrivateRecursiveAnalyst(RecursiveAnalyst): 2492 default_th_factory = transport.DNSQueryTransportHandlerDNSPrivateFactory() 2493