1# This file is part of Scapy
2# See http://www.secdev.org/projects/scapy for more information
3# Copyright (C) Philippe Biondi <phil@secdev.org>
4# This program is published under a GPLv2 license
5
6"""
7Global variables and functions for handling external data sets.
8"""
9
10import calendar
11import os
12import re
13import warnings
14
15
16from scapy.dadict import DADict, fixname
17from scapy.consts import FREEBSD, NETBSD, OPENBSD, WINDOWS
18from scapy.error import log_loading
19from scapy.compat import plain_str
20import scapy.modules.six as six
21
22from scapy.compat import (
23    Any,
24    Callable,
25    Dict,
26    Iterator,
27    List,
28    Optional,
29    Tuple,
30    cast,
31)
32
33
34############
35#  Consts  #
36############
37
38ETHER_ANY = b"\x00" * 6
39ETHER_BROADCAST = b"\xff" * 6
40
41# From bits/socket.h
42SOL_PACKET = 263
43# From asm/socket.h
44SO_ATTACH_FILTER = 26
45SO_TIMESTAMPNS = 35  # SO_TIMESTAMPNS_OLD: not 2038 safe
46
47ETH_P_ALL = 3
48ETH_P_IP = 0x800
49ETH_P_ARP = 0x806
50ETH_P_IPV6 = 0x86dd
51ETH_P_MACSEC = 0x88e5
52
53# From net/if_arp.h
54ARPHDR_ETHER = 1
55ARPHDR_METRICOM = 23
56ARPHDR_PPP = 512
57ARPHDR_LOOPBACK = 772
58ARPHDR_TUN = 65534
59
60# From pcap/dlt.h
61DLT_NULL = 0
62DLT_EN10MB = 1
63DLT_EN3MB = 2
64DLT_AX25 = 3
65DLT_PRONET = 4
66DLT_CHAOS = 5
67DLT_IEEE802 = 6
68DLT_ARCNET = 7
69DLT_SLIP = 8
70DLT_PPP = 9
71DLT_FDDI = 10
72if OPENBSD:
73    DLT_RAW = 14
74else:
75    DLT_RAW = 12
76DLT_RAW_ALT = 101  # At least in Argus
77if FREEBSD or NETBSD:
78    DLT_SLIP_BSDOS = 13
79    DLT_PPP_BSDOS = 14
80else:
81    DLT_SLIP_BSDOS = 15
82    DLT_PPP_BSDOS = 16
83if FREEBSD:
84    DLT_PFSYNC = 121
85else:
86    DLT_PFSYNC = 18
87    DLT_HHDLC = 121
88DLT_ATM_CLIP = 19
89DLT_PPP_SERIAL = 50
90DLT_PPP_ETHER = 51
91DLT_SYMANTEC_FIREWALL = 99
92DLT_C_HDLC = 104
93DLT_IEEE802_11 = 105
94DLT_FRELAY = 107
95if OPENBSD:
96    DLT_LOOP = 12
97    DLT_ENC = 13
98else:
99    DLT_LOOP = 108
100    DLT_ENC = 109
101DLT_LINUX_SLL = 113
102DLT_LTALK = 114
103DLT_PFLOG = 117
104DLT_PRISM_HEADER = 119
105DLT_AIRONET_HEADER = 120
106DLT_IP_OVER_FC = 122
107DLT_IEEE802_11_RADIO = 127
108DLT_ARCNET_LINUX = 129
109DLT_LINUX_IRDA = 144
110DLT_IEEE802_11_RADIO_AVS = 163
111DLT_LINUX_LAPD = 177
112DLT_BLUETOOTH_HCI_H4 = 187
113DLT_USB_LINUX = 189
114DLT_PPI = 192
115DLT_IEEE802_15_4_WITHFCS = 195
116DLT_BLUETOOTH_HCI_H4_WITH_PHDR = 201
117DLT_AX25_KISS = 202
118DLT_PPP_WITH_DIR = 204
119DLT_FC_2 = 224
120DLT_CAN_SOCKETCAN = 227
121if OPENBSD:
122    DLT_IPV4 = DLT_RAW
123    DLT_IPV6 = DLT_RAW
124else:
125    DLT_IPV4 = 228
126    DLT_IPV6 = 229
127DLT_IEEE802_15_4_NOFCS = 230
128DLT_USBPCAP = 249
129DLT_NETLINK = 253
130DLT_USB_DARWIN = 266
131DLT_BLUETOOTH_LE_LL = 251
132DLT_BLUETOOTH_LE_LL_WITH_PHDR = 256
133DLT_VSOCK = 271
134DLT_ETHERNET_MPACKET = 274
135
136# From net/ipv6.h on Linux (+ Additions)
137IPV6_ADDR_UNICAST = 0x01
138IPV6_ADDR_MULTICAST = 0x02
139IPV6_ADDR_CAST_MASK = 0x0F
140IPV6_ADDR_LOOPBACK = 0x10
141IPV6_ADDR_GLOBAL = 0x00
142IPV6_ADDR_LINKLOCAL = 0x20
143IPV6_ADDR_SITELOCAL = 0x40     # deprecated since Sept. 2004 by RFC 3879
144IPV6_ADDR_SCOPE_MASK = 0xF0
145# IPV6_ADDR_COMPATv4   = 0x80     # deprecated; i.e. ::/96
146# IPV6_ADDR_MAPPED     = 0x1000   # i.e.; ::ffff:0.0.0.0/96
147IPV6_ADDR_6TO4 = 0x0100   # Added to have more specific info (should be 0x0101 ?)  # noqa: E501
148IPV6_ADDR_UNSPECIFIED = 0x10000
149
150# from if_arp.h
151ARPHRD_ETHER = 1
152ARPHRD_EETHER = 2
153ARPHRD_AX25 = 3
154ARPHRD_PRONET = 4
155ARPHRD_CHAOS = 5
156ARPHRD_IEEE802 = 6
157ARPHRD_ARCNET = 7
158ARPHRD_DLCI = 15
159ARPHRD_ATM = 19
160ARPHRD_METRICOM = 23
161ARPHRD_SLIP = 256
162ARPHRD_CSLIP = 257
163ARPHRD_SLIP6 = 258
164ARPHRD_CSLIP6 = 259
165ARPHRD_ADAPT = 264
166ARPHRD_CAN = 280
167ARPHRD_PPP = 512
168ARPHRD_CISCO = 513
169ARPHRD_RAWHDLC = 518
170ARPHRD_TUNNEL = 768
171ARPHRD_FRAD = 770
172ARPHRD_LOOPBACK = 772
173ARPHRD_LOCALTLK = 773
174ARPHRD_FDDI = 774
175ARPHRD_SIT = 776
176ARPHRD_FCPP = 784
177ARPHRD_FCAL = 785
178ARPHRD_FCPL = 786
179ARPHRD_FCFABRIC = 787
180ARPHRD_IRDA = 783
181ARPHRD_IEEE802_TR = 800
182ARPHRD_IEEE80211 = 801
183ARPHRD_IEEE80211_PRISM = 802
184ARPHRD_IEEE80211_RADIOTAP = 803
185ARPHRD_IEEE802154 = 804
186ARPHRD_NETLINK = 824
187ARPHRD_VSOCKMON = 826  # from pcap/pcap-linux.c
188ARPHRD_LAPD = 8445  # from pcap/pcap-linux.c
189ARPHRD_NONE = 0xFFFE
190
191ARPHRD_TO_DLT = {  # netlink -> datalink
192    ARPHRD_ETHER: DLT_EN10MB,
193    ARPHRD_METRICOM: DLT_EN10MB,
194    ARPHRD_LOOPBACK: DLT_EN10MB,
195    ARPHRD_EETHER: DLT_EN3MB,
196    ARPHRD_AX25: DLT_AX25_KISS,
197    ARPHRD_PRONET: DLT_PRONET,
198    ARPHRD_CHAOS: DLT_CHAOS,
199    ARPHRD_CAN: DLT_LINUX_SLL,
200    ARPHRD_IEEE802_TR: DLT_IEEE802,
201    ARPHRD_IEEE802: DLT_IEEE802,
202    ARPHRD_ARCNET: DLT_ARCNET_LINUX,
203    ARPHRD_FDDI: DLT_FDDI,
204    ARPHRD_ATM: -1,
205    ARPHRD_IEEE80211: DLT_IEEE802_11,
206    ARPHRD_IEEE80211_PRISM: DLT_PRISM_HEADER,
207    ARPHRD_IEEE80211_RADIOTAP: DLT_IEEE802_11_RADIO,
208    ARPHRD_PPP: DLT_RAW,
209    ARPHRD_CISCO: DLT_C_HDLC,
210    ARPHRD_SIT: DLT_RAW,
211    ARPHRD_CSLIP: DLT_RAW,
212    ARPHRD_SLIP6: DLT_RAW,
213    ARPHRD_CSLIP6: DLT_RAW,
214    ARPHRD_ADAPT: DLT_RAW,
215    ARPHRD_SLIP: DLT_RAW,
216    ARPHRD_RAWHDLC: DLT_RAW,
217    ARPHRD_DLCI: DLT_RAW,
218    ARPHRD_FRAD: DLT_FRELAY,
219    ARPHRD_LOCALTLK: DLT_LTALK,
220    18: DLT_IP_OVER_FC,
221    ARPHRD_FCPP: DLT_FC_2,
222    ARPHRD_FCAL: DLT_FC_2,
223    ARPHRD_FCPL: DLT_FC_2,
224    ARPHRD_FCFABRIC: DLT_FC_2,
225    ARPHRD_IRDA: DLT_LINUX_IRDA,
226    ARPHRD_LAPD: DLT_LINUX_LAPD,
227    ARPHRD_NONE: DLT_RAW,
228    ARPHRD_IEEE802154: DLT_IEEE802_15_4_NOFCS,
229    ARPHRD_NETLINK: DLT_NETLINK,
230    ARPHRD_VSOCKMON: DLT_VSOCK,
231}
232
233# Constants for PPI header types.
234PPI_DOT11COMMON = 2
235PPI_DOT11NMAC = 3
236PPI_DOT11NMACPHY = 4
237PPI_SPECTRUM_MAP = 5
238PPI_PROCESS_INFO = 6
239PPI_CAPTURE_INFO = 7
240PPI_AGGREGATION = 8
241PPI_DOT3 = 9
242PPI_GPS = 30002
243PPI_VECTOR = 30003
244PPI_SENSOR = 30004
245PPI_ANTENNA = 30005
246PPI_BTLE = 30006
247
248# Human-readable type names for PPI header types.
249PPI_TYPES = {
250    PPI_DOT11COMMON: 'dot11-common',
251    PPI_DOT11NMAC: 'dot11-nmac',
252    PPI_DOT11NMACPHY: 'dot11-nmacphy',
253    PPI_SPECTRUM_MAP: 'spectrum-map',
254    PPI_PROCESS_INFO: 'process-info',
255    PPI_CAPTURE_INFO: 'capture-info',
256    PPI_AGGREGATION: 'aggregation',
257    PPI_DOT3: 'dot3',
258    PPI_GPS: 'gps',
259    PPI_VECTOR: 'vector',
260    PPI_SENSOR: 'sensor',
261    PPI_ANTENNA: 'antenna',
262    PPI_BTLE: 'btle',
263}
264
265
266# On windows, epoch is 01/02/1970 at 00:00
267EPOCH = calendar.timegm((1970, 1, 2, 0, 0, 0, 3, 1, 0)) - 86400
268
269MTU = 0xffff  # a.k.a give me all you have
270
271
272# In fact, IANA enterprise-numbers file available at
273# http://www.iana.org/assignments/enterprise-numbers
274# is simply huge (more than 2Mo and 600Ko in bz2). I'll
275# add only most common vendors, and encountered values.
276# -- arno
277IANA_ENTERPRISE_NUMBERS = {
278    9: "ciscoSystems",
279    35: "Nortel Networks",
280    43: "3Com",
281    311: "Microsoft",
282    2636: "Juniper Networks, Inc.",
283    4526: "Netgear",
284    5771: "Cisco Systems, Inc.",
285    5842: "Cisco Systems",
286    11129: "Google, Inc",
287    16885: "Nortel Networks",
288}
289
290
291def load_protocols(filename, _fallback=None, _integer_base=10,
292                   _cls=DADict[int, str]):
293    # type: (str, Optional[bytes], int, type) -> DADict[int, str]
294    """"Parse /etc/protocols and return values as a dictionary."""
295    spaces = re.compile(b"[ \t]+|\n")
296    dct = _cls(_name=filename)  # type: DADict[int, str]
297
298    def _process_data(fdesc):
299        # type: (Iterator[bytes]) -> None
300        for line in fdesc:
301            try:
302                shrp = line.find(b"#")
303                if shrp >= 0:
304                    line = line[:shrp]
305                line = line.strip()
306                if not line:
307                    continue
308                lt = tuple(re.split(spaces, line))
309                if len(lt) < 2 or not lt[0]:
310                    continue
311                dct[int(lt[1], _integer_base)] = fixname(lt[0])
312            except Exception as e:
313                log_loading.info(
314                    "Couldn't parse file [%s]: line [%r] (%s)",
315                    filename,
316                    line,
317                    e,
318                )
319    try:
320        if not filename:
321            raise IOError
322        with open(filename, "rb") as fdesc:
323            _process_data(fdesc)
324    except IOError:
325        if _fallback:
326            _process_data(iter(_fallback.split(b"\n")))
327        else:
328            log_loading.info("Can't open %s file", filename)
329    return dct
330
331
332class EtherDA(DADict[int, str]):
333    # Backward compatibility: accept
334    # ETHER_TYPES["MY_GREAT_TYPE"] = 12
335    def __setitem__(self, attr, val):
336        # type: (int, str) -> None
337        if isinstance(attr, str):
338            attr, val = val, attr
339            warnings.warn(
340                "ETHER_TYPES now uses the integer value as key !",
341                DeprecationWarning
342            )
343        super(EtherDA, self).__setitem__(attr, val)
344
345    def __getitem__(self, attr):
346        # type: (int) -> Any
347        if isinstance(attr, str):
348            warnings.warn(
349                "Please use 'ETHER_TYPES.%s'" % attr,
350                DeprecationWarning
351            )
352            return super(EtherDA, self).__getattr__(attr)
353        return super(EtherDA, self).__getitem__(attr)
354
355
356def load_ethertypes(filename):
357    # type: (Optional[str]) -> EtherDA
358    """"Parse /etc/ethertypes and return values as a dictionary.
359    If unavailable, use the copy bundled with Scapy."""
360    from scapy.libs.ethertypes import DATA
361    prot = load_protocols(filename or "Scapy's backup ETHER_TYPES",
362                          _fallback=DATA,
363                          _integer_base=16,
364                          _cls=EtherDA)
365    return cast(EtherDA, prot)
366
367
368def load_services(filename):
369    # type: (str) -> Tuple[DADict[int, str], DADict[int, str]]
370    spaces = re.compile(b"[ \t]+|\n")
371    tdct = DADict(_name="%s-tcp" % filename)  # type: DADict[int, str]
372    udct = DADict(_name="%s-udp" % filename)  # type: DADict[int, str]
373    try:
374        with open(filename, "rb") as fdesc:
375            for line in fdesc:
376                try:
377                    shrp = line.find(b"#")
378                    if shrp >= 0:
379                        line = line[:shrp]
380                    line = line.strip()
381                    if not line:
382                        continue
383                    lt = tuple(re.split(spaces, line))
384                    if len(lt) < 2 or not lt[0]:
385                        continue
386                    dtct = None
387                    if lt[1].endswith(b"/tcp"):
388                        dtct = tdct
389                    elif lt[1].endswith(b"/udp"):
390                        dtct = udct
391                    else:
392                        continue
393                    port = lt[1].split(b'/')[0]
394                    name = fixname(lt[0])
395                    if b"-" in port:
396                        sport, eport = port.split(b"-")
397                        for i in range(int(sport), int(eport) + 1):
398                            dtct[i] = name
399                    else:
400                        dtct[int(port)] = name
401                except Exception as e:
402                    log_loading.warning(
403                        "Couldn't parse file [%s]: line [%r] (%s)",
404                        filename,
405                        line,
406                        e,
407                    )
408    except IOError:
409        log_loading.info("Can't open /etc/services file")
410    return tdct, udct
411
412
413class ManufDA(DADict[str, Tuple[str, str]]):
414    def ident(self, v):
415        # type: (Any) -> str
416        return fixname(v[0] if isinstance(v, tuple) else v)
417
418    def _get_manuf_couple(self, mac):
419        # type: (str) -> Tuple[str, str]
420        oui = ":".join(mac.split(":")[:3]).upper()
421        return self.d.get(oui, (mac, mac))
422
423    def _get_manuf(self, mac):
424        # type: (str) -> str
425        return self._get_manuf_couple(mac)[1]
426
427    def _get_short_manuf(self, mac):
428        # type: (str) -> str
429        return self._get_manuf_couple(mac)[0]
430
431    def _resolve_MAC(self, mac):
432        # type: (str) -> str
433        oui = ":".join(mac.split(":")[:3]).upper()
434        if oui in self:
435            return ":".join([self[oui][0]] + mac.split(":")[3:])
436        return mac
437
438    def lookup(self, mac):
439        # type: (str) -> Tuple[str, str]
440        """Find OUI name matching to a MAC"""
441        return self._get_manuf_couple(mac)
442
443    def reverse_lookup(self, name, case_sensitive=False):
444        # type: (str, bool) -> Dict[str, str]
445        """
446        Find all MACs registered to a OUI
447
448        :param name: the OUI name
449        :param case_sensitive: default to False
450        :returns: a dict of mac:tuples (Name, Extended Name)
451        """
452        if case_sensitive:
453            filtr = lambda x, l: any(x in z for z in l)  # type: Callable[[str, Tuple[str, str]], bool]  # noqa: E501
454        else:
455            name = name.lower()
456            filtr = lambda x, l: any(x in z.lower() for z in l)
457        return {k: v for k, v in six.iteritems(self.d)
458                if filtr(name, v)}
459
460    def __dir__(self):
461        # type: () -> List[str]
462        return [
463            "_get_manuf",
464            "_get_short_manuf",
465            "_resolve_MAC",
466            "loopkup",
467            "reverse_lookup",
468        ] + super(ManufDA, self).__dir__()
469
470
471def load_manuf(filename):
472    # type: (str) -> ManufDA
473    """
474    Loads manuf file from Wireshark.
475
476    :param filename: the file to load the manuf file from
477    :returns: a ManufDA filled object
478    """
479    manufdb = ManufDA(_name=filename)
480    with open(filename, "rb") as fdesc:
481        for line in fdesc:
482            try:
483                line = line.strip()
484                if not line or line.startswith(b"#"):
485                    continue
486                parts = line.split(None, 2)
487                ouib, shrt = parts[:2]
488                lng = parts[2].lstrip(b"#").strip() if len(parts) > 2 else b""
489                lng = lng or shrt
490                oui = plain_str(ouib)
491                manufdb[oui] = plain_str(shrt), plain_str(lng)
492            except Exception:
493                log_loading.warning("Couldn't parse one line from [%s] [%r]",
494                                    filename, line, exc_info=True)
495    return manufdb
496
497
498def select_path(directories, filename):
499    # type: (List[str], str) -> Optional[str]
500    """Find filename among several directories"""
501    for directory in directories:
502        path = os.path.join(directory, filename)
503        if os.path.exists(path):
504            return path
505    return None
506
507
508if WINDOWS:
509    IP_PROTOS = load_protocols(os.environ["SystemRoot"] + "\\system32\\drivers\\etc\\protocol")  # noqa: E501
510    TCP_SERVICES, UDP_SERVICES = load_services(os.environ["SystemRoot"] + "\\system32\\drivers\\etc\\services")  # noqa: E501
511    # Default values, will be updated by arch.windows
512    ETHER_TYPES = load_ethertypes(None)
513    MANUFDB = ManufDA()
514else:
515    IP_PROTOS = load_protocols("/etc/protocols")
516    ETHER_TYPES = load_ethertypes("/etc/ethertypes")
517    TCP_SERVICES, UDP_SERVICES = load_services("/etc/services")
518    MANUFDB = ManufDA()
519    manuf_path = select_path(
520        ['/usr', '/usr/local', '/opt', '/opt/wireshark',
521         '/Applications/Wireshark.app/Contents/Resources'],
522        "share/wireshark/manuf"
523    )
524    if manuf_path:
525        try:
526            MANUFDB = load_manuf(manuf_path)
527        except (IOError, OSError):
528            log_loading.warning("Cannot read wireshark manuf database")
529
530
531#####################
532#  knowledge bases  #
533#####################
534
535class KnowledgeBase:
536    def __init__(self, filename):
537        # type: (Optional[Any]) -> None
538        self.filename = filename
539        self.base = None  # type: Optional[str]
540
541    def lazy_init(self):
542        # type: () -> None
543        self.base = ""
544
545    def reload(self, filename=None):
546        # type: (Optional[Any]) -> None
547        if filename is not None:
548            self.filename = filename
549        oldbase = self.base
550        self.base = None
551        self.lazy_init()
552        if self.base is None:
553            self.base = oldbase
554
555    def get_base(self):
556        # type: () -> str
557        if self.base is None:
558            self.lazy_init()
559        return cast(str, self.base)
560