1# This file is part of Scapy
2# See http://www.secdev.org/projects/scapy for more information
3# Copyright (C) Philippe Biondi <phil@secdev.org>
4# This program is published under a GPLv2 license
5
6"""
7Clone of p0f passive OS fingerprinting
8"""
9
10from __future__ import absolute_import
11from __future__ import print_function
12import time
13import struct
14import os
15import socket
16import random
17
18from scapy.data import KnowledgeBase, select_path
19from scapy.config import conf
20from scapy.compat import raw
21from scapy.layers.inet import IP, TCP, TCPOptions
22from scapy.packet import NoPayload, Packet
23from scapy.error import warning, Scapy_Exception, log_runtime
24from scapy.volatile import RandInt, RandByte, RandNum, RandShort, RandString
25from scapy.sendrecv import sniff
26from scapy.modules import six
27from scapy.modules.six.moves import map, range
28if conf.route is None:
29    # unused import, only to initialize conf.route
30    import scapy.route  # noqa: F401
31
32_p0fpaths = ["/usr/local/etc/p0f", "/usr/share/p0f", "/opt/local"]
33
34conf.p0f_base = select_path(_p0fpaths, "p0f.fp")
35conf.p0fa_base = select_path(_p0fpaths, "p0fa.fp")
36conf.p0fr_base = select_path(_p0fpaths, "p0fr.fp")
37conf.p0fo_base = select_path(_p0fpaths, "p0fo.fp")
38
39
40###############
41#  p0f stuff  #
42###############
43
44# File format (according to p0f.fp) :
45#
46# wwww:ttt:D:ss:OOO...:QQ:OS:Details
47#
48# wwww    - window size
49# ttt     - initial TTL
50# D       - don't fragment bit  (0=unset, 1=set)
51# ss      - overall SYN packet size
52# OOO     - option value and order specification
53# QQ      - quirks list
54# OS      - OS genre
55# details - OS description
56
57class p0fKnowledgeBase(KnowledgeBase):
58    def __init__(self, filename):
59        KnowledgeBase.__init__(self, filename)
60        # self.ttl_range=[255]
61
62    def lazy_init(self):
63        try:
64            f = open(self.filename)
65        except IOError:
66            warning("Can't open base %s", self.filename)
67            return
68        try:
69            self.base = []
70            for line in f:
71                if line[0] in ["#", "\n"]:
72                    continue
73                line = tuple(line.split(":"))
74                if len(line) < 8:
75                    continue
76
77                def a2i(x):
78                    if x.isdigit():
79                        return int(x)
80                    return x
81                li = [a2i(e) for e in line[1:4]]
82                # if li[0] not in self.ttl_range:
83                #    self.ttl_range.append(li[0])
84                #    self.ttl_range.sort()
85                self.base.append((line[0], li[0], li[1], li[2], line[4],
86                                  line[5], line[6], line[7][:-1]))
87        except Exception:
88            warning("Can't parse p0f database (new p0f version ?)")
89            self.base = None
90        f.close()
91
92
93p0f_kdb, p0fa_kdb, p0fr_kdb, p0fo_kdb = None, None, None, None
94
95
96def p0f_load_knowledgebases():
97    global p0f_kdb, p0fa_kdb, p0fr_kdb, p0fo_kdb
98    p0f_kdb = p0fKnowledgeBase(conf.p0f_base)
99    p0fa_kdb = p0fKnowledgeBase(conf.p0fa_base)
100    p0fr_kdb = p0fKnowledgeBase(conf.p0fr_base)
101    p0fo_kdb = p0fKnowledgeBase(conf.p0fo_base)
102
103
104p0f_load_knowledgebases()
105
106
107def p0f_selectdb(flags):
108    # tested flags: S, R, A
109    if flags & 0x16 == 0x2:
110        # SYN
111        return p0f_kdb
112    elif flags & 0x16 == 0x12:
113        # SYN/ACK
114        return p0fa_kdb
115    elif flags & 0x16 in [0x4, 0x14]:
116        # RST RST/ACK
117        return p0fr_kdb
118    elif flags & 0x16 == 0x10:
119        # ACK
120        return p0fo_kdb
121    else:
122        return None
123
124
125def packet2p0f(pkt):
126    pkt = pkt.copy()
127    pkt = pkt.__class__(raw(pkt))
128    while pkt.haslayer(IP) and pkt.haslayer(TCP):
129        pkt = pkt.getlayer(IP)
130        if isinstance(pkt.payload, TCP):
131            break
132        pkt = pkt.payload
133
134    if not isinstance(pkt, IP) or not isinstance(pkt.payload, TCP):
135        raise TypeError("Not a TCP/IP packet")
136    # if pkt.payload.flags & 0x7 != 0x02: #S,!F,!R
137    #    raise TypeError("Not a SYN or SYN/ACK packet")
138
139    db = p0f_selectdb(pkt.payload.flags)
140
141    # t = p0f_kdb.ttl_range[:]
142    # t += [pkt.ttl]
143    # t.sort()
144    # ttl=t[t.index(pkt.ttl)+1]
145    ttl = pkt.ttl
146
147    ss = len(pkt)
148    # from p0f/config.h : PACKET_BIG = 100
149    if ss > 100:
150        if db == p0fr_kdb:
151            # p0fr.fp: "Packet size may be wildcarded. The meaning of
152            #           wildcard is, however, hardcoded as 'size >
153            #           PACKET_BIG'"
154            ss = '*'
155        else:
156            ss = 0
157    if db == p0fo_kdb:
158        # p0fo.fp: "Packet size MUST be wildcarded."
159        ss = '*'
160
161    ooo = ""
162    mss = -1
163    qqT = False
164    qqP = False
165    # qqBroken = False
166    ilen = (pkt.payload.dataofs << 2) - 20  # from p0f.c
167    for option in pkt.payload.options:
168        ilen -= 1
169        if option[0] == "MSS":
170            ooo += "M" + str(option[1]) + ","
171            mss = option[1]
172            # FIXME: qqBroken
173            ilen -= 3
174        elif option[0] == "WScale":
175            ooo += "W" + str(option[1]) + ","
176            # FIXME: qqBroken
177            ilen -= 2
178        elif option[0] == "Timestamp":
179            if option[1][0] == 0:
180                ooo += "T0,"
181            else:
182                ooo += "T,"
183            if option[1][1] != 0:
184                qqT = True
185            ilen -= 9
186        elif option[0] == "SAckOK":
187            ooo += "S,"
188            ilen -= 1
189        elif option[0] == "NOP":
190            ooo += "N,"
191        elif option[0] == "EOL":
192            ooo += "E,"
193            if ilen > 0:
194                qqP = True
195        else:
196            if isinstance(option[0], str):
197                ooo += "?%i," % TCPOptions[1][option[0]]
198            else:
199                ooo += "?%i," % option[0]
200            # FIXME: ilen
201    ooo = ooo[:-1]
202    if ooo == "":
203        ooo = "."
204
205    win = pkt.payload.window
206    if mss != -1:
207        if mss != 0 and win % mss == 0:
208            win = "S" + str(win / mss)
209        elif win % (mss + 40) == 0:
210            win = "T" + str(win / (mss + 40))
211    win = str(win)
212
213    qq = ""
214
215    if db == p0fr_kdb:
216        if pkt.payload.flags & 0x10 == 0x10:
217            # p0fr.fp: "A new quirk, 'K', is introduced to denote
218            #           RST+ACK packets"
219            qq += "K"
220    # The two next cases should also be only for p0f*r*, but although
221    # it's not documented (or I have not noticed), p0f seems to
222    # support the '0' and 'Q' quirks on any databases (or at the least
223    # "classical" p0f.fp).
224    if pkt.payload.seq == pkt.payload.ack:
225        # p0fr.fp: "A new quirk, 'Q', is used to denote SEQ number
226        #           equal to ACK number."
227        qq += "Q"
228    if pkt.payload.seq == 0:
229        # p0fr.fp: "A new quirk, '0', is used to denote packets
230        #           with SEQ number set to 0."
231        qq += "0"
232    if qqP:
233        qq += "P"
234    if pkt.id == 0:
235        qq += "Z"
236    if pkt.options != []:
237        qq += "I"
238    if pkt.payload.urgptr != 0:
239        qq += "U"
240    if pkt.payload.reserved != 0:
241        qq += "X"
242    if pkt.payload.ack != 0:
243        qq += "A"
244    if qqT:
245        qq += "T"
246    if db == p0fo_kdb:
247        if pkt.payload.flags & 0x20 != 0:
248            # U
249            # p0fo.fp: "PUSH flag is excluded from 'F' quirk checks"
250            qq += "F"
251    else:
252        if pkt.payload.flags & 0x28 != 0:
253            # U or P
254            qq += "F"
255    if db != p0fo_kdb and not isinstance(pkt.payload.payload, NoPayload):
256        # p0fo.fp: "'D' quirk is not checked for."
257        qq += "D"
258    # FIXME : "!" - broken options segment: not handled yet
259
260    if qq == "":
261        qq = "."
262
263    return (db, (win, ttl, pkt.flags.DF, ss, ooo, qq))
264
265
266def p0f_correl(x, y):
267    d = 0
268    # wwww can be "*" or "%nn". "Tnn" and "Snn" should work fine with
269    # the x[0] == y[0] test.
270    d += (x[0] == y[0] or y[0] == "*" or (y[0][0] == "%" and x[0].isdigit() and (int(x[0]) % int(y[0][1:])) == 0))  # noqa: E501
271    # ttl
272    d += (y[1] >= x[1] and y[1] - x[1] < 32)
273    for i in [2, 5]:
274        d += (x[i] == y[i] or y[i] == '*')
275    # '*' has a special meaning for ss
276    d += x[3] == y[3]
277    xopt = x[4].split(",")
278    yopt = y[4].split(",")
279    if len(xopt) == len(yopt):
280        same = True
281        for i in range(len(xopt)):
282            if not (xopt[i] == yopt[i] or
283                    (len(yopt[i]) == 2 and len(xopt[i]) > 1 and
284                     yopt[i][1] == "*" and xopt[i][0] == yopt[i][0]) or
285                    (len(yopt[i]) > 2 and len(xopt[i]) > 1 and
286                     yopt[i][1] == "%" and xopt[i][0] == yopt[i][0] and
287                     int(xopt[i][1:]) % int(yopt[i][2:]) == 0)):
288                same = False
289                break
290        if same:
291            d += len(xopt)
292    return d
293
294
295@conf.commands.register
296def p0f(pkt):
297    """Passive OS fingerprinting: which OS emitted this TCP packet ?
298p0f(packet) -> accuracy, [list of guesses]
299"""
300    db, sig = packet2p0f(pkt)
301    if db:
302        pb = db.get_base()
303    else:
304        pb = []
305    if not pb:
306        warning("p0f base empty.")
307        return []
308    # s = len(pb[0][0])
309    r = []
310    max = len(sig[4].split(",")) + 5
311    for b in pb:
312        d = p0f_correl(sig, b)
313        if d == max:
314            r.append((b[6], b[7], b[1] - pkt[IP].ttl))
315    return r
316
317
318def prnp0f(pkt):
319    """Calls p0f and returns a user-friendly output"""
320    # we should print which DB we use
321    try:
322        r = p0f(pkt)
323    except Exception:
324        return
325    if r == []:
326        r = ("UNKNOWN", "[" + ":".join(map(str, packet2p0f(pkt)[1])) + ":?:?]", None)  # noqa: E501
327    else:
328        r = r[0]
329    uptime = None
330    try:
331        uptime = pkt2uptime(pkt)
332    except Exception:
333        pass
334    if uptime == 0:
335        uptime = None
336    res = pkt.sprintf("%IP.src%:%TCP.sport% - " + r[0] + " " + r[1])
337    if uptime is not None:
338        res += pkt.sprintf(" (up: " + str(uptime / 3600) + " hrs)\n  -> %IP.dst%:%TCP.dport% (%TCP.flags%)")  # noqa: E501
339    else:
340        res += pkt.sprintf("\n  -> %IP.dst%:%TCP.dport% (%TCP.flags%)")
341    if r[2] is not None:
342        res += " (distance " + str(r[2]) + ")"
343    print(res)
344
345
346@conf.commands.register
347def pkt2uptime(pkt, HZ=100):
348    """Calculate the date the machine which emitted the packet booted using TCP timestamp  # noqa: E501
349pkt2uptime(pkt, [HZ=100])"""
350    if not isinstance(pkt, Packet):
351        raise TypeError("Not a TCP packet")
352    if isinstance(pkt, NoPayload):
353        raise TypeError("Not a TCP packet")
354    if not isinstance(pkt, TCP):
355        return pkt2uptime(pkt.payload)
356    for opt in pkt.options:
357        if opt[0] == "Timestamp":
358            # t = pkt.time - opt[1][0] * 1.0/HZ
359            # return time.ctime(t)
360            t = opt[1][0] / HZ
361            return t
362    raise TypeError("No timestamp option")
363
364
365def p0f_impersonate(pkt, osgenre=None, osdetails=None, signature=None,
366                    extrahops=0, mtu=1500, uptime=None):
367    """Modifies pkt so that p0f will think it has been sent by a
368specific OS.  If osdetails is None, then we randomly pick up a
369personality matching osgenre. If osgenre and signature are also None,
370we use a local signature (using p0f_getlocalsigs). If signature is
371specified (as a tuple), we use the signature.
372
373For now, only TCP Syn packets are supported.
374Some specifications of the p0f.fp file are not (yet) implemented."""
375    pkt = pkt.copy()
376    # pkt = pkt.__class__(raw(pkt))
377    while pkt.haslayer(IP) and pkt.haslayer(TCP):
378        pkt = pkt.getlayer(IP)
379        if isinstance(pkt.payload, TCP):
380            break
381        pkt = pkt.payload
382
383    if not isinstance(pkt, IP) or not isinstance(pkt.payload, TCP):
384        raise TypeError("Not a TCP/IP packet")
385
386    db = p0f_selectdb(pkt.payload.flags)
387    if osgenre:
388        pb = db.get_base()
389        if pb is None:
390            pb = []
391        pb = [x for x in pb if x[6] == osgenre]
392        if osdetails:
393            pb = [x for x in pb if x[7] == osdetails]
394    elif signature:
395        pb = [signature]
396    else:
397        pb = p0f_getlocalsigs()[db]
398    if db == p0fr_kdb:
399        # 'K' quirk <=> RST+ACK
400        if pkt.payload.flags & 0x4 == 0x4:
401            pb = [x for x in pb if 'K' in x[5]]
402        else:
403            pb = [x for x in pb if 'K' not in x[5]]
404    if not pb:
405        raise Scapy_Exception("No match in the p0f database")
406    pers = pb[random.randint(0, len(pb) - 1)]
407
408    # options (we start with options because of MSS)
409    # Take the options already set as "hints" to use in the new packet if we
410    # can. MSS, WScale and Timestamp can all be wildcarded in a signature, so
411    # we'll use the already-set values if they're valid integers.
412    orig_opts = dict(pkt.payload.options)
413    int_only = lambda val: val if isinstance(val, six.integer_types) else None
414    mss_hint = int_only(orig_opts.get('MSS'))
415    wscale_hint = int_only(orig_opts.get('WScale'))
416    ts_hint = [int_only(o) for o in orig_opts.get('Timestamp', (None, None))]
417
418    options = []
419    if pers[4] != '.':
420        for opt in pers[4].split(','):
421            if opt[0] == 'M':
422                # MSS might have a maximum size because of window size
423                # specification
424                if pers[0][0] == 'S':
425                    maxmss = (2**16 - 1) // int(pers[0][1:])
426                else:
427                    maxmss = (2**16 - 1)
428                # disregard hint if out of range
429                if mss_hint and not 0 <= mss_hint <= maxmss:
430                    mss_hint = None
431                # If we have to randomly pick up a value, we cannot use
432                # scapy RandXXX() functions, because the value has to be
433                # set in case we need it for the window size value. That's
434                # why we use random.randint()
435                if opt[1:] == '*':
436                    if mss_hint is not None:
437                        options.append(('MSS', mss_hint))
438                    else:
439                        options.append(('MSS', random.randint(1, maxmss)))
440                elif opt[1] == '%':
441                    coef = int(opt[2:])
442                    if mss_hint is not None and mss_hint % coef == 0:
443                        options.append(('MSS', mss_hint))
444                    else:
445                        options.append((
446                            'MSS', coef * random.randint(1, maxmss // coef)))
447                else:
448                    options.append(('MSS', int(opt[1:])))
449            elif opt[0] == 'W':
450                if wscale_hint and not 0 <= wscale_hint < 2**8:
451                    wscale_hint = None
452                if opt[1:] == '*':
453                    if wscale_hint is not None:
454                        options.append(('WScale', wscale_hint))
455                    else:
456                        options.append(('WScale', RandByte()))
457                elif opt[1] == '%':
458                    coef = int(opt[2:])
459                    if wscale_hint is not None and wscale_hint % coef == 0:
460                        options.append(('WScale', wscale_hint))
461                    else:
462                        options.append((
463                            'WScale', coef * RandNum(min=1, max=(2**8 - 1) // coef)))  # noqa: E501
464                else:
465                    options.append(('WScale', int(opt[1:])))
466            elif opt == 'T0':
467                options.append(('Timestamp', (0, 0)))
468            elif opt == 'T':
469                # Determine first timestamp.
470                if uptime is not None:
471                    ts_a = uptime
472                elif ts_hint[0] and 0 < ts_hint[0] < 2**32:
473                    # Note: if first ts is 0, p0f registers it as "T0" not "T",
474                    # hence we don't want to use the hint if it was 0.
475                    ts_a = ts_hint[0]
476                else:
477                    ts_a = random.randint(120, 100 * 60 * 60 * 24 * 365)
478                # Determine second timestamp.
479                if 'T' not in pers[5]:
480                    ts_b = 0
481                elif ts_hint[1] and 0 < ts_hint[1] < 2**32:
482                    ts_b = ts_hint[1]
483                else:
484                    # FIXME: RandInt() here does not work (bug (?) in
485                    # TCPOptionsField.m2i often raises "OverflowError:
486                    # long int too large to convert to int" in:
487                    #    oval = struct.pack(ofmt, *oval)"
488                    # Actually, this is enough to often raise the error:
489                    #    struct.pack('I', RandInt())
490                    ts_b = random.randint(1, 2**32 - 1)
491                options.append(('Timestamp', (ts_a, ts_b)))
492            elif opt == 'S':
493                options.append(('SAckOK', ''))
494            elif opt == 'N':
495                options.append(('NOP', None))
496            elif opt == 'E':
497                options.append(('EOL', None))
498            elif opt[0] == '?':
499                if int(opt[1:]) in TCPOptions[0]:
500                    optname = TCPOptions[0][int(opt[1:])][0]
501                    optstruct = TCPOptions[0][int(opt[1:])][1]
502                    options.append((optname,
503                                    struct.unpack(optstruct,
504                                                  RandString(struct.calcsize(optstruct))._fix())))  # noqa: E501
505                else:
506                    options.append((int(opt[1:]), ''))
507            # FIXME: qqP not handled
508            else:
509                warning("unhandled TCP option %s", opt)
510            pkt.payload.options = options
511
512    # window size
513    if pers[0] == '*':
514        pkt.payload.window = RandShort()
515    elif pers[0].isdigit():
516        pkt.payload.window = int(pers[0])
517    elif pers[0][0] == '%':
518        coef = int(pers[0][1:])
519        pkt.payload.window = coef * RandNum(min=1, max=(2**16 - 1) // coef)
520    elif pers[0][0] == 'T':
521        pkt.payload.window = mtu * int(pers[0][1:])
522    elif pers[0][0] == 'S':
523        # needs MSS set
524        mss = [x for x in options if x[0] == 'MSS']
525        if not mss:
526            raise Scapy_Exception("TCP window value requires MSS, and MSS option not set")  # noqa: E501
527        pkt.payload.window = mss[0][1] * int(pers[0][1:])
528    else:
529        raise Scapy_Exception('Unhandled window size specification')
530
531    # ttl
532    pkt.ttl = pers[1] - extrahops
533    # DF flag
534    pkt.flags |= (2 * pers[2])
535    # FIXME: ss (packet size) not handled (how ? may be with D quirk
536    # if present)
537    # Quirks
538    if pers[5] != '.':
539        for qq in pers[5]:
540            # FIXME: not handled: P, I, X, !
541            # T handled with the Timestamp option
542            if qq == 'Z':
543                pkt.id = 0
544            elif qq == 'U':
545                pkt.payload.urgptr = RandShort()
546            elif qq == 'A':
547                pkt.payload.ack = RandInt()
548            elif qq == 'F':
549                if db == p0fo_kdb:
550                    pkt.payload.flags |= 0x20  # U
551                else:
552                    pkt.payload.flags |= random.choice([8, 32, 40])  # P/U/PU
553            elif qq == 'D' and db != p0fo_kdb:
554                pkt /= conf.raw_layer(load=RandString(random.randint(1, 10)))  # XXX p0fo.fp  # noqa: E501
555            elif qq == 'Q':
556                pkt.payload.seq = pkt.payload.ack
557            # elif qq == '0': pkt.payload.seq = 0
558        # if db == p0fr_kdb:
559        # '0' quirk is actually not only for p0fr.fp (see
560        # packet2p0f())
561    if '0' in pers[5]:
562        pkt.payload.seq = 0
563    elif pkt.payload.seq == 0:
564        pkt.payload.seq = RandInt()
565
566    while pkt.underlayer:
567        pkt = pkt.underlayer
568    return pkt
569
570
571def p0f_getlocalsigs():
572    """This function returns a dictionary of signatures indexed by p0f
573db (e.g., p0f_kdb, p0fa_kdb, ...) for the local TCP/IP stack.
574
575You need to have your firewall at least accepting the TCP packets
576from/to a high port (30000 <= x <= 40000) on your loopback interface.
577
578Please note that the generated signatures come from the loopback
579interface and may (are likely to) be different than those generated on
580"normal" interfaces."""
581    pid = os.fork()
582    port = random.randint(30000, 40000)
583    if pid > 0:
584        # parent: sniff
585        result = {}
586
587        def addresult(res):
588            # TODO: wildcard window size in some cases? and maybe some
589            # other values?
590            if res[0] not in result:
591                result[res[0]] = [res[1]]
592            else:
593                if res[1] not in result[res[0]]:
594                    result[res[0]].append(res[1])
595        # XXX could we try with a "normal" interface using other hosts
596        iface = conf.route.route('127.0.0.1')[0]
597        # each packet is seen twice: S + RA, S + SA + A + FA + A
598        # XXX are the packets also seen twice on non Linux systems ?
599        count = 14
600        pl = sniff(iface=iface, filter='tcp and port ' + str(port), count=count, timeout=3)  # noqa: E501
601        for pkt in pl:
602            for elt in packet2p0f(pkt):
603                addresult(elt)
604        os.waitpid(pid, 0)
605    elif pid < 0:
606        log_runtime.error("fork error")
607    else:
608        # child: send
609        # XXX erk
610        time.sleep(1)
611        s1 = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM)
612        # S & RA
613        try:
614            s1.connect(('127.0.0.1', port))
615        except socket.error:
616            pass
617        # S, SA, A, FA, A
618        s1.bind(('127.0.0.1', port))
619        s1.connect(('127.0.0.1', port))
620        # howto: get an RST w/o ACK packet
621        s1.close()
622        os._exit(0)
623    return result
624