Improve path detection & Add backup ethertypes

Author: gpotter2

.coveragerc

@@ -14,3 +14,4 @@ omit =
     # Libraries
     */scapy/modules/six.py
     */scapy/modules/winpcapy.py
+    */scapy/modules/ethertypes.py

.coveragerc.tox

@@ -7,6 +7,7 @@ omit =
     # Scapy external modules
     scapy/modules/six.py
     scapy/modules/winpcapy.py
+    scapy/modules/ethertypes.py
     # .tox specific path
     .tox/*
     # OS specific paths

scapy/data.py

@@ -154,39 +154,49 @@
 MTU = 0xffff  # a.k.a give me all you have
 
 
-def load_protocols(filename, _integer_base=10):
+def load_protocols(filename, _fallback=None, _integer_base=10):
     """"Parse /etc/protocols and return values as a dictionary."""
     spaces = re.compile(b"[ \t]+|\n")
     dct = DADict(_name=filename)
+
+    def _process_data(fdesc):
+        for line in fdesc:
+            try:
+                shrp = line.find(b"#")
+                if shrp >= 0:
+                    line = line[:shrp]
+                line = line.strip()
+                if not line:
+                    continue
+                lt = tuple(re.split(spaces, line))
+                if len(lt) < 2 or not lt[0]:
+                    continue
+                dct[lt[0]] = int(lt[1], _integer_base)
+            except Exception as e:
+                log_loading.info(
+                    "Couldn't parse file [%s]: line [%r] (%s)",
+                    filename,
+                    line,
+                    e,
+                )
     try:
+        if not filename:
+            raise IOError
         with open(filename, "rb") as fdesc:
-            for line in fdesc:
-                try:
-                    shrp = line.find(b"#")
-                    if shrp >= 0:
-                        line = line[:shrp]
-                    line = line.strip()
-                    if not line:
-                        continue
-                    lt = tuple(re.split(spaces, line))
-                    if len(lt) < 2 or not lt[0]:
-                        continue
-                    dct[lt[0]] = int(lt[1], _integer_base)
-                except Exception as e:
-                    log_loading.info(
-                        "Couldn't parse file [%s]: line [%r] (%s)",
-                        filename,
-                        line,
-                        e,
-                    )
+            _process_data(fdesc)
     except IOError:
-        log_loading.info("Can't open %s file", filename)
+        if _fallback:
+            _process_data(_fallback.split(b"\n"))
+        else:
+            log_loading.info("Can't open %s file", filename)
     return dct
 
 
 def load_ethertypes(filename):
-    """"Parse /etc/ethertypes and return values as a dictionary."""
-    return load_protocols(filename, _integer_base=16)
+    """"Parse /etc/ethertypes and return values as a dictionary.
+    If unavailable, use the copy bundled with Scapy."""
+    from scapy.modules.ethertypes import DATA
+    return load_protocols(filename, _fallback=DATA, _integer_base=16)
 
 
 def load_services(filename):
@@ -288,28 +298,35 @@ def load_manuf(filename):
     return manufdb
 
 
+def select_path(directories, filename):
+    """Find filename among several directories"""
+    for directory in directories:
+        path = os.path.join(directory, filename)
+        if os.path.exists(path):
+            return path
+
+
 if WINDOWS:
     IP_PROTOS = load_protocols(os.environ["SystemRoot"] + "\\system32\\drivers\\etc\\protocol")  # noqa: E501
     TCP_SERVICES, UDP_SERVICES = load_services(os.environ["SystemRoot"] + "\\system32\\drivers\\etc\\services")  # noqa: E501
     # Default values, will be updated by arch.windows
-    ETHER_TYPES = DADict()
+    ETHER_TYPES = load_ethertypes(None)
     MANUFDB = ManufDA()
 else:
     IP_PROTOS = load_protocols("/etc/protocols")
     ETHER_TYPES = load_ethertypes("/etc/ethertypes")
     TCP_SERVICES, UDP_SERVICES = load_services("/etc/services")
     MANUFDB = None
-    for prefix in ['/usr', '/usr/local', '/opt', '/opt/wireshark',
-                   '/Applications/Wireshark.app/Contents/Resources']:
+    manuf_path = select_path(
+        ['/usr', '/usr/local', '/opt', '/opt/wireshark',
+         '/Applications/Wireshark.app/Contents/Resources'],
+        "share/wireshark/manuf"
+    )
+    if manuf_path:
         try:
-            MANUFDB = load_manuf(os.path.join(prefix, "share", "wireshark",
-                                              "manuf"))
-            if MANUFDB:
-                break
-        except (IOError, OSError):  # Same as above
-            pass
-    if not MANUFDB:
-        log_loading.warning("Cannot read wireshark manuf database")
+            MANUFDB = load_manuf(manuf_path)
+        except (IOError, OSError):
+            log_loading.warning("Cannot read wireshark manuf database")
 
 
 #####################

scapy/modules/ethertypes.py

@@ -0,0 +1,138 @@
+# This file is part of Scapy
+# See http://www.secdev.org/projects/scapy for more information
+
+"""
+/*
+ * Copyright (c) 1982, 1986, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)if_ether.h	8.1 (Berkeley) 6/10/93
+ */
+"""
+
+# This file contains data automatically generated using
+# scapy/tools/generate_ethertypes.py
+# based on OpenBSD public source.
+
+DATA = b"""
+#
+# Ethernet frame types
+#       This file describes some of the various Ethernet
+#       protocol types that are used on Ethernet networks.
+#
+# This list could be found on:
+#         http://www.iana.org/assignments/ethernet-numbers
+#         http://www.iana.org/assignments/ieee-802-numbers
+#
+# <name>    <hexnumber> <alias1>...<alias35> #Comment
+#
+8023            0004                         # IEEE 802.3 packet
+PUP             0200                         # Xerox PUP protocol - see 0A00
+PUPAT           0200                         # PUP Address Translation - see 0A01
+NS              0600                         # XNS
+NSAT            0601                         # XNS Address Translation (3Mb only)
+DLOG1           0660                         # DLOG (?)
+DLOG2           0661                         # DLOG (?)
+IPv4            0800                         # IP protocol
+X75             0801                         # X.75 Internet
+NBS             0802                         # NBS Internet
+ECMA            0803                         # ECMA Internet
+CHAOS           0804                         # CHAOSnet
+X25             0805                         # X.25 Level 3
+ARP             0806                         # Address resolution protocol
+FRARP           0808                         # Frame Relay ARP (RFC1701)
+VINES           0BAD                         # Banyan VINES
+TRAIL           1000                         # Trailer packet
+DCA             1234                         # DCA - Multicast
+VALID           1600                         # VALID system protocol
+RCL             1995                         # Datapoint Corporation (RCL lan protocol)
+NBPCC           3C04                         # 3Com NBP Connect complete not registered
+NBPDG           3C07                         # 3Com NBP Datagram (like XNS IDP) not registered
+PCS             4242                         # PCS Basic Block Protocol
+IMLBL           4C42                         # Information Modes Little Big LAN
+MOPDL           6001                         # DEC MOP dump/load
+MOPRC           6002                         # DEC MOP remote console
+LAT             6004                         # DEC LAT
+SCA             6007                         # DEC LAVC, SCA
+AMBER           6008                         # DEC AMBER
+RAWFR           6559                         # Raw Frame Relay (RFC1701)
+UBDL            7000                         # Ungermann-Bass download
+UBNIU           7001                         # Ungermann-Bass NIUs
+UBNMC           7003                         # Ungermann-Bass ??? (NMC to/from UB Bridge)
+UBBST           7005                         # Ungermann-Bass Bridge Spanning Tree
+OS9             7007                         # OS/9 Microware
+RACAL           7030                         # Racal-Interlan
+HP              8005                         # HP Probe
+TIGAN           802F                         # Tigan, Inc.
+DECAM           8048                         # DEC Availability Manager for Distributed Systems DECamds (but someone at DEC says not)
+VEXP            805B                         # Stanford V Kernel exp.
+VPROD           805C                         # Stanford V Kernel prod.
+ES              805D                         # Evans & Sutherland
+VEECO           8067                         # Veeco Integrated Auto.
+ATT             8069                         # AT&T
+MATRA           807A                         # Matra
+DDE             807B                         # Dansk Data Elektronik
+MERIT           807C                         # Merit Internodal (or Univ of Michigan?)
+ATALK           809B                         # AppleTalk
+PACER           80C6                         # Pacer Software
+SNA             80D5                         # IBM SNA Services over Ethernet
+RETIX           80F2                         # Retix
+AARP            80F3                         # AppleTalk AARP
+VLAN            8100                         # IEEE 802.1Q VLAN tagging (XXX conflicts)
+BOFL            8102                         # Wellfleet; BOFL (Breath OF Life) pkts [every 5-10 secs.]
+HAYES           8130                         # Hayes Microcomputers (XXX which?)
+VGLAB           8131                         # VG Laboratory Systems
+IPX             8137                         # Novell (old) NetWare IPX (ECONFIG E option)
+MUMPS           813F                         # M/MUMPS data sharing
+FLIP            8146                         # Vrije Universiteit (NL) FLIP (Fast Local Internet Protocol)
+NCD             8149                         # Network Computing Devices
+ALPHA           814A                         # Alpha Micro
+SNMP            814C                         # SNMP over Ethernet (see RFC1089)
+XTP             817D                         # Protocol Engines XTP
+SGITW           817E                         # SGI/Time Warner prop.
+STP             8181                         # Scheduled Transfer STP, HIPPI-ST
+IPv6            86DD                         # IP protocol version 6
+RDP             8739                         # Control Technology Inc. RDP Without IP
+MICP            873A                         # Control Technology Inc. Mcast Industrial Ctrl Proto.
+IPAS            876C                         # IP Autonomous Systems (RFC1701)
+SLOW            8809                         # 803.3ad slow protocols (LACP/Marker)
+PPP             880B                         # PPP (obsolete by PPPOE)
+MPLS            8847                         # MPLS Unicast
+AXIS            8856                         # Axis Communications AB proprietary bootstrap/config
+PPPOE           8864                         # PPP Over Ethernet Session Stage
+PAE             888E                         # 802.1X Port Access Entity
+AOE             88A2                         # ATA over Ethernet
+QINQ            88A8                         # 802.1ad VLAN stacking
+LLDP            88CC                         # Link Layer Discovery Protocol
+PBB             88E7                         # 802.1Q Provider Backbone Bridging
+XNSSM           9001                         # 3Com (Formerly Bridge Communications), XNS Systems Management
+TCPSM           9002                         # 3Com (Formerly Bridge Communications), TCP/IP Systems Management
+DEBNI           AAAA                         # DECNET? Used by VAX 6220 DEBNI
+SONIX           FAF5                         # Sonix Arpeggio
+VITAL           FF00                         # BBN VITAL-LanBridge cache wakeups
+MAX             FFFF                         # Maximum valid ethernet type, reserved
+"""

scapy/modules/p0f.py

@@ -15,7 +15,7 @@
 import socket
 import random
 
-from scapy.data import KnowledgeBase
+from scapy.data import KnowledgeBase, select_path
 from scapy.config import conf
 from scapy.compat import raw
 from scapy.layers.inet import IP, TCP, TCPOptions
@@ -29,10 +29,12 @@
     # unused import, only to initialize conf.route
     import scapy.route  # noqa: F401
 
-conf.p0f_base = "/etc/p0f/p0f.fp"
-conf.p0fa_base = "/etc/p0f/p0fa.fp"
-conf.p0fr_base = "/etc/p0f/p0fr.fp"
-conf.p0fo_base = "/etc/p0f/p0fo.fp"
+_p0fpaths = ["/etc/p0f", "/usr/share/p0f", "/opt/local"]
+
+conf.p0f_base = select_path(_p0fpaths, "p0f.fp")
+conf.p0fa_base = select_path(_p0fpaths, "p0fa.fp")
+conf.p0fr_base = select_path(_p0fpaths, "p0fr.fp")
+conf.p0fo_base = select_path(_p0fpaths, "p0fo.fp")
 
 
 ###############

scapy/modules/winpcapy.py

@@ -10,9 +10,9 @@
 #-------------------------------------------------------------------------------  # noqa: E501
 # Modified for scapy's usage - Mainly to support Npcap/Monitor mode
 
-## This file is part of Scapy
-## See http://www.secdev.org/projects/scapy for more information
-## This program is published under a GPLv2 license
+# This file is part of Scapy
+# See http://www.secdev.org/projects/scapy for more information
+# This program is published under a GPLv2 license
 
 from ctypes import *
 from ctypes.util import find_library

scapy/tools/generate_ethertypes.py

@@ -0,0 +1,48 @@
+# This file is part of Scapy
+# See http://www.secdev.org/projects/scapy for more information
+# Copyright (C) Philippe Biondi <[email protected]>
+# Copyright (C) Gabriel Potter <[email protected]>
+# This program is published under a GPLv2 license
+
+"""Generate the ethertypes file (/etc/ethertypes)
+based on the OpenBSD source.
+
+It allows to have a file with the format of
+http://git.netfilter.org/ebtables/plain/ethertypes
+but up-to-date.
+"""
+
+import re
+import urllib.request
+
+URL = "https://github.com/raw/openbsd/src/master/sys/net/ethertypes.h"  # noqa: E501
+
+with urllib.request.urlopen(URL) as stream:
+    DATA = stream.read()
+
+reg = rb".*ETHERTYPE_([^\s]+)\s.0x([0-9A-Fa-f]+).*\/\*(.*)\*\/"
+COMPILED = b"""#
+# Ethernet frame types
+#       This file describes some of the various Ethernet
+#       protocol types that are used on Ethernet networks.
+#
+# This list could be found on:
+#         http://www.iana.org/assignments/ethernet-numbers
+#         http://www.iana.org/assignments/ieee-802-numbers
+#
+# <name>    <hexnumber> <alias1>...<alias35> #Comment
+#
+"""
+for line in DATA.split(b"\n"):
+    match = re.match(reg, line)
+    if match:
+        name = match.group(1).ljust(16)
+        number = match.group(2).upper()
+        comment = match.group(3).strip()
+        compiled_line = (b"%b%b" + b" " * 25 + b"# %b\n") % (
+            name, number, comment
+        )
+        COMPILED += compiled_line
+
+with open("ethertypes", "wb") as output:
+    print("Written: %s" % output.write(COMPILED))

tox.ini

@@ -125,3 +125,4 @@ commands = flake8 scapy/
 ignore = E731, W504
 exclude = scapy/modules/six.py,
           scapy/modules/winpcapy.py
+          scapy/modules/ethertypes.py