From cdef6c691f6a29c1f6f3352e3a1073dc62075fba Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Thu, 5 Nov 2020 06:42:21 +0100 Subject: [PATCH 01/28] works on mac --- UDPComms.py | 38 ++++++++++++++++++++++++++++++++------ __init__.py | 1 + 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index a5dc023..c7fd94e 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -10,7 +10,7 @@ import socket import struct -from collections import namedtuple +from enum import Enum, auto import msgpack @@ -26,21 +26,43 @@ MAX_SIZE = 65507 + +class Target(Enum): + LOCALHOST = auto() + BROADCAST = auto() + MULTICAST = auto() + class Publisher: - def __init__(self, port): + broadcast_ip = "10.0.0.255" + muticast_ip = '224.1.1.1' + def __init__(self, port, target = Target.BROADCAST): """ Create a Publisher Object Arguments: port -- the port to publish the messages on + target -- where to publish the message """ + print("magic") self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - self.broadcast_ip = "127.0.0.1" - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - self.broadcast_ip = "10.0.0.255" + if target == Target.BROADCAST: + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + self.ip = self.broadcast_ip + + elif target == Target.LOCALHOST: + self.ip = "127.0.0.1" + + elif target == Target.MULTICAST: + # raise NotImplementedError + self.ip = self.muticast_ip + self.sock.setsockopt(socket.IPPROTO_IP, + socket.IP_MULTICAST_TTL, + struct.pack('b', 1)) + else: + raise ValueError self.sock.settimeout(0.2) - self.sock.connect((self.broadcast_ip, port)) + self.sock.connect((self.ip, port)) self.port = port @@ -55,6 +77,7 @@ def __del__(self): class Subscriber: + muticast_ip = '224.1.1.1' def __init__(self, port, timeout=0.2): """ Create a Subscriber Object @@ -76,6 +99,9 @@ def __init__(self, port, timeout=0.2): if hasattr(socket, "SO_REUSEPORT"): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + mreq = struct.pack("4sl", socket.inet_aton(self.muticast_ip), socket.INADDR_ANY) + self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + self.sock.settimeout(timeout) self.sock.bind(("", port)) diff --git a/__init__.py b/__init__.py index 54b1443..b9c4e51 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ from .UDPComms import Publisher from .UDPComms import Subscriber from .UDPComms import timeout +from .UDPComms import Target From d31f07e6bdf2158216fd3e7bf624bcdefadeaeb2 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Thu, 5 Nov 2020 06:48:30 +0100 Subject: [PATCH 02/28] todos --- UDPComms.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/UDPComms.py b/UDPComms.py index c7fd94e..db4151b 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -26,6 +26,10 @@ MAX_SIZE = 65507 +##TODO: +# Nicer configuration +# Test on ubuntu and debian +# Documentation (targets, and security disclaimer) class Target(Enum): LOCALHOST = auto() From e68f2fc394d2954d63d40d07279fd20cd74f2387 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Fri, 6 Nov 2020 06:42:44 +0100 Subject: [PATCH 03/28] changed target method --- UDPComms.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index db4151b..a0ba20a 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -26,6 +26,9 @@ MAX_SIZE = 65507 +DEFAULT_BROADCAST = "10.0.0.255" +DEFAULT_MULTICAST = "239.255.20.22" + ##TODO: # Nicer configuration # Test on ubuntu and debian @@ -37,26 +40,25 @@ class Target(Enum): MULTICAST = auto() class Publisher: - broadcast_ip = "10.0.0.255" - muticast_ip = '224.1.1.1' - def __init__(self, port, target = Target.BROADCAST): + broadcast_ip = DEFAULT_BROADCAST + muticast_ip = DEFAULT_MULTICAST + target = Target.BROADCAST + def __init__(self, port): """ Create a Publisher Object Arguments: port -- the port to publish the messages on - target -- where to publish the message """ - print("magic") self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - if target == Target.BROADCAST: + if self.target == Target.BROADCAST: self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.ip = self.broadcast_ip - elif target == Target.LOCALHOST: + elif self.target == Target.LOCALHOST: self.ip = "127.0.0.1" - elif target == Target.MULTICAST: + elif self.target == Target.MULTICAST: # raise NotImplementedError self.ip = self.muticast_ip self.sock.setsockopt(socket.IPPROTO_IP, @@ -81,7 +83,8 @@ def __del__(self): class Subscriber: - muticast_ip = '224.1.1.1' + muticast_ip = DEFAULT_MULTICAST + broadcast_ip = DEFAULT_BROADCAST def __init__(self, port, timeout=0.2): """ Create a Subscriber Object From 380b14404d9e04385fe8ce34814c461568a9a744 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Fri, 6 Nov 2020 06:50:57 +0100 Subject: [PATCH 04/28] docmumentaiotn --- README.md | 60 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f1ab3f2..a3e0da4 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ Currently it works in python 2 and 3 but it should be relatively simple to exten This new verison of the library automatically determines the type of the message and trasmits it along with it, so the subscribers can decode it correctly. While faster to prototype with then systems with explicit type declaration (such as ROS) its easy to shoot yourself in the foot if types are mismatched between publisher and subscriber. +## Usage + ### To Send Messages ``` >>> from UDPComms import Publisher @@ -74,19 +76,44 @@ The port the subscriber will be listen on. - `timeout` If the `recv()` method don't get a message in `timeout` seconds it throws a `UDPComms.timeout` exception -### Rover +## Configuring targets -The library also comes with the `rover` command that can be used to interact with the messages manually. +By default the messages are sent using a broadcast on the `10.0.0.X` subnet +``` +>>> from UDPComms import Publisher, Target +>>> Publisher.target = Target.MULTICAST +>>> a = Publisher(5500) +>>> a.send("testing") +``` -| Command | Descripion | -|---------|------------| -| `rover peek port` | print messages sent on port `port` | -| `rover poke port rate` | send messages to `port` once every `rate` milliseconds. Type message in json format and press return | +TODO + +``` +>>> from UDPComms import Publisher, Target +>>> Publisher.target = Target.LOCALHOST +>>> a = Publisher(5500) +>>> a.send("testing") +``` + +### Developing without hardware -There are more commands used for starting and stoping services described in [this repo](https://github.com/stanfordroboticsclub/RPI-Setup/blob/master/README.md) -### To Install + +### Developing without hardware + +Because this library expects you to be connected to the robot () network you won't be able to send messages between two programs on your computer without any other hardware connected. You can get around this by forcing your (unused) ethernet interface to get an ip on the rover network without anything being connected to it. On my computer you can do this using this command: + +`sudo ifconfig en1 10.0.0.52 netmask 255.255.255.0` + +Note that the exact command depends which interface on your computer is unused and what ip you want. So only use this if you know what you are doing. + +If you have internet access a slightly cleaner way to do it is to setup [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) on your development computer and simply connect to a development network (given if you are the only computer there) + + + + +## Installation ``` $git clone https://github.com/stanfordroboticsclub/UDPComms.git @@ -101,15 +128,22 @@ $git pull $sudo bash install.sh ``` -### Developing without hardware -Because this library expects you to be connected to the robot (`10.0.0.X`) network you won't be able to send messages between two programs on your computer without any other hardware connected. You can get around this by forcing your (unused) ethernet interface to get an ip on the rover network without anything being connected to it. On my computer you can do this using this command: -`sudo ifconfig en1 10.0.0.52 netmask 255.255.255.0` +## Extras + +### Rover + +The library also comes with the `rover` command that can be used to interact with the messages manually. + +| Command | Descripion | +|---------|------------| +| `rover peek port` | print messages sent on port `port` | +| `rover poke port rate` | send messages to `port` once every `rate` milliseconds. Type message in json format and press return | + +There are more commands used for starting and stoping services described in [this repo](https://github.com/stanfordroboticsclub/RPI-Setup/blob/master/README.md) -Note that the exact command depends which interface on your computer is unused and what ip you want. So only use this if you know what you are doing. -If you have internet access a slightly cleaner way to do it is to setup [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) on your development computer and simply connect to a development network (given if you are the only computer there) ### Known issues: From 41f9f7e9bdcb57535701e1c2e30792b8ad264475 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Mon, 21 Dec 2020 21:16:52 +0100 Subject: [PATCH 05/28] added scope abstraction --- UDPComms.py | 104 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index dd3d8d8..d970c56 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -26,52 +26,80 @@ MAX_SIZE = 65507 -DEFAULT_BROADCAST = "10.0.0.255" -DEFAULT_MULTICAST = "239.255.20.22" - ##TODO: -# Nicer configuration # Test on ubuntu and debian # Documentation (targets, and security disclaimer) -class Target(Enum): - LOCALHOST = auto() - BROADCAST = auto() - MULTICAST = auto() +class Scope: + class Target(Enum): + LOCALHOST = auto() + BROADCAST = auto() + MULTICAST = auto() + UNICAST = auto() + ALL = auto() + + def __init__(self, option, ip): + assert isinstance(option, Target) + self.option = option + self.ip = ip + + @classmethod + def Local(cls): return cls(Target.LOCALHOST, "127.0.0.1") + + @classmethod + def Broadcast(cls, ip): return cls(Target.BROADCAST,ip) + + @classmethod + def Multicast(cls, ip): return cls(Target.MULTICAST,ip) + + @classmethod + def Unicast(cls, ip): return cls(Target.UNICAST,ip) + + @classmethod + def All(cls): return cls(Target.ALL,"") + + def isPublishable(self): + return self.option in (Target.LOCALHOST, + Target.BROADCAS, + Target.MULTICAST, + Target.UNICAST) + + def isSubscribable(self): + return self.option in (Target.LOCALHOST, + Target.BROADCAST, + Target.MULTICAST, + Target.ALL) + + def isBroadcast(self): return self.option is Target.BROADCAST + def isMulticast(self): return self.option is Target.MULTICAST + def isLocal(self): return self.option is Target.LOCALHOST + def isUnicast(self): return self.option is Target.UNICAST + def isAll(self): return self.option is Target.ALL class Publisher: - broadcast_ip = DEFAULT_BROADCAST - muticast_ip = DEFAULT_MULTICAST - target = Target.BROADCAST - def __init__(self, port): + def __init__(self, port, scope = Scope.Local() ): """ Create a Publisher Object Arguments: port -- the port to publish the messages on - ip -- the ip to send the messages to + scope -- the scope the messages will be sent to """ - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + assert scope.isPublishable() - if self.target == Target.BROADCAST: - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - self.ip = self.broadcast_ip + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.scope = scope + self.port = port - elif self.target == Target.LOCALHOST: - self.ip = "127.0.0.1" + if self.scope.isBroadcast(): + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - elif self.target == Target.MULTICAST: - # raise NotImplementedError - self.ip = self.muticast_ip + if self.scope.isMulticast(): self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, struct.pack('b', 1)) - else: - raise ValueError self.sock.settimeout(0.2) - self.sock.connect((self.ip, port)) - - self.port = port + self.sock.connect((self.scope.ip, port)) def send(self, obj): """ Publish a message. The obj can be any nesting of standard python types """ @@ -84,34 +112,36 @@ def __del__(self): class Subscriber: - muticast_ip = DEFAULT_MULTICAST - broadcast_ip = DEFAULT_BROADCAST - def __init__(self, port, timeout=0.2): + def __init__(self, port, timeout=0.2, scope = Scope.All() ): """ Create a Subscriber Object Arguments: port -- the port to listen to messages on timeout -- how long to wait before a message is considered out of date + scope -- where to expect messages to come from """ - self.max_size = MAX_SIZE - + assert scope.isSubscribable() + self.scope = scope self.port = port self.timeout = timeout self.last_data = None self.last_time = float('-inf') - self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if hasattr(socket, "SO_REUSEPORT"): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - mreq = struct.pack("4sl", socket.inet_aton(self.muticast_ip), socket.INADDR_ANY) - self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + if self.scope.isBroadcast(): + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + + if self.scope.isMulticast(): + mreq = struct.pack("4sl", socket.inet_aton(self.scope.ip), socket.INADDR_ANY) + self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) self.sock.settimeout(timeout) - self.sock.bind(("", port)) + self.sock.bind((scope.ip, port)) def recv(self): """ Receive a single message from the socket buffer. It blocks for up to timeout seconds. From a5755979e6c72579d3de97906df8581aaa510dcb Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 22 Dec 2020 02:48:40 +0100 Subject: [PATCH 06/28] fixes --- UDPComms.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index d970c56..c69237e 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -38,43 +38,45 @@ class Target(Enum): UNICAST = auto() ALL = auto() + locals().update(Target.__members__) + def __init__(self, option, ip): - assert isinstance(option, Target) + assert isinstance(option, self.Target) self.option = option self.ip = ip @classmethod - def Local(cls): return cls(Target.LOCALHOST, "127.0.0.1") + def Local(cls): return cls(cls.LOCALHOST, "127.0.0.1") @classmethod - def Broadcast(cls, ip): return cls(Target.BROADCAST,ip) + def Broadcast(cls, ip): return cls(cls.BROADCAST,ip) @classmethod - def Multicast(cls, ip): return cls(Target.MULTICAST,ip) + def Multicast(cls, ip): return cls(cls.MULTICAST,ip) @classmethod - def Unicast(cls, ip): return cls(Target.UNICAST,ip) + def Unicast(cls, ip): return cls(cls.UNICAST,ip) @classmethod - def All(cls): return cls(Target.ALL,"") + def All(cls): return cls(cls.ALL,"") def isPublishable(self): - return self.option in (Target.LOCALHOST, - Target.BROADCAS, - Target.MULTICAST, - Target.UNICAST) + return self.option in (self.LOCALHOST, + self.BROADCAST, + self.MULTICAST, + self.UNICAST) def isSubscribable(self): - return self.option in (Target.LOCALHOST, - Target.BROADCAST, - Target.MULTICAST, - Target.ALL) - - def isBroadcast(self): return self.option is Target.BROADCAST - def isMulticast(self): return self.option is Target.MULTICAST - def isLocal(self): return self.option is Target.LOCALHOST - def isUnicast(self): return self.option is Target.UNICAST - def isAll(self): return self.option is Target.ALL + return self.option in (self.LOCALHOST, + self.BROADCAST, + self.MULTICAST, + self.ALL) + + def isBroadcast(self): return self.option is self.BROADCAST + def isMulticast(self): return self.option is self.MULTICAST + def isLocal(self): return self.option is self.LOCALHOST + def isUnicast(self): return self.option is self.UNICAST + def isAll(self): return self.option is self.ALL class Publisher: def __init__(self, port, scope = Scope.Local() ): From 826eb6fd9b7d1b46707dcab18032bd86850e5b8e Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 22 Dec 2020 03:13:11 +0100 Subject: [PATCH 07/28] cleaned up scopes --- UDPComms.py | 32 ++++++++++++++++++-------------- __init__.py | 2 +- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index c69237e..f6a538d 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -25,24 +25,25 @@ timeout = socket.timeout MAX_SIZE = 65507 +DEFAULT_MULTICAST = "239.255.20.22" ##TODO: # Test on ubuntu and debian # Documentation (targets, and security disclaimer) class Scope: - class Target(Enum): + class Mode(Enum): LOCALHOST = auto() BROADCAST = auto() MULTICAST = auto() UNICAST = auto() ALL = auto() - locals().update(Target.__members__) + locals().update(Mode.__members__) - def __init__(self, option, ip): - assert isinstance(option, self.Target) - self.option = option + def __init__(self, mode, ip): + assert isinstance(mode, self.Mode) + self.mode = mode self.ip = ip @classmethod @@ -52,7 +53,7 @@ def Local(cls): return cls(cls.LOCALHOST, "127.0.0.1") def Broadcast(cls, ip): return cls(cls.BROADCAST,ip) @classmethod - def Multicast(cls, ip): return cls(cls.MULTICAST,ip) + def Multicast(cls, ip=DEFAULT_MULTICAST): return cls(cls.MULTICAST,ip) @classmethod def Unicast(cls, ip): return cls(cls.UNICAST,ip) @@ -61,22 +62,25 @@ def Unicast(cls, ip): return cls(cls.UNICAST,ip) def All(cls): return cls(cls.ALL,"") def isPublishable(self): - return self.option in (self.LOCALHOST, + return self.mode in (self.LOCALHOST, self.BROADCAST, self.MULTICAST, self.UNICAST) def isSubscribable(self): - return self.option in (self.LOCALHOST, + return self.mode in (self.LOCALHOST, self.BROADCAST, self.MULTICAST, self.ALL) - def isBroadcast(self): return self.option is self.BROADCAST - def isMulticast(self): return self.option is self.MULTICAST - def isLocal(self): return self.option is self.LOCALHOST - def isUnicast(self): return self.option is self.UNICAST - def isAll(self): return self.option is self.ALL + def isBroadcast(self): return self.mode is self.BROADCAST + def isMulticast(self): return self.mode is self.MULTICAST + def isLocal(self): return self.mode is self.LOCALHOST + def isUnicast(self): return self.mode is self.UNICAST + def isAll(self): return self.mode is self.ALL + + def __repr__(self): + return "Scope(Scope." + self.mode.name + ", '" + self.ip + "')" class Publisher: def __init__(self, port, scope = Scope.Local() ): @@ -150,7 +154,7 @@ def recv(self): If no message is received before timeout it raises a UDPComms.timeout exception""" try: - self.last_data, address = self.sock.recvfrom(self.max_size) + self.last_data, address = self.sock.recvfrom(MAX_SIZE) except BlockingIOError: raise socket.timeout("no messages in buffer and called with timeout = 0") diff --git a/__init__.py b/__init__.py index b9c4e51..75a0458 100644 --- a/__init__.py +++ b/__init__.py @@ -1,4 +1,4 @@ from .UDPComms import Publisher from .UDPComms import Subscriber from .UDPComms import timeout -from .UDPComms import Target +from .UDPComms import Scope From ae3ab9194a08320c14696ec6600d11efd872cc7b Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 22 Dec 2020 04:23:47 +0100 Subject: [PATCH 08/28] documentation --- README.md | 55 +++++++++++++++++++++-------------------------------- UDPComms.py | 2 +- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 2fa63a6..d23f049 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@ # UDPComms -This is a simple library to enable communication between different processes (potentially on different machines) over a network using UDP. It's goals a simplicity and easy of understanding and reliability. It works for devices on the `10.0.0.X` subnet although this can easiliy be changed. +This is a simple library to enable communication between different processes (potentially on different machines) over a network using UDP. It's goals a simplicity and easy of understanding. It can send messages to processes running on the same device or to proceses on other devices on the same network (configured using scopes). Currently it works in python 2 and 3 but it should be relatively simple to extend it to other languages such as C (to run on embeded devices) or Julia (to interface with faster solvers). -This new verison of the library automatically determines the type of the message and trasmits it along with it, so the subscribers can decode it correctly. While faster to prototype with then systems with explicit type declaration (such as ROS) its easy to shoot yourself in the foot if types are mismatched between publisher and subscriber. +The library automatically determines the type of the message and trasmits it along with it, so the subscribers can decode it correctly. While faster to prototype with then systems with explicit type declaration (such as ROS) its easy to shoot yourself in the foot if types are mismatched between publisher and subscriber. + +Note that this library doesn't provide any message security. Unless you are using `Scope.Local()` on both ends, not only can anyone on your network evesdrop on messages they can also spoof messages very easily. ## Usage @@ -17,8 +19,9 @@ This new verison of the library automatically determines the type of the message ### To Receive Messages -#### recv Method +**TLDR** - you probably want the `get()` method +#### recv Method Note: before using the `Subsciber.recv()` method read about the `Subsciber.get()` and understand the difference between them. The `Subsciber.recv()` method will pull a message from the socket buffer and it won't necessary be the most recent message. If you are calling it too slowly and there is a lot of messages you will be getting old messages. The `Subsciber.recv()` can also block for up to `timeout` seconds messing up timing. @@ -64,11 +67,12 @@ Although UDPComms isn't ideal for commands that need to be processed in order (a >>> for message in messages: >>> print("got", message) ``` +- ### Publisher Arguments - `port` -The port the messages will be sent on. If you are part of Stanford Student Robotics make sure there isn't any port conflicts by checking the `UDP Ports` sheet of the [CS Comms System](https://docs.google.com/spreadsheets/d/1pqduUwYa1_sWiObJDrvCCz4Al3pl588ytE4u-Dwa6Pw/edit?usp=sharing) document. If you are not I recommend keep track of your port numbers somewhere. It's possible that in the future UDPComms will have a system of naming (with a string) as opposed to numbering publishers. -- `ip` By default UDPComms sends to the `10.0.0.X` subnet, but can be changed to a different ip using this argument. Set to localhost (`127.0.0.1`) for development on the same computer. +The port the messages will be sent on. I recommend keep track of your port numbers somewhere. It's possible that in the future UDPComms will have a system of naming (with a string) as opposed to numbering publishers. +- `scope` Leave this as the defualt (`Scope.Local()`) to send messages to only this computer. Set to `Scope.Multicast()` to send to others on the network. See Scopes explained for details ### Subscriber Arguments @@ -76,42 +80,29 @@ The port the messages will be sent on. If you are part of Stanford Student Robot The port the subscriber will be listen on. - `timeout` If the `recv()` method don't get a message in `timeout` seconds it throws a `UDPComms.timeout` exception +- `scope` Leave this as the defualt (`Scope.All()`) to receive messages from everyone. Set to `Scope.Multicast()` to register that we want to get multicast messages from other devices . See Scopes explained for details -## Configuring targets - -By default the messages are sent using a broadcast on the `10.0.0.X` subnet - -``` ->>> from UDPComms import Publisher, Target ->>> Publisher.target = Target.MULTICAST ->>> a = Publisher(5500) ->>> a.send("testing") -``` - -TODO - -``` ->>> from UDPComms import Publisher, Target ->>> Publisher.target = Target.LOCALHOST ->>> a = Publisher(5500) ->>> a.send("testing") -``` - -### Developing without hardware +## Scopes Explained +**TLDR** - use `Publisher(port, scope=Scope.Multicast())` and `Subscriber(port, scope=Scope.Multicast())` to get messages to work between different devices -### Developing without hardware +The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms sends messages only to processes on the same device (`Scope.Local()`). To send messages to other computers on the same network use `Scope.Multicast()`. This will default to using the multicast group `239.255.20.22`, but it can be changed by passing an argument. -Because this library expects you to be connected to the robot () network you won't be able to send messages between two programs on your computer without any other hardware connected. You can get around this by forcing your (unused) ethernet interface to get an ip on the rover network without anything being connected to it. On my computer you can do this using this command: +Older versions of the library defulted to using a broadcast on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour use `Scope.Broadcast('10.0.0.255')` -`sudo ifconfig en1 10.0.0.52 netmask 255.255.255.0` +Here are all the avalible options: -Note that the exact command depends which interface on your computer is unused and what ip you want. So only use this if you know what you are doing. +- `Scope.Local` - Messages meant for this device only +- `Scope.Multicast`- Messages meant for the specified multicast group +- `Scope.Broadcast` - Messages meant for all devices on this subnet +- `Scope.Unicast` - Publisher only argument that sends messages to a specific ip address only +- `Scope.All` - Subscriber only that listens to packets from all (except for multicast) sources -If you have internet access a slightly cleaner way to do it is to setup [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) on your development computer and simply connect to a development network (given if you are the only computer there) +### Connecting to devices on different networks +If you want to talk to devices aross the internet use [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) to get them all on the same virtual network and you should be able to use `Scope.Multicast()` from there ## Installation @@ -129,8 +120,6 @@ $git pull $sudo bash install.sh ``` - - ## Extras ### Rover diff --git a/UDPComms.py b/UDPComms.py index f6a538d..a6dfcb3 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -139,7 +139,7 @@ def __init__(self, port, timeout=0.2, scope = Scope.All() ): if hasattr(socket, "SO_REUSEPORT"): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if self.scope.isBroadcast(): + if self.scope.isBroadcast() or self.scope.isAll(): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) if self.scope.isMulticast(): From 149784afaa9faf16cfb1dcd2c11f9e25724b878b Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 22 Dec 2020 04:25:13 +0100 Subject: [PATCH 09/28] doc --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index d23f049..db2648c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # UDPComms -This is a simple library to enable communication between different processes (potentially on different machines) over a network using UDP. It's goals a simplicity and easy of understanding. It can send messages to processes running on the same device or to proceses on other devices on the same network (configured using scopes). - -Currently it works in python 2 and 3 but it should be relatively simple to extend it to other languages such as C (to run on embeded devices) or Julia (to interface with faster solvers). +This is a simple library to enable communication between different processes (potentially on different machines, see `scopes`) over a network using UDP. It's goals a simplicity and easy of understanding. Currently it works in python 2 and 3 but it should be relatively simple to extend it to other languages such as C (to run on embeded devices) or Julia (to interface with faster solvers). The library automatically determines the type of the message and trasmits it along with it, so the subscribers can decode it correctly. While faster to prototype with then systems with explicit type declaration (such as ROS) its easy to shoot yourself in the foot if types are mismatched between publisher and subscriber. From 943cf5d7f62e8a973ba40401aa90369c12ab9f37 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 24 Jan 2021 20:55:38 -0800 Subject: [PATCH 10/28] simpler interface --- UDPComms.py | 101 +++++++++++++++++++--------------------------------- 1 file changed, 37 insertions(+), 64 deletions(-) diff --git a/UDPComms.py b/UDPComms.py index a6dfcb3..d3e6a9b 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -26,86 +26,53 @@ MAX_SIZE = 65507 DEFAULT_MULTICAST = "239.255.20.22" +DEFAULT_BROADCAST = "10.0.0.255" ##TODO: # Test on ubuntu and debian # Documentation (targets, and security disclaimer) -class Scope: - class Mode(Enum): - LOCALHOST = auto() - BROADCAST = auto() - MULTICAST = auto() - UNICAST = auto() - ALL = auto() - - locals().update(Mode.__members__) - - def __init__(self, mode, ip): - assert isinstance(mode, self.Mode) - self.mode = mode - self.ip = ip - - @classmethod - def Local(cls): return cls(cls.LOCALHOST, "127.0.0.1") - - @classmethod - def Broadcast(cls, ip): return cls(cls.BROADCAST,ip) - - @classmethod - def Multicast(cls, ip=DEFAULT_MULTICAST): return cls(cls.MULTICAST,ip) - - @classmethod - def Unicast(cls, ip): return cls(cls.UNICAST,ip) - - @classmethod - def All(cls): return cls(cls.ALL,"") - - def isPublishable(self): - return self.mode in (self.LOCALHOST, - self.BROADCAST, - self.MULTICAST, - self.UNICAST) - - def isSubscribable(self): - return self.mode in (self.LOCALHOST, - self.BROADCAST, - self.MULTICAST, - self.ALL) - - def isBroadcast(self): return self.mode is self.BROADCAST - def isMulticast(self): return self.mode is self.MULTICAST - def isLocal(self): return self.mode is self.LOCALHOST - def isUnicast(self): return self.mode is self.UNICAST - def isAll(self): return self.mode is self.ALL - - def __repr__(self): - return "Scope(Scope." + self.mode.name + ", '" + self.ip + "')" +class Scope(Enum): + LOCAL = auto() + NETWORK = auto() + BROADCAST = auto() class Publisher: - def __init__(self, port, scope = Scope.Local() ): + MULTICAST_IP = DEFAULT_MULTICAST + BROADCAST_IP = DEFAULT_BROADCAST + + def __init__(self, port, scope = Scope.LOCAL): """ Create a Publisher Object Arguments: port -- the port to publish the messages on scope -- the scope the messages will be sent to """ - assert scope.isPublishable() self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.scope = scope self.port = port - if self.scope.isBroadcast(): + if self.scope == Scope.BROADCAST: + ip = self.BROADCAST_IP self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - if self.scope.isMulticast(): + else: + ip = self.MULTICAST_IP + if self.scope == Scope.LOCAL: + ttl = 0 + elif self.scope == Scope.NETWORK: + ttl = 1 + else: + raise ValueError("Unknown Scope") + self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, - struct.pack('b', 1)) + struct.pack('b', ttl)) + self.sock.settimeout(0.2) - self.sock.connect((self.scope.ip, port)) + self.sock.connect((ip, port)) def send(self, obj): """ Publish a message. The obj can be any nesting of standard python types """ @@ -118,7 +85,10 @@ def __del__(self): class Subscriber: - def __init__(self, port, timeout=0.2, scope = Scope.All() ): + MULTICAST_IP = DEFAULT_MULTICAST + BROADCAST_IP = DEFAULT_BROADCAST + + def __init__(self, port, timeout=0.2, scope = Scope.LOCAL ): """ Create a Subscriber Object Arguments: @@ -126,7 +96,6 @@ def __init__(self, port, timeout=0.2, scope = Scope.All() ): timeout -- how long to wait before a message is considered out of date scope -- where to expect messages to come from """ - assert scope.isSubscribable() self.scope = scope self.port = port self.timeout = timeout @@ -139,15 +108,19 @@ def __init__(self, port, timeout=0.2, scope = Scope.All() ): if hasattr(socket, "SO_REUSEPORT"): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if self.scope.isBroadcast() or self.scope.isAll(): + if self.scope == Scope.BROADCAST: + ip = self.BROADCAST_IP self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - if self.scope.isMulticast(): - mreq = struct.pack("4sl", socket.inet_aton(self.scope.ip), socket.INADDR_ANY) + elif self.scope == Scope.LOCAL or self.scope == Scope.NETWORK: + ip = self.MULTICAST_IP + mreq = struct.pack("4sl", socket.inet_aton(ip), socket.INADDR_ANY) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + else: + raise ValueError("Unknown Scope") self.sock.settimeout(timeout) - self.sock.bind((scope.ip, port)) + self.sock.bind((ip, port)) def recv(self): """ Receive a single message from the socket buffer. It blocks for up to timeout seconds. @@ -167,7 +140,7 @@ def get(self): try: self.sock.settimeout(0) while True: - self.last_data, address = self.sock.recvfrom(self.max_size) + self.last_data, address = self.sock.recvfrom(MAX_SIZE) self.last_time = monotonic() except socket.error: pass @@ -188,7 +161,7 @@ def get_list(self): try: self.sock.settimeout(0) while True: - self.last_data, address = self.sock.recvfrom(self.max_size) + self.last_data, address = self.sock.recvfrom(MAX_SIZE) self.last_time = monotonic() msg = msgpack.loads(self.last_data, raw=USING_PYTHON_2) msg_bufer.append(msg) From db1df8116d2f8ab66841b90b01202f4d0010d5b3 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 24 Jan 2021 21:53:39 -0800 Subject: [PATCH 11/28] attempted tests --- UDPComms.py | 2 +- test.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 test.py diff --git a/UDPComms.py b/UDPComms.py index d3e6a9b..043756b 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -76,7 +76,7 @@ def __init__(self, port, scope = Scope.LOCAL): def send(self, obj): """ Publish a message. The obj can be any nesting of standard python types """ - msg = msgpack.dumps(obj, use_bin_type=False) + msg = msgpack.dumps(obj, use_bin_type= not USING_PYTHON_2) assert len(msg) < MAX_SIZE, "Encoded message too big!" self.sock.send(msg) diff --git a/test.py b/test.py new file mode 100644 index 0000000..c170215 --- /dev/null +++ b/test.py @@ -0,0 +1,69 @@ + +from UDPComms import Publisher, Subscriber, Scope, timeout +import time + +import unittest + +class SingleProcessTestCase(unittest.TestCase): + + def setUp(self): + self.local_pub = Publisher(8001, scope = Scope.LOCAL ) + self.local_sub = Subscriber(8001, timeout=1, scope = Scope.LOCAL ) + self.local_sub2 = Subscriber(8001, timeout=1, scope = Scope.LOCAL ) + + def tearDown(self): + pass + + def test_simple(self): + msg = [123, "testing", {"one":2} ] + + self.local_pub.send( msg ) + recv_msg = self.local_sub.recv() + + self.assertEqual(msg, recv_msg) + + def test_bytes(self): + msg = ["testing", b"bytes"] + + self.local_pub.send( msg ) + recv_msg = self.local_sub.recv() + + self.assertEqual(msg, recv_msg) + + def test_dual_recv(self): + msg = [124, "testing", {"one":3} ] + + self.local_pub.send( msg ) + recv_msg = self.local_sub.recv() + recv_msg2 = self.local_sub2.recv() + + self.assertEqual(msg, recv_msg) + self.assertEqual(msg, recv_msg2) + + def test_get_recv(self): + msg = [123, "testing", {"one":2} ] + + self.local_pub.send( msg ) + time.sleep(0.1) + + self.assertEqual(msg, self.local_sub.get()) + self.assertEqual(msg, self.local_sub.get()) + + msg2 = "hello" + self.local_pub.send( msg2 ) + time.sleep(0.1) + self.assertEqual(msg2, self.local_sub.get()) + + time.sleep(1) + + with self.assertRaises(timeout): + self.local_sub.get() + + + +if __name__ == "__main__": + unittest.main() + + + +# socket.gethostbyname('www.google.com') From 7c3bb6f3b3984d63d998b0682f75efe6f2668691 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 24 Jan 2021 23:27:17 -0800 Subject: [PATCH 12/28] utliproc test --- test.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test.py b/test.py index c170215..59348aa 100644 --- a/test.py +++ b/test.py @@ -1,6 +1,7 @@ from UDPComms import Publisher, Subscriber, Scope, timeout import time +import subprocess import unittest @@ -59,7 +60,53 @@ def test_get_recv(self): with self.assertRaises(timeout): self.local_sub.get() +class SingleProcessNetworkTestCase(unittest.TestCase): + def setUp(self): + self.local_pub = Publisher(8002, scope = Scope.NETWORK ) + self.local_sub = Subscriber(8002, timeout=1, scope = Scope.NETWORK ) + + def tearDown(self): + pass + + def test_simple(self): + msg = [123, "testing", {"one":2} ] + + self.local_pub.send( msg ) + recv_msg = self.local_sub.recv() + + self.assertEqual(msg, recv_msg) + + + +class MultiProcessTestCase(unittest.TestCase): + + def setUp(self): + mirror_server = """(""" \ + """echo "from UDPComms import *";""" \ + """echo "incomming = Subscriber(8003, timeout=5, scope = Scope.NETWORK )";""" \ + """echo "outgoing = Publisher(8000, scope = Scope.NETWORK )";""" \ + """echo "while 1: outgoing.send(incomming.recv())";""" \ + """) | python""" + + self.local_pub = Publisher(8003, scope = Scope.LOCAL ) + self.return_path = Subscriber(8000, timeout = 5, scope = Scope.LOCAL ) + + self.p = subprocess.Popen(mirror_server, shell=True, stderr = subprocess.DEVNULL) + time.sleep(1) + + def tearDown(self): + self.p.wait() + + def test_simple(self): + msg = [123, "testing", {"one":2} ] + + self.local_pub.send( msg ) + self.assertEqual(msg, self.return_path.recv()) + + msg2 = "hi" + self.local_pub.send(msg2) + self.assertEqual(msg2, self.return_path.recv()) if __name__ == "__main__": unittest.main() From 28d4379d1aab4b9ce69e76949c087e989f1f3355 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 24 Jan 2021 23:35:34 -0800 Subject: [PATCH 13/28] fix --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 59348aa..6564f37 100644 --- a/test.py +++ b/test.py @@ -93,7 +93,7 @@ def setUp(self): self.return_path = Subscriber(8000, timeout = 5, scope = Scope.LOCAL ) self.p = subprocess.Popen(mirror_server, shell=True, stderr = subprocess.DEVNULL) - time.sleep(1) + time.sleep(1)# gives enough time to initialize def tearDown(self): self.p.wait() From a08c98bd1acf3d21ea6e81396f4db62e21bb6f1f Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 7 Feb 2021 14:22:42 -0800 Subject: [PATCH 14/28] subprocess clean --- test.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/test.py b/test.py index 6564f37..f3ca9f0 100644 --- a/test.py +++ b/test.py @@ -2,6 +2,7 @@ from UDPComms import Publisher, Subscriber, Scope, timeout import time import subprocess +import os import unittest @@ -82,20 +83,28 @@ def test_simple(self): class MultiProcessTestCase(unittest.TestCase): def setUp(self): - mirror_server = """(""" \ - """echo "from UDPComms import *";""" \ - """echo "incomming = Subscriber(8003, timeout=5, scope = Scope.NETWORK )";""" \ - """echo "outgoing = Publisher(8000, scope = Scope.NETWORK )";""" \ - """echo "while 1: outgoing.send(incomming.recv())";""" \ - """) | python""" + mirror_server = b""" +from UDPComms import *; +import sys +incomming = Subscriber(8003, timeout=10, scope = Scope.NETWORK ); +outgoing = Publisher(8000, scope = Scope.NETWORK ); +print("ready"); +sys.stdout.flush() +while 1: outgoing.send(incomming.recv()); + +""" + + self.p = subprocess.Popen('python', stdin = subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + self.p.stdin.write(mirror_server) + self.p.stdin.close() + self.p.stdout.readline() #wait for program to be ready self.local_pub = Publisher(8003, scope = Scope.LOCAL ) self.return_path = Subscriber(8000, timeout = 5, scope = Scope.LOCAL ) - self.p = subprocess.Popen(mirror_server, shell=True, stderr = subprocess.DEVNULL) - time.sleep(1)# gives enough time to initialize - def tearDown(self): + self.p.stdout.close() + self.p.terminate() self.p.wait() def test_simple(self): From 6a68bc4933ea0c75b38b1897852d24c5688f734a Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 7 Feb 2021 14:26:00 -0800 Subject: [PATCH 15/28] python3 --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index f3ca9f0..8a858a0 100644 --- a/test.py +++ b/test.py @@ -94,7 +94,7 @@ def setUp(self): """ - self.p = subprocess.Popen('python', stdin = subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + self.p = subprocess.Popen('python3', stdin = subprocess.PIPE, stdout=subprocess.PIPE) self.p.stdin.write(mirror_server) self.p.stdin.close() self.p.stdout.readline() #wait for program to be ready From db1bb1e1a3211768f68e8a7b562f0419258e7d59 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 7 Feb 2021 15:09:16 -0800 Subject: [PATCH 16/28] any_addr bind --- UDPComms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/UDPComms.py b/UDPComms.py index 043756b..c0d029a 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -110,17 +110,19 @@ def __init__(self, port, timeout=0.2, scope = Scope.LOCAL ): if self.scope == Scope.BROADCAST: ip = self.BROADCAST_IP + bind_ip = self.BROADCAST_IP self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) elif self.scope == Scope.LOCAL or self.scope == Scope.NETWORK: ip = self.MULTICAST_IP + bind_ip = "0.0.0.0" mreq = struct.pack("4sl", socket.inet_aton(ip), socket.INADDR_ANY) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) else: raise ValueError("Unknown Scope") self.sock.settimeout(timeout) - self.sock.bind((ip, port)) + self.sock.bind((bind_ip, port)) def recv(self): """ Receive a single message from the socket buffer. It blocks for up to timeout seconds. From 4f16c0d2fcd6179c23ce6ed789068b09b12dedad Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sat, 13 Feb 2021 04:58:56 -0800 Subject: [PATCH 17/28] readme --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index db2648c..823eb4d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # UDPComms -This is a simple library to enable communication between different processes (potentially on different machines, see `scopes`) over a network using UDP. It's goals a simplicity and easy of understanding. Currently it works in python 2 and 3 but it should be relatively simple to extend it to other languages such as C (to run on embeded devices) or Julia (to interface with faster solvers). +This is a simple library to enable communication between different processes (potentially on different machines) over a network using UDP. It's goals a simplicity and easy of understanding. Currently it works in python 2 and 3 but it should be relatively simple to extend it to other languages such as C (to run on embeded devices) or Julia (to interface with faster solvers). The library automatically determines the type of the message and trasmits it along with it, so the subscribers can decode it correctly. While faster to prototype with then systems with explicit type declaration (such as ROS) its easy to shoot yourself in the foot if types are mismatched between publisher and subscriber. -Note that this library doesn't provide any message security. Unless you are using `Scope.Local()` on both ends, not only can anyone on your network evesdrop on messages they can also spoof messages very easily. +Note that this library doesn't provide any message security. Not only can anyone on your network evesdrop on messages they can also spoof messages very easily. ## Usage @@ -70,7 +70,7 @@ Although UDPComms isn't ideal for commands that need to be processed in order (a ### Publisher Arguments - `port` The port the messages will be sent on. I recommend keep track of your port numbers somewhere. It's possible that in the future UDPComms will have a system of naming (with a string) as opposed to numbering publishers. -- `scope` Leave this as the defualt (`Scope.Local()`) to send messages to only this computer. Set to `Scope.Multicast()` to send to others on the network. See Scopes explained for details +- `scope` Leave this as the defualt (`Scope.LOCAL`) to send messages to only this computer. Set to `Scope.NETWORK` to send to others on the network. See Scopes explained for details ### Subscriber Arguments @@ -78,24 +78,23 @@ The port the messages will be sent on. I recommend keep track of your port numbe The port the subscriber will be listen on. - `timeout` If the `recv()` method don't get a message in `timeout` seconds it throws a `UDPComms.timeout` exception -- `scope` Leave this as the defualt (`Scope.All()`) to receive messages from everyone. Set to `Scope.Multicast()` to register that we want to get multicast messages from other devices . See Scopes explained for details - +- `scope` There is currently no difference in behaviour between `Scope.LOCAL` and `Scope.NETWORK` - both will receive any messages that get to the device. This is planned to change in the future and `Scope.LOCAL` will only receive local messages. ## Scopes Explained -**TLDR** - use `Publisher(port, scope=Scope.Multicast())` and `Subscriber(port, scope=Scope.Multicast())` to get messages to work between different devices +**TLDR** - use `Publisher(port, scope=Scope.NETWORK())` and `Subscriber(port, scope=Scope.NETWORK())` to get messages to work between different devices. + +The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms sends messages only to processes on the same device (`Scope.LOCAL()`). Those are still sent over multicast however the `TTL` (time to live) field is set to 0 so they aren't passed to the network. To send messages to other computers on the same network use `Scope.NETWORK()`. This will default to using the multicast group `239.255.20.22`. -The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms sends messages only to processes on the same device (`Scope.Local()`). To send messages to other computers on the same network use `Scope.Multicast()`. This will default to using the multicast group `239.255.20.22`, but it can be changed by passing an argument. +Older versions of the library defaulted to using a broadcast on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour for compatibility use `Scope.BROADCAST`. -Older versions of the library defulted to using a broadcast on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour use `Scope.Broadcast('10.0.0.255')` +Both the multicast group and the broadcast subnet can be changed by overwriting the class varaibles `MULTICAST_IP` and `BROADCAST_IP` respectivly. Here are all the avalible options: -- `Scope.Local` - Messages meant for this device only -- `Scope.Multicast`- Messages meant for the specified multicast group -- `Scope.Broadcast` - Messages meant for all devices on this subnet -- `Scope.Unicast` - Publisher only argument that sends messages to a specific ip address only -- `Scope.All` - Subscriber only that listens to packets from all (except for multicast) sources +- `Scope.LOCAL` - Messages meant for this device only +- `Scope.NETOWORK`- Messages meant for the specified multicast group +- `Scope.BROADCAST ` - Messages meant for all devices on this subnet ### Connecting to devices on different networks From 2986eade15f9f2d0970b0f4e590ff676ed369d2a Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sat, 13 Feb 2021 15:18:55 -0800 Subject: [PATCH 18/28] documentaiotn --- README.md | 23 +++++++---------------- setup.py | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 823eb4d..504f55b 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ The library automatically determines the type of the message and trasmits it alo Note that this library doesn't provide any message security. Not only can anyone on your network evesdrop on messages they can also spoof messages very easily. +## Installation + +``` $pip3 install UDPComms ``` :) + +Note that if you have an old version of the library installed (before we setup installing via pip) you'll have to uninstll that version manually by removing it from the `site-packages` folder inside your distribution. See this [StackOverflow question](https://stackoverflow.com/questions/402359/how-do-you-uninstall-a-python-package-that-was-installed-using-distutils). Alternativly you could use [virtual environments](https://docs.python.org/3/library/venv.html) to avoid this. + ## Usage ### To Send Messages @@ -102,26 +108,11 @@ Here are all the avalible options: If you want to talk to devices aross the internet use [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) to get them all on the same virtual network and you should be able to use `Scope.Multicast()` from there -## Installation - -``` -$git clone https://github.com/stanfordroboticsclub/UDPComms.git -$sudo bash UDPComms/install.sh -``` - -### To Update - -``` -$cd UDPComms -$git pull -$sudo bash install.sh -``` - ## Extras ### Rover -The library also comes with the `rover` command that can be used to interact with the messages manually. +This repo also comes with the `rover` command that can be used to interact with the messages manually. It doesn't get installed with pip but its here. It depends on the pexpect package you'll have to install manually | Command | Descripion | |---------|------------| diff --git a/setup.py b/setup.py index bdadaf8..d11e211 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,22 @@ #!/usr/bin/env python -from distutils.core import setup +import pathlib +from setuptools import setup + +# The directory containing this file +HERE = pathlib.Path(__file__).parent + +# The text of the README file +README = (HERE / "README.md").read_text() setup(name='UDPComms', - version='1.1dev', + version='2.1', py_modules=['UDPComms'], description='Simple library for sending messages over UDP', + long_description=README, + long_description_content_type="text/markdown", author='Michal Adamkiewicz', author_email='mikadam@stanford.edu', url='https://github.com/stanfordroboticsclub/UDP-Comms', + install_requires=["msgpack>=1.0.0"], ) From a75a0cd6a012042b0a8fb55b27dadbd3f6772685 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sun, 14 Feb 2021 14:55:03 -0800 Subject: [PATCH 19/28] install --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index a20b368..3ad2290 100644 --- a/install.sh +++ b/install.sh @@ -7,8 +7,8 @@ cd $FOLDER yes | sudo pip3 install msgpack yes | sudo pip install msgpack -#The `clean --all` removes the build directory automatically which makes reinstalling new versions possible with the same command. -python3 setup.py clean --all install +# install in editable mode +pip3 install -e . # used for rover command yes | sudo pip3 install pexpect From b622ab94ab6e4ebc2fb109939ec64a95c5aaa94e Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 16 Feb 2021 04:53:30 -0800 Subject: [PATCH 20/28] network is now defualt --- README.md | 4 +--- UDPComms.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 504f55b..8896a2f 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Although UDPComms isn't ideal for commands that need to be processed in order (a ### Publisher Arguments - `port` The port the messages will be sent on. I recommend keep track of your port numbers somewhere. It's possible that in the future UDPComms will have a system of naming (with a string) as opposed to numbering publishers. -- `scope` Leave this as the defualt (`Scope.LOCAL`) to send messages to only this computer. Set to `Scope.NETWORK` to send to others on the network. See Scopes explained for details +- `scope` `Scope.LOCAL` will only send messages to only this computer. `Scope.NETWORK` will to send to others on the network. See Scopes explained for details. ### Subscriber Arguments @@ -88,8 +88,6 @@ If the `recv()` method don't get a message in `timeout` seconds it throws a `UDP ## Scopes Explained -**TLDR** - use `Publisher(port, scope=Scope.NETWORK())` and `Subscriber(port, scope=Scope.NETWORK())` to get messages to work between different devices. - The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms sends messages only to processes on the same device (`Scope.LOCAL()`). Those are still sent over multicast however the `TTL` (time to live) field is set to 0 so they aren't passed to the network. To send messages to other computers on the same network use `Scope.NETWORK()`. This will default to using the multicast group `239.255.20.22`. Older versions of the library defaulted to using a broadcast on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour for compatibility use `Scope.BROADCAST`. diff --git a/UDPComms.py b/UDPComms.py index c0d029a..1965648 100644 --- a/UDPComms.py +++ b/UDPComms.py @@ -41,7 +41,7 @@ class Publisher: MULTICAST_IP = DEFAULT_MULTICAST BROADCAST_IP = DEFAULT_BROADCAST - def __init__(self, port, scope = Scope.LOCAL): + def __init__(self, port, scope = Scope.NETWORK): """ Create a Publisher Object Arguments: @@ -88,7 +88,7 @@ class Subscriber: MULTICAST_IP = DEFAULT_MULTICAST BROADCAST_IP = DEFAULT_BROADCAST - def __init__(self, port, timeout=0.2, scope = Scope.LOCAL ): + def __init__(self, port, timeout=0.2, scope = Scope.NETWORK ): """ Create a Subscriber Object Arguments: From 0e78277803521017f19b14c0208849a56893e07f Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Tue, 16 Feb 2021 05:02:10 -0800 Subject: [PATCH 21/28] verions --- __init__.py | 1 + setup.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index 75a0458..732c62a 100644 --- a/__init__.py +++ b/__init__.py @@ -2,3 +2,4 @@ from .UDPComms import Subscriber from .UDPComms import timeout from .UDPComms import Scope +from .version import __version__ diff --git a/setup.py b/setup.py index d11e211..6ffaffb 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,10 @@ # The text of the README file README = (HERE / "README.md").read_text() +exec(open(HERE / "version.py").read()) + setup(name='UDPComms', - version='2.1', + version=__version__, py_modules=['UDPComms'], description='Simple library for sending messages over UDP', long_description=README, From 00690d03dc37ad746df3ae07e62a082669ad9f58 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Thu, 18 Feb 2021 01:30:37 -0800 Subject: [PATCH 22/28] folder? --- UDPComms.py => UDPComms/UDPComms.py | 0 __init__.py => UDPComms/__init__.py | 0 UDPComms/version.py | 5 +++++ setup.py | 5 +++-- 4 files changed, 8 insertions(+), 2 deletions(-) rename UDPComms.py => UDPComms/UDPComms.py (100%) rename __init__.py => UDPComms/__init__.py (100%) create mode 100644 UDPComms/version.py diff --git a/UDPComms.py b/UDPComms/UDPComms.py similarity index 100% rename from UDPComms.py rename to UDPComms/UDPComms.py diff --git a/__init__.py b/UDPComms/__init__.py similarity index 100% rename from __init__.py rename to UDPComms/__init__.py diff --git a/UDPComms/version.py b/UDPComms/version.py new file mode 100644 index 0000000..a42545e --- /dev/null +++ b/UDPComms/version.py @@ -0,0 +1,5 @@ +# Store the version here so: +# 1) we don't load dependencies by storing it in __init__.py +# 2) we can import it in setup.py for the same reason +# 3) we can import it into your module module +__version__ = '2.1.1' diff --git a/setup.py b/setup.py index 6ffaffb..abd0e9e 100644 --- a/setup.py +++ b/setup.py @@ -9,16 +9,17 @@ # The text of the README file README = (HERE / "README.md").read_text() -exec(open(HERE / "version.py").read()) +exec(open(HERE / "UDPComms/version.py").read()) setup(name='UDPComms', version=__version__, - py_modules=['UDPComms'], description='Simple library for sending messages over UDP', long_description=README, long_description_content_type="text/markdown", author='Michal Adamkiewicz', author_email='mikadam@stanford.edu', + license='MIT', url='https://github.com/stanfordroboticsclub/UDP-Comms', + packages=['UDPComms'], install_requires=["msgpack>=1.0.0"], ) From 93c3abb0a388780d640c0786b91057ea3337c3cb Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Thu, 18 Feb 2021 01:59:22 -0800 Subject: [PATCH 23/28] __all__ --- LICENSE.md | 19 +++++++++++++++++++ UDPComms/__init__.py | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..9cf1062 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,19 @@ +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/UDPComms/__init__.py b/UDPComms/__init__.py index 732c62a..ab1ae20 100644 --- a/UDPComms/__init__.py +++ b/UDPComms/__init__.py @@ -1,3 +1,5 @@ +__all__ = ["Publisher", "Subscriber", "timeout", "Scope"] + from .UDPComms import Publisher from .UDPComms import Subscriber from .UDPComms import timeout From ce900249f65b7ff04edf1de58be720ded580460e Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Thu, 18 Feb 2021 02:04:04 -0800 Subject: [PATCH 24/28] removed msgpack from install.sh --- install.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/install.sh b/install.sh index 3ad2290..abe8371 100644 --- a/install.sh +++ b/install.sh @@ -4,9 +4,6 @@ FOLDER=$(python -c "import os; print(os.path.dirname(os.path.realpath('$0')))") cd $FOLDER -yes | sudo pip3 install msgpack -yes | sudo pip install msgpack - # install in editable mode pip3 install -e . From b2520bf47ff38526c2a56b0c27e93e10f646f7e0 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Fri, 2 Apr 2021 06:12:23 -0700 Subject: [PATCH 25/28] nice interface selection --- UDPComms/UDPComms.py | 84 +++++++++++++++++++++++++------------------- UDPComms/__init__.py | 3 +- UDPComms/version.py | 2 +- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/UDPComms/UDPComms.py b/UDPComms/UDPComms.py index 1965648..b9af563 100644 --- a/UDPComms/UDPComms.py +++ b/UDPComms/UDPComms.py @@ -13,6 +13,7 @@ from enum import Enum, auto import msgpack +import netifaces from sys import version_info @@ -26,50 +27,58 @@ MAX_SIZE = 65507 DEFAULT_MULTICAST = "239.255.20.22" -DEFAULT_BROADCAST = "10.0.0.255" ##TODO: # Test on ubuntu and debian # Documentation (targets, and security disclaimer) -class Scope(Enum): - LOCAL = auto() - NETWORK = auto() - BROADCAST = auto() +def get_iface_info(target): + if target in netifaces.interfaces(): + return netifaces.ifaddresses(target)[netifaces.AF_INET][0] + + else: + for iface in netifaces.interfaces(): + for addr in netifaces.ifaddresses(iface)[socket.AF_INET]: + if target == addr['addr']: + return addr + + ValueError("target needs to be valid interface name or interface ip") class Publisher: MULTICAST_IP = DEFAULT_MULTICAST - BROADCAST_IP = DEFAULT_BROADCAST - def __init__(self, port, scope = Scope.NETWORK): + def __init__(self, port, target = "127.0.0.1", use_multicast = True): """ Create a Publisher Object Arguments: - port -- the port to publish the messages on - scope -- the scope the messages will be sent to + port -- the port to publish the messages on + target -- name or ip of interface to sent messages to + use_multicast -- use multicast transport instead of broadcast """ + self.iface = get_iface_info(target) + + if self.iface['addr'] == "127.0.0.1" and not use_multicast: + raise ValueError("Broadcast not supported on lo0") + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - self.scope = scope self.port = port - if self.scope == Scope.BROADCAST: - ip = self.BROADCAST_IP - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - - else: + if use_multicast: ip = self.MULTICAST_IP - if self.scope == Scope.LOCAL: - ttl = 0 - elif self.scope == Scope.NETWORK: - ttl = 1 - else: - raise ValueError("Unknown Scope") + ttl = 1 # local is restricted by interface so ttl can just be 1 self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, struct.pack('b', ttl)) + self.sock.setsockopt(socket.IPPROTO_IP, + socket.IP_MULTICAST_IF, + socket.inet_aton(self.iface['addr'])); + else: + ip = self.iface['broadcast'] + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + self.sock.settimeout(0.2) self.sock.connect((ip, port)) @@ -86,17 +95,16 @@ def __del__(self): class Subscriber: MULTICAST_IP = DEFAULT_MULTICAST - BROADCAST_IP = DEFAULT_BROADCAST - def __init__(self, port, timeout=0.2, scope = Scope.NETWORK ): + def __init__(self, port, timeout=0.2, target = "127.0.0.1", use_multicast = True ): """ Create a Subscriber Object Arguments: - port -- the port to listen to messages on - timeout -- how long to wait before a message is considered out of date - scope -- where to expect messages to come from + port -- the port to listen to messages on + timeout -- how long to wait before a message is considered out of date + target -- the name or address of interface from which to recv messages + use_multicast -- use multicast transport instead of broadcast """ - self.scope = scope self.port = port self.timeout = timeout @@ -108,18 +116,22 @@ def __init__(self, port, timeout=0.2, scope = Scope.NETWORK ): if hasattr(socket, "SO_REUSEPORT"): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - if self.scope == Scope.BROADCAST: - ip = self.BROADCAST_IP - bind_ip = self.BROADCAST_IP - self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + if target in ("", "all", "ALL", "0.0.0.0"): + if not use_multicast: + raise ValueError("broadcast can't listen to all interfaces") - elif self.scope == Scope.LOCAL or self.scope == Scope.NETWORK: - ip = self.MULTICAST_IP - bind_ip = "0.0.0.0" - mreq = struct.pack("4sl", socket.inet_aton(ip), socket.INADDR_ANY) + self.iface = {'addr':"0.0.0.0"} + else: + self.iface = get_iface_info(target) + + if use_multicast: + bind_ip = self.MULTICAST_IP + mreq = struct.pack("=4s4s", socket.inet_aton(self.MULTICAST_IP), + socket.inet_aton(self.iface['addr'])) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) else: - raise ValueError("Unknown Scope") + bind_ip = self.iface['broadcast'] #binding to interface address doesn't work + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.sock.settimeout(timeout) self.sock.bind((bind_ip, port)) diff --git a/UDPComms/__init__.py b/UDPComms/__init__.py index ab1ae20..cd8ddbf 100644 --- a/UDPComms/__init__.py +++ b/UDPComms/__init__.py @@ -1,7 +1,6 @@ -__all__ = ["Publisher", "Subscriber", "timeout", "Scope"] +__all__ = ["Publisher", "Subscriber", "timeout"] from .UDPComms import Publisher from .UDPComms import Subscriber from .UDPComms import timeout -from .UDPComms import Scope from .version import __version__ diff --git a/UDPComms/version.py b/UDPComms/version.py index a42545e..34ddc9a 100644 --- a/UDPComms/version.py +++ b/UDPComms/version.py @@ -2,4 +2,4 @@ # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = '2.1.1' +__version__ = '2.2.1' From ab77792b03a5526b3263105cfcc9b86f0b29a58b Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Fri, 2 Apr 2021 06:15:58 -0700 Subject: [PATCH 26/28] added dependancy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abd0e9e..41f354f 100644 --- a/setup.py +++ b/setup.py @@ -21,5 +21,5 @@ license='MIT', url='https://github.com/stanfordroboticsclub/UDP-Comms', packages=['UDPComms'], - install_requires=["msgpack>=1.0.0"], + install_requires=["msgpack>=1.0.0", "netifaces>=0.10.9"], ) From ef44364f2b5b21b504ef192de4df5e722ceb20d5 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Fri, 2 Apr 2021 13:51:58 -0700 Subject: [PATCH 27/28] readme --- README.md | 30 ++++++++++++------------------ UDPComms/UDPComms.py | 32 ++++++++++++++++---------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 8896a2f..740109a 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,10 @@ Although UDPComms isn't ideal for commands that need to be processed in order (a - ### Publisher Arguments -- `port` +- `port`: The port the messages will be sent on. I recommend keep track of your port numbers somewhere. It's possible that in the future UDPComms will have a system of naming (with a string) as opposed to numbering publishers. -- `scope` `Scope.LOCAL` will only send messages to only this computer. `Scope.NETWORK` will to send to others on the network. See Scopes explained for details. +- `target`: The name (`"lo0"`, `"en0"` etc) or ip address (`"127.0.0.1"`, `"10.0.0.23"` etc) of the [network interface](https://goinbigdata.com/demystifying-ifconfig-and-network-interfaces-in-linux/) to use for sending the messages. It defaults to the loopback interface (`"127.0.0.1"`) so keeping the messages only on the local computer. +- `multicast_ip`: the multicast group ip to use. It defualts to ` "239.255.20.22"`. It can able be set to `None` for compatiblity with old versions of the library ### Subscriber Arguments @@ -84,31 +85,25 @@ The port the messages will be sent on. I recommend keep track of your port numbe The port the subscriber will be listen on. - `timeout` If the `recv()` method don't get a message in `timeout` seconds it throws a `UDPComms.timeout` exception -- `scope` There is currently no difference in behaviour between `Scope.LOCAL` and `Scope.NETWORK` - both will receive any messages that get to the device. This is planned to change in the future and `Scope.LOCAL` will only receive local messages. - -## Scopes Explained +- `target`: The name (`"lo0"`, `"en0"` etc) or ip address (`"127.0.0.1"`, `"10.0.0.23"` etc) of the [network interface](https://goinbigdata.com/demystifying-ifconfig-and-network-interfaces-in-linux/) to use for listening for messages. It can also be set to `"0.0.0.0"` or `"all"` to listen on all interfaces (which is defualts to). +- `multicast_ip`: the multicast group ip to use. It defualts to ` "239.255.20.22"`. It can able be set to `None` for compatiblity with old versions of the library -The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms sends messages only to processes on the same device (`Scope.LOCAL()`). Those are still sent over multicast however the `TTL` (time to live) field is set to 0 so they aren't passed to the network. To send messages to other computers on the same network use `Scope.NETWORK()`. This will default to using the multicast group `239.255.20.22`. -Older versions of the library defaulted to using a broadcast on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour for compatibility use `Scope.BROADCAST`. -Both the multicast group and the broadcast subnet can be changed by overwriting the class varaibles `MULTICAST_IP` and `BROADCAST_IP` respectivly. - -Here are all the avalible options: +## Extras -- `Scope.LOCAL` - Messages meant for this device only -- `Scope.NETOWORK`- Messages meant for the specified multicast group -- `Scope.BROADCAST ` - Messages meant for all devices on this subnet +### Connecting to devices on different networks +If you want to talk to devices aross the internet use [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) to get them all on the same virtual network and then you can use the virtual network interface name as the `target` argument. -### Connecting to devices on different networks +### Behind the scenes -If you want to talk to devices aross the internet use [RemoteVPN](https://github.com/stanfordroboticsclub/RemoteVPN) to get them all on the same virtual network and you should be able to use `Scope.Multicast()` from there +The protocol underlying UDPComms - UDP has a number of differnt [options](https://en.wikipedia.org/wiki/Routing#Delivery_schemes) for how packets can be delivered. By default UDPComms will send packets using multicast to the loopback interface. +Older versions of the library defaulted to using a broadcast specifically on the `10.0.0.X` subnet. However, now that the library is often used on differnt networks that is no longer the defualt. To emulate the old behvaiour for compatibility set the `multicast_ip` to `None` to force broadcast transport, and `target` to the computers ip on the `10.0.0.X` subnet. -## Extras -### Rover +### Rover Command This repo also comes with the `rover` command that can be used to interact with the messages manually. It doesn't get installed with pip but its here. It depends on the pexpect package you'll have to install manually @@ -120,7 +115,6 @@ This repo also comes with the `rover` command that can be used to interact with There are more commands used for starting and stoping services described in [this repo](https://github.com/stanfordroboticsclub/RPI-Setup/blob/master/README.md) - ### Known issues: - Macs have issues sending large messages. They are fine receiving them. I think it is related to [this issue](https://github.com/BanTheRewind/Cinder-Asio/issues/9). I wonder does it work on Linux by chance (as the packets happen to be in order) but so far we didn't have issues. diff --git a/UDPComms/UDPComms.py b/UDPComms/UDPComms.py index b9af563..91af023 100644 --- a/UDPComms/UDPComms.py +++ b/UDPComms/UDPComms.py @@ -45,27 +45,27 @@ def get_iface_info(target): ValueError("target needs to be valid interface name or interface ip") class Publisher: - MULTICAST_IP = DEFAULT_MULTICAST - - def __init__(self, port, target = "127.0.0.1", use_multicast = True): + def __init__(self, port, target = "127.0.0.1", multicast_ip = DEFAULT_MULTICAST): """ Create a Publisher Object Arguments: port -- the port to publish the messages on target -- name or ip of interface to sent messages to - use_multicast -- use multicast transport instead of broadcast + multicast_ip -- if specified the multicast group ip to send messages + to. If None fallback to broadcast """ self.iface = get_iface_info(target) - if self.iface['addr'] == "127.0.0.1" and not use_multicast: - raise ValueError("Broadcast not supported on lo0") + if self.iface['addr'] == "127.0.0.1" and multicast_ip is None: + raise ValueError("Broadcast not supported on loopback") self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.port = port + self.multicast_ip = multicast_ip - if use_multicast: - ip = self.MULTICAST_IP + if multicast_ip: + ip = multicast_ip ttl = 1 # local is restricted by interface so ttl can just be 1 self.sock.setsockopt(socket.IPPROTO_IP, @@ -94,19 +94,19 @@ def __del__(self): class Subscriber: - MULTICAST_IP = DEFAULT_MULTICAST - - def __init__(self, port, timeout=0.2, target = "127.0.0.1", use_multicast = True ): + def __init__(self, port, timeout=0.2, target = "all", multicast_ip = DEFAULT_MULTICAST ): """ Create a Subscriber Object Arguments: port -- the port to listen to messages on timeout -- how long to wait before a message is considered out of date target -- the name or address of interface from which to recv messages - use_multicast -- use multicast transport instead of broadcast + multicast_ip -- if specified the multicast group ip to send messages + to. If None fallback to broadcast """ self.port = port self.timeout = timeout + self.multicast_ip = multicast_ip self.last_data = None self.last_time = float('-inf') @@ -117,16 +117,16 @@ def __init__(self, port, timeout=0.2, target = "127.0.0.1", use_multicast = True self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) if target in ("", "all", "ALL", "0.0.0.0"): - if not use_multicast: + if multicast_ip is None: raise ValueError("broadcast can't listen to all interfaces") self.iface = {'addr':"0.0.0.0"} else: self.iface = get_iface_info(target) - if use_multicast: - bind_ip = self.MULTICAST_IP - mreq = struct.pack("=4s4s", socket.inet_aton(self.MULTICAST_IP), + if multicast_ip: + bind_ip = multicast_ip + mreq = struct.pack("=4s4s", socket.inet_aton(multicast_ip), socket.inet_aton(self.iface['addr'])) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) else: From c5b92536fbb3c0a8a66b40695fe60e1ca5c92301 Mon Sep 17 00:00:00 2001 From: Michal Adamkiewicz Date: Sat, 10 Apr 2021 02:43:01 -0700 Subject: [PATCH 28/28] new tests --- UDPComms/UDPComms.py | 8 +++-- test.py | 76 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/UDPComms/UDPComms.py b/UDPComms/UDPComms.py index 91af023..d81a94f 100644 --- a/UDPComms/UDPComms.py +++ b/UDPComms/UDPComms.py @@ -38,7 +38,7 @@ def get_iface_info(target): else: for iface in netifaces.interfaces(): - for addr in netifaces.ifaddresses(iface)[socket.AF_INET]: + for addr in netifaces.ifaddresses(iface).get(socket.AF_INET, ()): if target == addr['addr']: return addr @@ -94,7 +94,7 @@ def __del__(self): class Subscriber: - def __init__(self, port, timeout=0.2, target = "all", multicast_ip = DEFAULT_MULTICAST ): + def __init__(self, port, timeout=0.2, target = "127.0.0.1", multicast_ip = DEFAULT_MULTICAST ): """ Create a Subscriber Object Arguments: @@ -104,6 +104,7 @@ def __init__(self, port, timeout=0.2, target = "all", multicast_ip = DEFAULT_MUL multicast_ip -- if specified the multicast group ip to send messages to. If None fallback to broadcast """ + #TODO: target=all doesn't work. TODO: change target=all to defult self.port = port self.timeout = timeout self.multicast_ip = multicast_ip @@ -125,7 +126,8 @@ def __init__(self, port, timeout=0.2, target = "all", multicast_ip = DEFAULT_MUL self.iface = get_iface_info(target) if multicast_ip: - bind_ip = multicast_ip + # bind_ip = multicast_ip + bind_ip = "0.0.0.0" mreq = struct.pack("=4s4s", socket.inet_aton(multicast_ip), socket.inet_aton(self.iface['addr'])) self.sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) diff --git a/test.py b/test.py index 8a858a0..aeb4c7d 100644 --- a/test.py +++ b/test.py @@ -1,17 +1,18 @@ -from UDPComms import Publisher, Subscriber, Scope, timeout +from UDPComms import Publisher, Subscriber, timeout import time import subprocess import os +import socket import unittest class SingleProcessTestCase(unittest.TestCase): def setUp(self): - self.local_pub = Publisher(8001, scope = Scope.LOCAL ) - self.local_sub = Subscriber(8001, timeout=1, scope = Scope.LOCAL ) - self.local_sub2 = Subscriber(8001, timeout=1, scope = Scope.LOCAL ) + self.local_pub = Publisher(8001, target = '127.0.0.1' ) + self.local_sub = Subscriber(8001, timeout=1, target = '127.0.0.1' ) + self.local_sub2 = Subscriber(8001, timeout=1, target = '127.0.0.1' ) def tearDown(self): pass @@ -64,8 +65,13 @@ def test_get_recv(self): class SingleProcessNetworkTestCase(unittest.TestCase): def setUp(self): - self.local_pub = Publisher(8002, scope = Scope.NETWORK ) - self.local_sub = Subscriber(8002, timeout=1, scope = Scope.NETWORK ) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect(('8.8.8.8', 1)) #gets ip of default interface + network_ip_address = s.getsockname()[0] + + self.pub = Publisher(8002, target = network_ip_address ) + self.sub = Subscriber(8002, timeout=1, target = network_ip_address ) + self.sub2 = Subscriber(8002, timeout=1, target = network_ip_address ) def tearDown(self): pass @@ -73,11 +79,55 @@ def tearDown(self): def test_simple(self): msg = [123, "testing", {"one":2} ] - self.local_pub.send( msg ) - recv_msg = self.local_sub.recv() + self.pub.send( msg ) + recv_msg = self.sub.recv() + + self.assertEqual(msg, recv_msg) + + def test_dual_recv(self): + msg = [125, "testing", {"one":3} ] + + self.pub.send( msg ) + recv_msg = self.sub.recv() + recv_msg2 = self.sub2.recv() self.assertEqual(msg, recv_msg) + self.assertEqual(msg, recv_msg2) + +class MixTargetsTestCase(unittest.TestCase): + + def setUp(self): + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect(('8.8.8.8', 1)) #gets ip of default interface + network_ip_address = s.getsockname()[0] + self.pub_net = Publisher(8005, target = network_ip_address ) + self.pub_local = Publisher(8005, target = "127.0.0.1" ) + + self.sub_local = Subscriber(8005, timeout=1, target = "127.0.0.1" ) + self.sub_net = Subscriber(8005, timeout=1, target = network_ip_address ) + self.sub_all = Subscriber(8005, timeout=1, target = "0.0.0.0" ) + + def tearDown(self): + pass + + def test_local_send(self): + msg = [123, "local", {"one":2.2} ] + + self.pub_local.send( msg ) + + self.assertEqual(msg, self.sub_all.recv()) + self.assertEqual(msg, self.sub_local.recv()) + self.assertRaises(timeout, self.sub_net.recv) + + def test_net_send(self): + msg = [123, "net", {"one":2.3} ] + + self.pub_net.send( msg ) + + self.assertEqual(msg, self.sub_all.recv()) + self.assertEqual(msg, self.sub_net.recv()) + self.assertRaises(timeout, self.sub_local.recv) class MultiProcessTestCase(unittest.TestCase): @@ -86,8 +136,8 @@ def setUp(self): mirror_server = b""" from UDPComms import *; import sys -incomming = Subscriber(8003, timeout=10, scope = Scope.NETWORK ); -outgoing = Publisher(8000, scope = Scope.NETWORK ); +incomming = Subscriber(8003, timeout=10); +outgoing = Publisher(8000); print("ready"); sys.stdout.flush() while 1: outgoing.send(incomming.recv()); @@ -99,8 +149,8 @@ def setUp(self): self.p.stdin.close() self.p.stdout.readline() #wait for program to be ready - self.local_pub = Publisher(8003, scope = Scope.LOCAL ) - self.return_path = Subscriber(8000, timeout = 5, scope = Scope.LOCAL ) + self.local_pub = Publisher(8003) + self.return_path = Subscriber(8000, timeout = 5) def tearDown(self): self.p.stdout.close() @@ -121,5 +171,3 @@ def test_simple(self): unittest.main() - -# socket.gethostbyname('www.google.com')