gh-82082: Make our test suite pass on an IPv6-only Linux host

Lib/logging/config.py

@@ -29,6 +29,7 @@
 import logging
 import logging.handlers
 import re
+import socket
 import struct
 import sys
 import threading
@@ -885,7 +886,11 @@ class ConfigSocketReceiver(ThreadingTCPServer):
 
         def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT,
                      handler=None, ready=None, verify=None):
-            ThreadingTCPServer.__init__(self, (host, port), handler)
+            try:
+                ThreadingTCPServer.__init__(self, (host, port), handler)
+            except OSError as err:
+                self.address_family = socket.AF_INET6
+                ThreadingTCPServer.__init__(self, (host, port), handler)
             logging._acquireLock()
             self.abort = 0
             logging._releaseLock()

Lib/multiprocessing/connection.py

@@ -51,6 +51,9 @@
     default_family = 'AF_UNIX'
     families += ['AF_UNIX']
 
+if hasattr(socket, 'AF_INET6') and socket.has_ipv6:
+    families.append('AF_INET6')
+
 if sys.platform == 'win32':
     default_family = 'AF_PIPE'
     families += ['AF_PIPE']
@@ -70,7 +73,7 @@ def arbitrary_address(family):
     '''
     Return an arbitrary free address for the given family
     '''
-    if family == 'AF_INET':
+    if family in {'AF_INET', 'AF_INET6'}:
         return ('localhost', 0)
     elif family == 'AF_UNIX':
         # Prefer abstract sockets if possible to avoid problems with the address
@@ -101,9 +104,16 @@ def address_type(address):
     '''
     Return the types of the address
 
-    This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
+    This can be 'AF_INET', 'AF_INET6', 'AF_UNIX', or 'AF_PIPE'
     '''
     if type(address) == tuple:
+        if '.' in address[0]:
+            return 'AF_INET'
+        if ':' in address[0]:
+            return 'AF_INET6'
+        addr_info = socket.getaddrinfo(*address[:2])
+        if addr_info:
+            return addr_info[0][0].name
         return 'AF_INET'
     elif type(address) is str and address.startswith('\\\\'):
         return 'AF_PIPE'

Lib/test/_test_eintr.py

@@ -285,28 +285,28 @@ def test_sendmsg(self):
         self._test_send(lambda sock, data: sock.sendmsg([data]))
 
     def test_accept(self):
-        sock = socket.create_server((socket_helper.HOST, 0))
-        self.addCleanup(sock.close)
-        port = sock.getsockname()[1]
-
-        code = '\n'.join((
-            'import socket, time',
-            '',
-            'host = %r' % socket_helper.HOST,
-            'port = %s' % port,
-            'sleep_time = %r' % self.sleep_time,
-            '',
-            '# let parent block on accept()',
-            'time.sleep(sleep_time)',
-            'with socket.create_connection((host, port)):',
-            '    time.sleep(sleep_time)',
-        ))
-
-        proc = self.subprocess(code)
-        with kill_on_error(proc):
-            client_sock, _ = sock.accept()
-            client_sock.close()
-            self.assertEqual(proc.wait(), 0)
+        with socket_helper.bind_ip_socket_and_port() as sock_port:
+            sock, port = sock_port
+            sock.listen()
+
+            code = '\n'.join((
+                'import socket, time',
+                '',
+                'host = %r' % socket_helper.HOST,
+                'port = %s' % port,
+                'sleep_time = %r' % self.sleep_time,
+                '',
+                '# let parent block on accept()',
+                'time.sleep(sleep_time)',
+                'with socket.create_connection((host, port)):',
+                '    time.sleep(sleep_time)',
+            ))
+
+            proc = self.subprocess(code)
+            with kill_on_error(proc):
+                client_sock, _ = sock.accept()
+                client_sock.close()
+                self.assertEqual(proc.wait(), 0)
 
     # Issue #25122: There is a race condition in the FreeBSD kernel on
     # handling signals in the FIFO device. Skip the test until the bug is

Lib/test/_test_multiprocessing.py

@@ -184,6 +184,14 @@ class BaseTestCase(object):
 
     ALLOWED_TYPES = ('processes', 'manager', 'threads')
 
+    def get_families(self):
+        fams = set(self.connection.families)
+        if not socket_helper.IPV6_ENABLED:
+            fams -= {'AF_INET6'}
+        if not socket_helper.IPV4_ENABLED:
+            fams -= {'AF_INET'}
+        return fams
+
     def assertTimingAlmostEqual(self, a, b):
         if CHECK_TIMINGS:
             self.assertAlmostEqual(a, b, 1)
@@ -3284,7 +3292,7 @@ class _TestListener(BaseTestCase):
     ALLOWED_TYPES = ('processes',)
 
     def test_multiple_bind(self):
-        for family in self.connection.families:
+        for family in self.get_families():
             l = self.connection.Listener(family=family)
             self.addCleanup(l.close)
             self.assertRaises(OSError, self.connection.Listener,
@@ -3324,7 +3332,7 @@ def _test(cls, address):
         conn.close()
 
     def test_listener_client(self):
-        for family in self.connection.families:
+        for family in self.get_families():
             l = self.connection.Listener(family=family)
             p = self.Process(target=self._test, args=(l.address,))
             p.daemon = True
@@ -3351,7 +3359,7 @@ def test_issue14725(self):
         l.close()
 
     def test_issue16955(self):
-        for fam in self.connection.families:
+        for fam in self.get_families():
             l = self.connection.Listener(family=fam)
             c = self.connection.Client(l.address)
             a = l.accept()
@@ -3464,7 +3472,8 @@ def _listener(cls, conn, families):
             new_conn.close()
             l.close()
 
-        l = socket.create_server((socket_helper.HOST, 0))
+        l = socket.create_server((socket_helper.HOST, 0),
+                                 family=socket_helper.get_family())
         conn.send(l.getsockname())
         new_conn, addr = l.accept()
         conn.send(new_conn)
@@ -3481,15 +3490,15 @@ def _remote(cls, conn):
             client.close()
 
         address, msg = conn.recv()
-        client = socket.socket()
+        client = socket_helper.tcp_socket()
         client.connect(address)
         client.sendall(msg.upper())
         client.close()
 
         conn.close()
 
     def test_pickling(self):
-        families = self.connection.families
+        families = self.get_families()
 
         lconn, lconn0 = self.Pipe()
         lp = self.Process(target=self._listener, args=(lconn0, families))
@@ -4638,7 +4647,7 @@ def test_wait(self, slow=False):
 
     @classmethod
     def _child_test_wait_socket(cls, address, slow):
-        s = socket.socket()
+        s = socket_helper.tcp_socket()
         s.connect(address)
         for i in range(10):
             if slow:
@@ -4648,7 +4657,8 @@ def _child_test_wait_socket(cls, address, slow):
 
     def test_wait_socket(self, slow=False):
         from multiprocessing.connection import wait
-        l = socket.create_server((socket_helper.HOST, 0))
+        l = socket.create_server((socket_helper.HOST, 0),
+                                 family=socket_helper.get_family())
         addr = l.getsockname()
         readers = []
         procs = []
@@ -4836,7 +4846,8 @@ def test_timeout(self):
         try:
             socket.setdefaulttimeout(0.1)
             parent, child = multiprocessing.Pipe(duplex=True)
-            l = multiprocessing.connection.Listener(family='AF_INET')
+            l = multiprocessing.connection.Listener(
+                    family=socket_helper.get_family().name)
             p = multiprocessing.Process(target=self._test_timeout,
                                         args=(child, l.address))
             p.start()
@@ -4910,11 +4921,11 @@ def get_high_socket_fd(self):
             # The child process will not have any socket handles, so
             # calling socket.fromfd() should produce WSAENOTSOCK even
             # if there is a handle of the same number.
-            return socket.socket().detach()
+            return socket_helper.tcp_socket().detach()
         else:
             # We want to produce a socket with an fd high enough that a
             # freshly created child process will not have any fds as high.
-            fd = socket.socket().detach()
+            fd = socket_helper.tcp_socket().detach()
             to_close = []
             while fd < 50:
                 to_close.append(fd)
@@ -4925,7 +4936,7 @@ def get_high_socket_fd(self):
 
     def close(self, fd):
         if WIN32:
-            socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close()
+            socket.socket(socket_helper.get_family(), socket.SOCK_STREAM, fileno=fd).close()
         else:
             os.close(fd)
 

Lib/test/ssl_servers.py

@@ -20,6 +20,8 @@
 
 class HTTPSServer(_HTTPServer):
 
+    address_family = socket_helper.get_family()
+
     def __init__(self, server_address, handler_class, context):
         _HTTPServer.__init__(self, server_address, handler_class)
         self.context = context

Lib/test/support/socket_helper.py

@@ -12,25 +12,28 @@
 HOSTv6 = "::1"
 
 
-def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
+def find_unused_port(family=None, socktype=socket.SOCK_STREAM):
     """Returns an unused port that should be suitable for binding.  This is
     achieved by creating a temporary socket with the same family and type as
     the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to
     the specified host address (defaults to 0.0.0.0) with the port set to 0,
     eliciting an unused ephemeral port from the OS.  The temporary socket is
     then closed and deleted, and the ephemeral port is returned.
 
+    When family is None, we use to the result of get_family() instead.
+
     Either this method or bind_port() should be used for any tests where a
     server socket needs to be bound to a particular port for the duration of
     the test.  Which one to use depends on whether the calling code is creating
     a python socket, or if an unused port needs to be provided in a constructor
     or passed to an external program (i.e. the -accept argument to openssl's
-    s_server mode).  Always prefer bind_port() over find_unused_port() where
-    possible.  Hard coded ports should *NEVER* be used.  As soon as a server
-    socket is bound to a hard coded port, the ability to run multiple instances
-    of the test simultaneously on the same host is compromised, which makes the
-    test a ticking time bomb in a buildbot environment. On Unix buildbots, this
-    may simply manifest as a failed test, which can be recovered from without
+    s_server mode).  Always prefer bind_port(), bind_ip_socket_and_port(),
+    and get_bound_ip_socket_and_port() over find_unused_port() where possible.
+    Hard coded ports should *NEVER* be used.  As soon as a server socket is
+    bound to a hard coded port, the ability to run multiple instances of the
+    test simultaneously on the same host is compromised, which makes the test a
+    ticking time bomb in a buildbot environment. On Unix buildbots, this may
+    simply manifest as a failed test, which can be recovered from without
     intervention in most cases, but on Windows, the entire python process can
     completely and utterly wedge, requiring someone to log in to the buildbot
     and manually kill the affected process.
@@ -66,19 +69,26 @@ def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
     other process when we close and delete our temporary socket but before our
     calling code has a chance to bind the returned port.  We can deal with this
     issue if/when we come across it.
+
+    TODO(gpshead): We should support a https://pypi.org/project/portpicker/
+    portserver or equivalent running on our buildbot workers and use that
+    that for more reliability at avoiding conflicts between parallel tests.
     """
 
+    if family is None:
+        family = get_family()
     with socket.socket(family, socktype) as tempsock:
         port = bind_port(tempsock)
     del tempsock
     return port
 
+
 def bind_port(sock, host=HOST):
     """Bind the socket to a free port and return the port number.  Relies on
     ephemeral ports in order to ensure we are using an unbound port.  This is
     important as many tests may be running simultaneously, especially in a
     buildbot environment.  This method raises an exception if the sock.family
-    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
+    is AF_INET* and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
     or SO_REUSEPORT set on it.  Tests should *never* set these socket options
     for TCP/IP sockets.  The only case for setting these options is testing
     multicasting via multiple UDP sockets.
@@ -88,7 +98,8 @@ def bind_port(sock, host=HOST):
     from bind()'ing to our host/port for the duration of the test.
     """
 
-    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
+    if (sock.family in {socket.AF_INET, socket.AF_INET6} and
+        sock.type == socket.SOCK_STREAM):
         if hasattr(socket, 'SO_REUSEADDR'):
             if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                 raise support.TestFailed("tests should never set the "
@@ -112,6 +123,60 @@ def bind_port(sock, host=HOST):
     port = sock.getsockname()[1]
     return port
 
+
+def get_family():
+    """Get a host appropriate socket AF_INET or AF_INET6 family."""
+    if IPV4_ENABLED:
+        return socket.AF_INET
+    if IPV6_ENABLED:
+        return socket.AF_INET6
+    raise unittest.SkipTest('Neither IPv4 or IPv6 is enabled.')
+
+
+def tcp_socket():
+    """Get a new host appropriate IPv4 or IPv6 TCP STREAM socket.socket()."""
+    return socket.socket(get_family(), socket.SOCK_STREAM)
+
+
+def udp_socket(proto=-1):
+    """Get a new host appropriate IPv4 or IPv6 UDP DGRAM socket.socket()."""
+    return socket.socket(get_family(), socket.SOCK_DGRAM, proto)
+
+
+def get_bound_ip_socket_and_port(*, hostname=HOST, socktype=socket.SOCK_STREAM):
+    """Get an IP socket bound to a port as a sock, port tuple.
+
+    Creates a socket of socktype bound to hostname using whichever of IPv6 or
+    IPv4 is available.  Context is a (socket, port) tuple.  Exiting the context
+    closes the socket.
+
+    Prefer the bind_ip_socket_and_port context manager within a test method.
+    """
+    family = get_family()
+    sock = socket.socket(family, socktype)
+    try:
+        port = bind_port(sock)
+    except support.TestFailed:
+        sock.close()
+        raise
+    return sock, port
+
+
+@contextlib.contextmanager
+def bind_ip_socket_and_port(*, hostname=HOST, socktype=socket.SOCK_STREAM):
+    """A context manager that creates a socket of socktype.
+
+    It uses whichever of IPv6 or IPv4 is available based on get_family().
+    Context is a (socket, port) tuple. The socket is closed on context exit.
+    """
+    sock, port = get_bound_ip_socket_and_port(
+            hostname=hostname, socktype=socktype)
+    try:
+        yield sock, port
+    finally:
+        sock.close()
+
+
 def bind_unix_socket(sock, addr):
     """Bind a unix socket, raising SkipTest if PermissionError is raised."""
     assert sock.family == socket.AF_UNIX
@@ -139,6 +204,22 @@ def _is_ipv6_enabled():
 IPV6_ENABLED = _is_ipv6_enabled()
 
 
+def _is_ipv4_enabled():
+    """Check whether IPv4 is enabled on this host."""
+    sock = None
+    try:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        sock.bind((HOSTv4, 0))
+        return True
+    except OSError:
+        return False
+    finally:
+        if sock:
+            sock.close()
+
+IPV4_ENABLED = _is_ipv4_enabled()
+
+
 _bind_nix_socket_error = None
 def skip_unless_bind_unix_socket(test):
     """Decorator for tests requiring a functional bind() for unix sockets."""