Treat messages from unknown devices as implicit joins and discover IEEE addresses

tests/api/test_response.py

@@ -5,7 +5,7 @@
 
 import zigpy_znp.types as t
 import zigpy_znp.commands as c
-from zigpy_znp.api import _deduplicate_commands
+from zigpy_znp.utils import deduplicate_commands
 
 pytestmark = [pytest.mark.asyncio]
 
@@ -215,15 +215,15 @@ async def test_command_deduplication_simple():
     c1 = c.SYS.Ping.Rsp(partial=True)
     c2 = c.UTIL.TimeAlive.Rsp(Seconds=12)
 
-    assert _deduplicate_commands([]) == ()
-    assert _deduplicate_commands([c1]) == (c1,)
-    assert _deduplicate_commands([c1, c1]) == (c1,)
-    assert _deduplicate_commands([c1, c2]) == (c1, c2)
-    assert _deduplicate_commands([c2, c1, c2]) == (c2, c1)
+    assert deduplicate_commands([]) == ()
+    assert deduplicate_commands([c1]) == (c1,)
+    assert deduplicate_commands([c1, c1]) == (c1,)
+    assert deduplicate_commands([c1, c2]) == (c1, c2)
+    assert deduplicate_commands([c2, c1, c2]) == (c2, c1)
 
 
 async def test_command_deduplication_complex():
-    result = _deduplicate_commands(
+    result = deduplicate_commands(
         [
             c.SYS.Ping.Rsp(Capabilities=t.MTCapabilities.SYS),
             # Duplicating matching commands shouldn't do anything
@@ -302,7 +302,7 @@ async def async_callback(response):
         c.UTIL.TimeAlive.Rsp(Seconds=10),
     ]
 
-    assert set(_deduplicate_commands(responses)) == {
+    assert set(deduplicate_commands(responses)) == {
         c.SYS.Ping.Rsp(partial=True),
         c.UTIL.TimeAlive.Rsp(Seconds=12),
         c.UTIL.TimeAlive.Rsp(Seconds=10),

tests/application/test_joining.py

@@ -508,3 +508,82 @@ def bind_req_callback(request):
             await cluster.bind()
 
     await app.shutdown()
+
+
+@pytest.mark.parametrize("device", FORMED_DEVICES)
+async def test_unknown_device_discovery(device, make_application, mocker):
+    app, znp_server = make_application(server_cls=device)
+    await app.startup(auto_form=False)
+
+    mocker.spy(app, "handle_join")
+
+    # Existing devices do not need to be discovered
+    existing_nwk = 0x1234
+    existing_ieee = t.EUI64(range(8))
+    device = app.add_initialized_device(ieee=existing_ieee, nwk=existing_nwk)
+
+    assert (await app._get_or_discover_device(nwk=existing_nwk)) is device
+    assert app.handle_join.call_count == 0
+
+    # If the device changes its NWK but doesn't tell zigpy, it will be re-discovered
+    did_ieee_addr_req1 = znp_server.reply_once_to(
+        request=c.ZDO.IEEEAddrReq.Req(
+            NWK=existing_nwk + 1,
+            RequestType=c.zdo.AddrRequestType.SINGLE,
+            StartIndex=0,
+        ),
+        responses=[
+            c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS),
+            c.ZDO.IEEEAddrRsp.Callback(
+                Status=t.ZDOStatus.SUCCESS,
+                IEEE=existing_ieee,
+                NWK=existing_nwk + 1,
+                Index=0,
+                Devices=[],
+            ),
+        ],
+    )
+
+    # The same device is discovered and its NWK was updated. Handles concurrency.
+    devices = await asyncio.gather(
+        app._get_or_discover_device(nwk=existing_nwk + 1),
+        app._get_or_discover_device(nwk=existing_nwk + 1),
+        app._get_or_discover_device(nwk=existing_nwk + 1),
+        app._get_or_discover_device(nwk=existing_nwk + 1),
+        app._get_or_discover_device(nwk=existing_nwk + 1),
+    )
+
+    assert devices == [device] * 5
+
+    # Only a single request is sent, since the coroutines are grouped
+    await did_ieee_addr_req1
+    assert device.nwk == existing_nwk + 1
+    assert app.handle_join.call_count == 0
+
+    # If a completely unknown device joins the network, it will be treated as a new join
+    new_nwk = 0x5678
+    new_ieee = t.EUI64(range(1, 9))
+    did_ieee_addr_req2 = znp_server.reply_once_to(
+        request=c.ZDO.IEEEAddrReq.Req(
+            NWK=new_nwk,
+            RequestType=c.zdo.AddrRequestType.SINGLE,
+            StartIndex=0,
+        ),
+        responses=[
+            c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS),
+            c.ZDO.IEEEAddrRsp.Callback(
+                Status=t.ZDOStatus.SUCCESS,
+                IEEE=new_ieee,
+                NWK=new_nwk,
+                Index=0,
+                Devices=[],
+            ),
+        ],
+    )
+    new_dev = await app._get_or_discover_device(nwk=new_nwk)
+    await did_ieee_addr_req2
+    assert app.handle_join.call_count == 1
+    assert new_dev.nwk == new_nwk
+    assert new_dev.ieee == new_ieee
+
+    await app.pre_shutdown()

tests/application/test_requests.py

@@ -487,7 +487,6 @@ async def test_nonstandard_profile(device, make_application):
     await app.startup(auto_form=False)
 
     device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xFA9E)
-    device.node_desc, _ = device.node_desc.deserialize(bytes(14))
 
     ep = device.add_endpoint(2)
     ep.status = zigpy.endpoint.Status.ZDO_INIT
@@ -605,7 +604,6 @@ async def test_request_recovery_route_rediscovery_zdo(device, make_application,
     app._znp._config[conf.CONF_ZNP_CONFIG][conf.CONF_ARSP_TIMEOUT] = 1
 
     device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xABCD)
-    device.node_desc, _ = device.node_desc.deserialize(bytes(14))
 
     # Fail the first time
     route_discovered = False
@@ -669,7 +667,6 @@ async def test_request_recovery_route_rediscovery_af(device, make_application, m
     app._znp._config[conf.CONF_ZNP_CONFIG][conf.CONF_ARSP_TIMEOUT] = 1
 
     device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xABCD)
-    device.node_desc, _ = device.node_desc.deserialize(bytes(14))
 
     # Fail the first time
     route_discovered = False
@@ -728,6 +725,70 @@ def set_route_discovered(req):
     await app.shutdown()
 
 
+@pytest.mark.parametrize("device", [FormedLaunchpadCC26X2R1])
+async def test_request_recovery_use_ieee_addr(device, make_application, mocker):
+    app, znp_server = make_application(server_cls=device)
+
+    await app.startup(auto_form=False)
+
+    # The data confirm timeout must be shorter than the ARSP timeout
+    mocker.patch("zigpy_znp.zigbee.application.DATA_CONFIRM_TIMEOUT", new=0.1)
+    app._znp._config[conf.CONF_ZNP_CONFIG][conf.CONF_ARSP_TIMEOUT] = 1
+
+    device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xABCD)
+
+    was_ieee_addr_used = False
+
+    def data_confirm_replier(req):
+        nonlocal was_ieee_addr_used
+
+        if req.DstAddrModeAddress.mode == t.AddrMode.IEEE:
+            status = t.Status.SUCCESS
+            was_ieee_addr_used = True
+        else:
+            status = t.Status.MAC_NO_ACK
+
+        return c.AF.DataConfirm.Callback(Status=status, Endpoint=1, TSN=1)
+
+    znp_server.reply_once_to(
+        c.ZDO.ExtRouteDisc.Req(
+            Dst=device.nwk, Options=c.zdo.RouteDiscoveryOptions.UNICAST, partial=True
+        ),
+        responses=[c.ZDO.ExtRouteDisc.Rsp(Status=t.Status.SUCCESS)],
+    )
+
+    znp_server.reply_to(
+        c.AF.DataRequestExt.Req(partial=True),
+        responses=[
+            c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS),
+            data_confirm_replier,
+        ],
+    )
+
+    # Ignore the source routing request as well
+    znp_server.reply_to(
+        c.AF.DataRequestSrcRtg.Req(partial=True),
+        responses=[
+            c.AF.DataRequestSrcRtg.Rsp(Status=t.Status.SUCCESS),
+            c.AF.DataConfirm.Callback(Status=t.Status.MAC_NO_ACK, Endpoint=1, TSN=1),
+        ],
+    )
+
+    await app.request(
+        device=device,
+        profile=260,
+        cluster=1,
+        src_ep=1,
+        dst_ep=1,
+        sequence=1,
+        data=b"\x00",
+    )
+
+    assert was_ieee_addr_used
+
+    await app.shutdown()
+
+
 @pytest.mark.parametrize("device_cls", FORMED_DEVICES)
 @pytest.mark.parametrize("fw_assoc_remove", [True, False])
 @pytest.mark.parametrize("final_status", [t.Status.SUCCESS, t.Status.APS_NO_ACK])
@@ -744,7 +805,6 @@ async def test_request_recovery_assoc_remove(
     app._znp._config[conf.CONF_ZNP_CONFIG][conf.CONF_ARSP_TIMEOUT] = 1
 
     device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xABCD)
-    device.node_desc, _ = device.node_desc.deserialize(bytes(14))
 
     assoc_device, _ = c.util.Device.deserialize(b"\xFF" * 100)
     assoc_device.shortAddr = device.nwk
@@ -880,7 +940,6 @@ async def test_request_recovery_manual_source_route(
 
     device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xABCD)
     device.relays = relays
-    device.node_desc, _ = device.node_desc.deserialize(bytes(14))
 
     def data_confirm_replier(req):
         if isinstance(req, c.AF.DataRequestExt.Req) or not succeed:

tests/application/test_startup.py

@@ -71,6 +71,7 @@ async def test_info(
     assert app.channel is None
     assert app.channels is None
     assert app.network_key is None
+    assert app.network_key_seq is None
 
     await app.startup(auto_form=False)
 
@@ -84,6 +85,7 @@ async def test_info(
     assert app.channel == channel
     assert app.channels == channels
     assert app.network_key == network_key
+    assert app.network_key_seq == 0
 
     assert app.zigpy_device.manufacturer == "Texas Instruments"
     assert app.zigpy_device.model == model

tests/application/test_zigpy_callbacks.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 
 import pytest
@@ -6,20 +7,34 @@
 import zigpy_znp.types as t
 import zigpy_znp.commands as c
 
-from ..conftest import FORMED_DEVICES
+from ..conftest import FORMED_DEVICES, CoroutineMock
 
 pytestmark = [pytest.mark.asyncio]
 
 
+def awaitable_mock(return_value):
+    mock_called = asyncio.get_running_loop().create_future()
+
+    def side_effect(*args, **kwargs):
+        mock_called.set_result((args, kwargs))
+
+        return return_value
+
+    return mock_called, CoroutineMock(side_effect=side_effect)
+
+
 @pytest.mark.parametrize("device", FORMED_DEVICES)
 async def test_on_zdo_relays_message_callback(device, make_application, mocker):
     app, znp_server = make_application(server_cls=device)
     await app.startup(auto_form=False)
 
     device = mocker.Mock()
-    mocker.patch.object(app, "get_device", return_value=device)
+    discover_called, discover_mock = awaitable_mock(return_value=device)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
 
     znp_server.send(c.ZDO.SrcRtgInd.Callback(DstAddr=0x1234, Relays=[0x5678, 0xABCD]))
+
+    await discover_called
     assert device.relays == [0x5678, 0xABCD]
 
     await app.shutdown()
@@ -32,8 +47,13 @@ async def test_on_zdo_relays_message_callback_unknown(
     app, znp_server = make_application(server_cls=device)
     await app.startup(auto_form=False)
 
+    discover_called, discover_mock = awaitable_mock(return_value=None)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
+
     caplog.set_level(logging.WARNING)
     znp_server.send(c.ZDO.SrcRtgInd.Callback(DstAddr=0x1234, Relays=[0x5678, 0xABCD]))
+
+    await discover_called
     assert "unknown device" in caplog.text
 
     await app.shutdown()
@@ -98,12 +118,10 @@ async def test_on_af_message_callback(device, make_application, mocker):
     await app.startup(auto_form=False)
 
     device = mocker.Mock()
-    mocker.patch.object(
-        app,
-        "get_device",
-        side_effect=[device, device, device, KeyError("No such device")],
-    )
+    discover_called, discover_mock = awaitable_mock(return_value=device)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
     mocker.patch.object(app, "handle_message")
+    mocker.patch.object(app, "get_device")
 
     af_message = c.AF.IncomingMsg.Callback(
         GroupId=1,
@@ -123,43 +141,56 @@ async def test_on_af_message_callback(device, make_application, mocker):
 
     # Normal message
     znp_server.send(af_message)
-    app.get_device.assert_called_once_with(nwk=0xABCD)
+
+    await discover_called
     device.radio_details.assert_called_once_with(lqi=19, rssi=None)
     app.handle_message.assert_called_once_with(
         sender=device, profile=260, cluster=2, src_ep=4, dst_ep=1, message=b"test"
     )
 
-    # ZLL message
     device.reset_mock()
     app.handle_message.reset_mock()
     app.get_device.reset_mock()
 
+    # ZLL message
+    discover_called, discover_mock = awaitable_mock(return_value=device)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
+
     znp_server.send(af_message.replace(DstEndpoint=2))
-    app.get_device.assert_called_once_with(nwk=0xABCD)
+
+    await discover_called
     device.radio_details.assert_called_once_with(lqi=19, rssi=None)
     app.handle_message.assert_called_once_with(
         sender=device, profile=49246, cluster=2, src_ep=4, dst_ep=2, message=b"test"
     )
 
-    # Message on an unknown endpoint (is this possible?)
     device.reset_mock()
     app.handle_message.reset_mock()
     app.get_device.reset_mock()
 
+    # Message on an unknown endpoint (is this possible?)
+    discover_called, discover_mock = awaitable_mock(return_value=device)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
+
     znp_server.send(af_message.replace(DstEndpoint=3))
-    app.get_device.assert_called_once_with(nwk=0xABCD)
+
+    await discover_called
     device.radio_details.assert_called_once_with(lqi=19, rssi=None)
     app.handle_message.assert_called_once_with(
         sender=device, profile=260, cluster=2, src_ep=4, dst_ep=3, message=b"test"
     )
 
-    # Message from an unknown device
     device.reset_mock()
     app.handle_message.reset_mock()
     app.get_device.reset_mock()
 
+    # Message from an unknown device
+    discover_called, discover_mock = awaitable_mock(return_value=None)
+    mocker.patch.object(app, "_get_or_discover_device", new=discover_mock)
+
     znp_server.send(af_message)
-    app.get_device.assert_called_once_with(nwk=0xABCD)
+
+    await discover_called
     assert device.radio_details.call_count == 0
     assert app.handle_message.call_count == 0