From bea417a548ed58c6efada0457ce183c25ffa0ad4 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 8 Dec 2021 17:10:46 -0500 Subject: [PATCH 01/19] Implement half-documented `SendData` command --- zigpy_znp/commands/zdo.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/zigpy_znp/commands/zdo.py b/zigpy_znp/commands/zdo.py index 878ef11a..6becf2bb 100644 --- a/zigpy_znp/commands/zdo.py +++ b/zigpy_znp/commands/zdo.py @@ -696,18 +696,19 @@ class ZDO(t.CommandsBase, subsystem=t.Subsystem.ZDO): rsp_schema=t.STATUS_SCHEMA, ) - # set rejoin backoff duration and rejoin scan duration for an end device - SetRejoinParams = t.CommandDef( + # XXX: Undocumented + SendData = t.CommandDef( t.CommandType.SREQ, - # in documentation CmdId=0x26 which conflict with discover req 0x28, req_schema=( + t.Param("Dst", t.NWK, "Short address of the destination"), + t.Param("TSN", t.uint8_t, "Transaction sequence number"), + t.Param("CommandId", t.uint16_t, "ZDO Command ID"), t.Param( - "BackoffDuraation", - t.uint32_t, - "Rejoin backoff duration for end device", + "Data", + t.Bytes, + "Data to send", ), - t.Param("ScanDuration", t.uint32_t, "Rejoin scan duration for end device"), ), rsp_schema=t.STATUS_SCHEMA, ) @@ -1429,3 +1430,19 @@ class ZDO(t.CommandsBase, subsystem=t.Subsystem.ZDO): 0xCB, rsp_schema=(t.Param("Duration", t.uint8_t, "Permit join duration"),), ) + + # set rejoin backoff duration and rejoin scan duration for an end device + SetRejoinParams = t.CommandDef( + t.CommandType.SREQ, + # in documentation CmdId=0x26 which conflict with discover req + 0xCC, + req_schema=( + t.Param( + "BackoffDuraation", + t.uint32_t, + "Rejoin backoff duration for end device", + ), + t.Param("ScanDuration", t.uint32_t, "Rejoin scan duration for end device"), + ), + rsp_schema=t.STATUS_SCHEMA, + ) From d90c8646d6a49a74dc9bc413ddf886a39bdb7837 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 8 Dec 2021 17:37:50 -0500 Subject: [PATCH 02/19] Replace ZDO converters with explicit ZDO callback handlers --- zigpy_znp/zigbee/application.py | 148 +++----------- zigpy_znp/zigbee/zdo_converters.py | 308 ----------------------------- 2 files changed, 28 insertions(+), 428 deletions(-) delete mode 100644 zigpy_znp/zigbee/zdo_converters.py diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 28e8a629..9661e710 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -21,8 +21,8 @@ import zigpy.zdo.types as zdo_t import zigpy.application from zigpy.zcl import clusters -from zigpy.types import ExtendedPanId, deserialize as list_deserialize -from zigpy.zdo.types import CLUSTERS as ZDO_CLUSTERS, ZDOCmd, ZDOHeader, MultiAddress +from zigpy.types import ExtendedPanId +from zigpy.zdo.types import ZDOCmd, MultiAddress from zigpy.exceptions import DeliveryError import zigpy_znp.const as const @@ -34,7 +34,6 @@ from zigpy_znp.utils import combine_concurrent_calls from zigpy_znp.exceptions import CommandNotRecognized, InvalidCommandResponse from zigpy_znp.types.nvids import OsalNvIds -from zigpy_znp.zigbee.zdo_converters import ZDO_CONVERTERS ZDO_ENDPOINT = 0 ZHA_ENDPOINT = 1 @@ -298,6 +297,10 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): await self.load_network_info() await self._register_endpoints() + # Receive a callback for every known ZDO command + for cluster_id in zdo_t.ZDOCmd: + await self._znp.request(c.ZDO.MsgCallbackRegister.Req(ClusterId=cluster_id)) + # Setup the coordinator as a zigpy device and initialize it to request node info self.devices[self.ieee] = ZNPCoordinator(self, self.ieee, self.nwk) await self.zigpy_device.schedule_initialize() @@ -655,6 +658,10 @@ def _bind_callbacks(self) -> None: c.ZDO.PermitJoinInd.Callback(partial=True), self.on_zdo_permit_join_message ) + self._znp.callback_for_response( + c.ZDO.MsgCbIncoming.Callback(partial=True), self.on_zdo_message + ) + # No-op handle commands that just create unnecessary WARNING logs self._znp.callback_for_response( c.ZDO.ParentAnnceRsp.Callback(partial=True), @@ -685,6 +692,22 @@ def on_intentionally_unhandled_message(self, msg: t.CommandBase) -> None: pass + async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: + """ + Global callback for all ZDO messages + """ + + device = await self._get_or_discover_device(nwk=msg.Src) + + self.handle_message( + sender=device, + profile=zigpy.profiles.zha.PROFILE_ID, + cluster=msg.ClusterId, + src_ep=ZDO_ENDPOINT, + dst_ep=ZDO_ENDPOINT, + message=t.uint8_t(msg.TSN).serialize() + msg.Data, + ) + def on_zdo_permit_join_message(self, msg: c.ZDO.PermitJoinInd.Callback) -> None: """ Coordinator join status change message. Only sent with Z-Stack 1.2 and 3.0. @@ -1070,37 +1093,6 @@ async def _limit_concurrency(self): if was_locked: self._currently_waiting_requests -= 1 - def _receive_zdo_message( - self, - cluster: ZDOCmd, - *, - tsn: t.uint8_t, - sender: zigpy.device.Device, - **zdo_kwargs, - ) -> None: - """ - Internal method that is mainly called by our ZDO request/response converters to - receive a "fake" ZDO message constructed from a cluster and args/kwargs. - """ - - field_names, field_types = ZDO_CLUSTERS[cluster] - assert set(zdo_kwargs) == set(field_names) - - # Type cast all of the field args and kwargs - zdo_args = [t(zdo_kwargs[name]) for name, t in zip(field_names, field_types)] - message = t.serialize_list([t.uint8_t(tsn)] + zdo_args) - - LOGGER.debug("Pretending we received a ZDO message: %s", message) - - self.handle_message( - sender=sender, - profile=ZDO_PROFILE, - cluster=cluster, - src_ep=ZDO_ENDPOINT, - dst_ep=ZDO_ENDPOINT, - message=message, - ) - async def _reconnect(self) -> None: """ Endlessly tries to reconnect to the currently configured radio. @@ -1230,83 +1222,6 @@ def _find_endpoint(self, dst_ep: int, profile: int, cluster: int) -> int: return candidates[-1] - async def _send_zdo_request( - self, dst_addr, dst_ep, src_ep, cluster, sequence, options, radius, data - ): - """ - Zigpy doesn't send ZDO requests via TI's ZDO_* MT commands, so it will never - receive a reply because ZNP intercepts ZDO replies, never sends a DataConfirm, - and instead replies with one of its ZDO_* MT responses. - - This method translates the ZDO_* MT response into one zigpy can handle. - """ - - LOGGER.debug( - "Intercepted a ZDO request: dst_addr=%s, dst_ep=%s, src_ep=%s, " - "cluster=%s, sequence=%s, options=%s, radius=%s, data=%s", - dst_addr, - dst_ep, - src_ep, - cluster, - sequence, - options, - radius, - data, - ) - - assert dst_ep == ZDO_ENDPOINT - - # Deserialize the ZDO request - zdo_hdr, data = ZDOHeader.deserialize(cluster, data) - field_names, field_types = ZDO_CLUSTERS[cluster] - zdo_args, _ = list_deserialize(data, field_types) - zdo_kwargs = dict(zip(field_names, zdo_args)) - - # TODO: Check out `ZDO.MsgCallbackRegister` - - if cluster not in ZDO_CONVERTERS: - LOGGER.error( - "ZDO converter for cluster %s has not been implemented!" - " Please open a GitHub issue and attach a debug log:" - " https://github.com/zigpy/zigpy-znp/issues/new", - cluster, - ) - raise RuntimeError("No ZDO converter") - - # Call the converter with the ZDO request's kwargs - req_factory, rsp_factory, zdo_rsp_factory = ZDO_CONVERTERS[cluster] - request = req_factory(dst_addr, **zdo_kwargs) - callback = rsp_factory(dst_addr, **zdo_kwargs) - - LOGGER.debug( - "Intercepted AP ZDO request %s(%s) and replaced with %s", - cluster, - zdo_kwargs, - request, - ) - - # The coordinator responds to broadcasts - if dst_addr.mode == t.AddrMode.Broadcast: - callback = callback.replace(Src=0x0000) - - async with async_timeout.timeout(ZDO_REQUEST_TIMEOUT): - response = await self._znp.request_callback_rsp( - request=request, RspStatus=t.Status.SUCCESS, callback=callback - ) - - # We should only send zigpy unicast responses - if dst_addr.mode == t.AddrMode.NWK: - zdo_rsp_cluster, zdo_response_kwargs = zdo_rsp_factory(response) - - self._receive_zdo_message( - cluster=zdo_rsp_cluster, - tsn=sequence, - sender=self.get_device(nwk=dst_addr.address), - **zdo_response_kwargs, - ) - - return response - async def _send_request_raw( self, dst_addr, @@ -1326,13 +1241,6 @@ async def _send_request_raw( Picks the correct request sending mechanism and fixes endpoint information. """ - # ZDO requests must be handled by the translation layer, since Z-Stack will - # "steal" the responses - if dst_ep == ZDO_ENDPOINT: - return await self._send_zdo_request( - dst_addr, dst_ep, src_ep, cluster, sequence, options, radius, data - ) - # Zigpy just sets src == dst, which doesn't work for devices with many endpoints # We pick ours based on the registered endpoints when using an older firmware src_ep = self._find_endpoint(dst_ep=dst_ep, profile=profile, cluster=cluster) @@ -1362,8 +1270,8 @@ async def _send_request_raw( Data=data, ) - if dst_addr.mode == t.AddrMode.Broadcast: - # Broadcasts will not receive a confirmation + if dst_addr.mode == t.AddrMode.Broadcast or dst_ep == ZDO_ENDPOINT: + # Broadcasts and ZDO requests will not receive a confirmation response = await self._znp.request( request=request, RspStatus=t.Status.SUCCESS ) diff --git a/zigpy_znp/zigbee/zdo_converters.py b/zigpy_znp/zigbee/zdo_converters.py deleted file mode 100644 index 05704bc3..00000000 --- a/zigpy_znp/zigbee/zdo_converters.py +++ /dev/null @@ -1,308 +0,0 @@ -from zigpy.zdo.types import ZDOCmd - -import zigpy_znp.commands as c - -""" -Zigpy expects to be able to directly send and receive ZDO commands. -Z-Stack, however, intercepts all responses and rewrites them to use its MT command set. -We must use proxy functions that rewrite requests and responses based on their cluster. -""" - - -# The structure of this dict is: -# { -# cluster: ( -# zigpy_req_to_mt_req_converter, -# mt_rsp_callback_matcher, -# mt_rsp_callback_to_zigpy_rsp_converter, -# ) -# - -ZDO_CONVERTERS = { - ZDOCmd.NWK_addr_req: ( - ( - lambda addr, IEEEAddr, RequestType, StartIndex: c.ZDO.NwkAddrReq.Req( - IEEE=IEEEAddr, - RequestType=c.zdo.AddrRequestType(RequestType), - StartIndex=StartIndex, - ) - ), - ( - lambda addr, IEEEAddr, **kwargs: c.ZDO.NwkAddrRsp.Callback( - partial=True, IEEE=IEEEAddr - ) - ), - ( - lambda rsp: ( - ZDOCmd.NWK_addr_rsp, - { - "Status": rsp.Status, - "IEEEAddr": rsp.IEEE, - "NWKAddr": rsp.NWK, - "NumAssocDev": len(rsp.Devices), - "StartIndex": rsp.Index, - "NWKAddressAssocDevList": rsp.Devices, # XXX: this is inconsistent - }, - ) - ), - ), - ZDOCmd.IEEE_addr_req: ( - ( - lambda addr, NWKAddrOfInterest, RequestType, StartIndex: ( - c.ZDO.IEEEAddrReq.Req( - NWK=NWKAddrOfInterest, - RequestType=c.zdo.AddrRequestType(RequestType), - StartIndex=StartIndex, - ) - ) - ), - ( - lambda addr, NWKAddrOfInterest, **kwargs: c.ZDO.IEEEAddrRsp.Callback( - partial=True, NWK=NWKAddrOfInterest - ) - ), - ( - lambda rsp: ( - ZDOCmd.IEEE_addr_rsp, - { - "Status": rsp.Status, - "IEEEAddr": rsp.IEEE, - "NWKAddr": rsp.NWK, - "NumAssocDev": len(rsp.Devices), - "StartIndex": rsp.Index, - "NWKAddrAssocDevList": rsp.Devices, - }, - ) - ), - ), - ZDOCmd.Node_Desc_req: ( - ( - lambda addr, NWKAddrOfInterest: c.ZDO.NodeDescReq.Req( - DstAddr=addr.address, NWKAddrOfInterest=NWKAddrOfInterest - ) - ), - ( - lambda addr, **kwargs: c.ZDO.NodeDescRsp.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Node_Desc_rsp, - { - "Status": rsp.Status, - "NWKAddrOfInterest": rsp.NWK, - "NodeDescriptor": rsp.NodeDescriptor, - }, - ) - ), - ), - ZDOCmd.Active_EP_req: ( - ( - lambda addr, NWKAddrOfInterest: c.ZDO.ActiveEpReq.Req( - DstAddr=addr.address, NWKAddrOfInterest=NWKAddrOfInterest - ) - ), - ( - lambda addr, **kwargs: c.ZDO.ActiveEpRsp.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Active_EP_rsp, - { - "Status": rsp.Status, - "NWKAddrOfInterest": rsp.NWK, - "ActiveEPList": rsp.ActiveEndpoints, - }, - ) - ), - ), - ZDOCmd.Simple_Desc_req: ( - ( - lambda addr, NWKAddrOfInterest, EndPoint: ( - c.ZDO.SimpleDescReq.Req( - DstAddr=addr.address, - NWKAddrOfInterest=NWKAddrOfInterest, - Endpoint=EndPoint, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.SimpleDescRsp.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Simple_Desc_rsp, - { - "Status": rsp.Status, - "NWKAddrOfInterest": rsp.NWK, - "SimpleDescriptor": rsp.SimpleDescriptor, - }, - ) - ), - ), - ZDOCmd.Mgmt_Permit_Joining_req: ( - ( - lambda addr, PermitDuration, TC_Significant: ( - c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=addr.mode, - Dst=addr.address, - Duration=PermitDuration, - TCSignificance=TC_Significant, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtPermitJoinRsp.Callback( - partial=True, Src=addr.address - ) - ), - (lambda rsp: (ZDOCmd.Mgmt_Permit_Joining_rsp, {"Status": rsp.Status})), - ), - ZDOCmd.Mgmt_Leave_req: ( - ( - lambda addr, DeviceAddress, Options: c.ZDO.MgmtLeaveReq.Req( - DstAddr=addr.address, - IEEE=DeviceAddress, - RemoveChildren_Rejoin=c.zdo.LeaveOptions(Options), - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtLeaveRsp.Callback( - partial=True, Src=addr.address - ) - ), - (lambda rsp: (ZDOCmd.Mgmt_Leave_rsp, {"Status": rsp.Status})), - ), - ZDOCmd.Mgmt_NWK_Update_req: ( - ( - lambda addr, NwkUpdate: c.ZDO.MgmtNWKUpdateReq.Req( - Dst=addr.address, - DstAddrMode=addr.mode, - Channels=NwkUpdate.ScanChannels, - ScanDuration=NwkUpdate.ScanDuration, - ScanCount=NwkUpdate.ScanCount or 0x00, - # XXX: nwkUpdateId is hard-coded to `_NIB.nwkUpdateId + 1` - NwkManagerAddr=NwkUpdate.nwkManagerAddr or 0x0000, - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtNWKUpdateNotify.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Mgmt_NWK_Update_rsp, - { - "Status": rsp.Status, - "ScannedChannels": rsp.ScannedChannels, - "TotalTransmissions": rsp.TotalTransmissions, - "TransmissionFailures": rsp.TransmissionFailures, - "EnergyValues": rsp.EnergyValues, - }, - ) - ), - ), - ZDOCmd.Bind_req: ( - ( - lambda addr, SrcAddress, SrcEndpoint, ClusterID, DstAddress: ( - c.ZDO.BindReq.Req( - Dst=addr.address, - Src=SrcAddress, - SrcEndpoint=SrcEndpoint, - ClusterId=ClusterID, - Address=DstAddress, - ) - ) - ), - (lambda addr, **kwargs: c.ZDO.BindRsp.Callback(partial=True, Src=addr.address)), - (lambda rsp: (ZDOCmd.Bind_rsp, {"Status": rsp.Status})), - ), - ZDOCmd.Unbind_req: ( - ( - lambda addr, SrcAddress, SrcEndpoint, ClusterID, DstAddress: ( - c.ZDO.UnBindReq.Req( - Dst=addr.address, - Src=SrcAddress, - SrcEndpoint=SrcEndpoint, - ClusterId=ClusterID, - Address=DstAddress, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.UnBindRsp.Callback( - partial=True, Src=addr.address - ) - ), - (lambda rsp: (ZDOCmd.Unbind_rsp, {"Status": rsp.Status})), - ), - ZDOCmd.Mgmt_Lqi_req: ( - ( - lambda addr, StartIndex: ( - c.ZDO.MgmtLqiReq.Req( - Dst=addr.address, - StartIndex=StartIndex, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtLqiRsp.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Mgmt_Lqi_rsp, - {"Status": rsp.Status, "Neighbors": rsp.Neighbors}, - ) - ), - ), - ZDOCmd.Mgmt_Rtg_req: ( - ( - lambda addr, StartIndex: ( - c.ZDO.MgmtRtgReq.Req( - Dst=addr.address, - StartIndex=StartIndex, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtRtgRsp.Callback( - partial=True, Src=addr.address - ) - ), - (lambda rsp: (ZDOCmd.Mgmt_Rtg_rsp, {"Status": rsp.Status})), - ), - ZDOCmd.Mgmt_Bind_req: ( - ( - lambda addr, StartIndex: ( - c.ZDO.MgmtBindReq.Req( - Dst=addr.address, - StartIndex=StartIndex, - ) - ) - ), - ( - lambda addr, **kwargs: c.ZDO.MgmtBindRsp.Callback( - partial=True, Src=addr.address - ) - ), - ( - lambda rsp: ( - ZDOCmd.Mgmt_Bind_rsp, - { - "Status": rsp.Status, - "BindingTableEntries": rsp.BindTableEntries, - "StartIndex": rsp.StartIndex, - "BindingTableList": rsp.BindTableList, - }, - ) - ), - ), -} From 0639262afe48fcbac159546db79938f1fc7a2307 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 8 Dec 2021 21:35:04 -0500 Subject: [PATCH 03/19] Fix join broadcasts when using raw ZDO requests --- zigpy_znp/zigbee/application.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 9661e710..fd48d635 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -284,11 +284,6 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): self._version_rsp = await self._znp.request(c.SYS.Version.Req()) - # XXX: The CC2531 running Z-Stack Home 1.2 permanently permits joins on startup - # unless they are explicitly disabled. We can't fix this but we can disable them - # as early as possible to shrink the window of opportunity for unwanted joins. - await self.permit(time_s=0) - # The CC2531 running Z-Stack Home 1.2 overrides the LED setting if it is changed # before the coordinator has started. if self.znp_config[conf.CONF_LED_MODE] is not None: @@ -337,6 +332,11 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): self._watchdog_task = asyncio.create_task(self._watchdog_loop()) + # XXX: The CC2531 running Z-Stack Home 1.2 permanently permits joins on startup + # unless they are explicitly disabled. We can't fix this but we can disable them + # as early as possible to shrink the window of opportunity for unwanted joins. + await self.permit(time_s=0) + async def set_tx_power(self, dbm: int) -> None: """ Sets the radio TX power. @@ -540,19 +540,10 @@ async def permit(self, time_s=60, node=None): # through the coordinator itself. # # Fixed in https://github.com/Koenkk/Z-Stack-firmware/commit/efac5ee46b9b437 - if time_s == 0 or self._zstack_build_id < 20210708 or node == self.ieee: - response = await self._znp.request_callback_rsp( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, - Dst=0x0000, - Duration=time_s, - TCSignificance=1, - ), - RspStatus=t.Status.SUCCESS, - callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), - ) + if time_s == 0 or self._zstack_build_id < 20210708 or node in (None, self.ieee): + response = await self.zigpy_device.zdo.Mgmt_Permit_Joining_req(time_s, 1) - if response.Status != t.Status.SUCCESS: + if response[0] != t.Status.SUCCESS: raise RuntimeError(f"Failed to permit joins on coordinator: {response}") await super().permit(time_s=time_s, node=node) From da205916e80313655b95391af7063520e20a98ab Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 8 Dec 2021 21:35:58 -0500 Subject: [PATCH 04/19] Handle ZDO device announcements --- zigpy_znp/zigbee/application.py | 86 ++++++++++++++++----------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index fd48d635..b8187fec 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -22,7 +22,6 @@ import zigpy.application from zigpy.zcl import clusters from zigpy.types import ExtendedPanId -from zigpy.zdo.types import ZDOCmd, MultiAddress from zigpy.exceptions import DeliveryError import zigpy_znp.const as const @@ -298,6 +297,7 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): # Setup the coordinator as a zigpy device and initialize it to request node info self.devices[self.ieee] = ZNPCoordinator(self, self.ieee, self.nwk) + self.zigpy_device.zdo.add_listener(self) await self.zigpy_device.schedule_initialize() # Now that we know what device we are, set the max concurrent requests @@ -414,7 +414,7 @@ async def form_network(self): await self._write_stack_settings(reset_if_changed=False) await self._znp.reset() - def get_dst_address(self, cluster: zigpy.zcl.Cluster) -> MultiAddress: + def get_dst_address(self, cluster: zigpy.zcl.Cluster) -> zdo_t.MultiAddress: """ Helper to get a dst address for bind/unbind operations. @@ -422,7 +422,7 @@ def get_dst_address(self, cluster: zigpy.zcl.Cluster) -> MultiAddress: on specific endpoints only. """ - dst_addr = MultiAddress() + dst_addr = zdo_t.MultiAddress() dst_addr.addrmode = 0x03 dst_addr.ieee = self.ieee dst_addr.endpoint = self._find_endpoint( @@ -626,12 +626,6 @@ def _bind_callbacks(self) -> None: c.AF.IncomingMsg.Callback(partial=True), self.on_af_message ) - # ZDO requests need to be handled explicitly, one by one - self._znp.callback_for_response( - c.ZDO.EndDeviceAnnceInd.Callback(partial=True), - self.on_zdo_device_announce, - ) - self._znp.callback_for_response( c.ZDO.TCDevInd.Callback.Callback(partial=True), self.on_zdo_tc_device_join, @@ -653,21 +647,23 @@ def _bind_callbacks(self) -> None: c.ZDO.MsgCbIncoming.Callback(partial=True), self.on_zdo_message ) - # No-op handle commands that just create unnecessary WARNING logs - self._znp.callback_for_response( - c.ZDO.ParentAnnceRsp.Callback(partial=True), - self.on_intentionally_unhandled_message, - ) - - self._znp.callback_for_response( - c.ZDO.ConcentratorInd.Callback(partial=True), - self.on_intentionally_unhandled_message, - ) - - self._znp.callback_for_response( - c.ZDO.MgmtNWKUpdateNotify.Callback(partial=True), - self.on_intentionally_unhandled_message, - ) + # Handle messages that we do not use to prevent unnecessary WARNINGs in logs + for ignored_msg in [ + c.ZDO.EndDeviceAnnceInd, + c.ZDO.LeaveInd, + c.ZDO.PermitJoinInd, + c.ZDO.ParentAnnceRsp, + c.ZDO.ConcentratorInd, + c.ZDO.MgmtNWKUpdateNotify, + c.ZDO.MgmtPermitJoinRsp, + c.ZDO.NodeDescRsp, + c.ZDO.SimpleDescRsp, + c.ZDO.ActiveEpRsp, + ]: + self._znp.callback_for_response( + ignored_msg.Callback(partial=True), + self.on_intentionally_unhandled_message, + ) # These are responses to a broadcast but we ignore all but the first self._znp.callback_for_response( @@ -685,20 +681,31 @@ def on_intentionally_unhandled_message(self, msg: t.CommandBase) -> None: async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: """ - Global callback for all ZDO messages + Global callback for all ZDO messages. """ device = await self._get_or_discover_device(nwk=msg.Src) + if device is None: + LOGGER.warning("Received a ZDO message from an unknown device: %s", msg.Src) + return + + message = t.uint8_t(msg.TSN).serialize() + msg.Data + self.handle_message( sender=device, profile=zigpy.profiles.zha.PROFILE_ID, cluster=msg.ClusterId, src_ep=ZDO_ENDPOINT, dst_ep=ZDO_ENDPOINT, - message=t.uint8_t(msg.TSN).serialize() + msg.Data, + message=message, ) + hdr, args = device.zdo.deserialize(msg.ClusterId, message) + + if msg.ClusterId == zdo_t.ZDOCmd.Device_annce: + self.on_zdo_device_announce(*args) + def on_zdo_permit_join_message(self, msg: c.ZDO.PermitJoinInd.Callback) -> None: """ Coordinator join status change message. Only sent with Z-Stack 1.2 and 3.0. @@ -725,31 +732,24 @@ async def on_zdo_relays_message(self, msg: c.ZDO.SrcRtgInd.Callback) -> None: # `relays` is a property with a setter that emits an event device.relays = msg.Relays - def on_zdo_device_announce(self, msg: c.ZDO.EndDeviceAnnceInd.Callback) -> None: + def on_zdo_device_announce(self, nwk: t.NWK, ieee: t.EUI64, capabilities) -> None: """ ZDO end device announcement callback """ - LOGGER.info("ZDO device announce: %s", msg) + LOGGER.info( + "ZDO device announce: nwk=%s, ieee=%s, capabilities=%s", + nwk, + ieee, + capabilities, + ) # Cancel an existing join timer so we don't double announce - if msg.IEEE in self._join_announce_tasks: - self._join_announce_tasks.pop(msg.IEEE).cancel() + if ieee in self._join_announce_tasks: + self._join_announce_tasks.pop(ieee).cancel() # Sometimes devices change their NWK when announcing so re-join it. - self.handle_join(nwk=msg.NWK, ieee=msg.IEEE, parent_nwk=None) - - device = self.get_device(ieee=msg.IEEE) - - # We turn this back into a ZDO message and let zigpy handle it - self._receive_zdo_message( - cluster=ZDOCmd.Device_annce, - tsn=0xFF, - sender=device, - NWKAddr=msg.NWK, - IEEEAddr=msg.IEEE, - Capability=msg.Capabilities, - ) + self.handle_join(nwk=nwk, ieee=ieee, parent_nwk=None) def on_zdo_tc_device_join(self, msg: c.ZDO.TCDevInd.Callback) -> None: """ From 4bf893c49b0636e9a55bbd30db3a9960197392d6 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 8 Dec 2021 21:39:12 -0500 Subject: [PATCH 05/19] Remove old constants --- tests/application/test_connect.py | 2 +- zigpy_znp/zigbee/application.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/application/test_connect.py b/tests/application/test_connect.py index 238a226e..2a0fb022 100644 --- a/tests/application/test_connect.py +++ b/tests/application/test_connect.py @@ -63,7 +63,7 @@ async def test_probe_unsuccessful_slow(device, make_znp_server, mocker): # Don't respond to anything znp_server._listeners.clear() - mocker.patch("zigpy_znp.zigbee.application.PROBE_TIMEOUT", new=0.1) + mocker.patch("zigpy_znp.api.CONNECT_PROBE_TIMEOUT", new=0.1) assert not ( await ControllerApplication.probe( diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index b8187fec..aebf76fb 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -41,15 +41,11 @@ ZDO_PROFILE = 0x0000 # All of these are in seconds -PROBE_TIMEOUT = 5 STARTUP_TIMEOUT = 5 -ZDO_REQUEST_TIMEOUT = 15 DATA_CONFIRM_TIMEOUT = 8 IEEE_ADDR_DISCOVERY_TIMEOUT = 5 DEVICE_JOIN_MAX_DELAY = 5 WATCHDOG_PERIOD = 30 -BROADCAST_SEND_WAIT_DURATION = 3 -MULTICAST_SEND_WAIT_DURATION = 3 REQUEST_MAX_RETRIES = 5 REQUEST_ERROR_RETRY_DELAY = 0.5 From 718afd784e4b6b0c96c6d5726a589fe8ad89db0e Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 31 Dec 2021 12:16:05 -0500 Subject: [PATCH 06/19] Begin fixing unit testing infrastructure --- tests/application/test_requests.py | 55 +----- tests/conftest.py | 282 ++++++++++++++++++++--------- 2 files changed, 197 insertions(+), 140 deletions(-) diff --git a/tests/application/test_requests.py b/tests/application/test_requests.py index f96ceef4..979c629a 100644 --- a/tests/application/test_requests.py +++ b/tests/application/test_requests.py @@ -4,7 +4,7 @@ import zigpy.zdo import zigpy.endpoint import zigpy.profiles -from zigpy.zdo.types import ZDOCmd, SizePrefixedSimpleDescriptor +from zigpy.zdo.types import ZDOCmd from zigpy.exceptions import DeliveryError import zigpy_znp.types as t @@ -15,55 +15,6 @@ from ..conftest import FORMED_DEVICES, CoroutineMock, FormedLaunchpadCC26X2R1 -@pytest.mark.parametrize("device", FORMED_DEVICES) -async def test_zdo_request_interception(device, make_application): - app, znp_server = make_application(server_cls=device) - await app.startup(auto_form=False) - - device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xFA9E) - - # Send back a request response - active_ep_req = znp_server.reply_once_to( - request=c.ZDO.SimpleDescReq.Req( - DstAddr=device.nwk, NWKAddrOfInterest=device.nwk, Endpoint=1 - ), - responses=[ - c.ZDO.SimpleDescReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.SimpleDescRsp.Callback( - Src=device.nwk, - Status=t.ZDOStatus.SUCCESS, - NWK=device.nwk, - SimpleDescriptor=SizePrefixedSimpleDescriptor( - *dict( - endpoint=1, - profile=49246, - device_type=256, - device_version=2, - input_clusters=[0, 3, 4, 5, 6, 8, 2821, 4096], - output_clusters=[5, 25, 32, 4096], - ).values() - ), - ), - ], - ) - - status, message = await app.request( - device=device, - profile=260, - cluster=ZDOCmd.Simple_Desc_req, - src_ep=0, - dst_ep=0, - sequence=1, - data=b"\x01\x9e\xfa\x01", - use_ieee=False, - ) - - assert status == t.Status.SUCCESS - await active_ep_req - - await app.shutdown() - - @pytest.mark.parametrize("device", FORMED_DEVICES) async def test_chosen_dst_endpoint(device, make_application, mocker): app, znp_server = make_application(device) @@ -91,7 +42,7 @@ async def test_zigpy_request(device, make_application): app, znp_server = make_application(device) await app.startup(auto_form=False) - TSN = 6 + TSN = 7 device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xAABB) @@ -151,7 +102,7 @@ async def test_zigpy_request_failure(device, make_application, mocker): app, znp_server = make_application(device) await app.startup(auto_form=False) - TSN = 6 + TSN = 7 device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xAABB) diff --git a/tests/conftest.py b/tests/conftest.py index f9dac9a6..c7a6f9a5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ from unittest.mock import Mock, PropertyMock import pytest +import zigpy.types import zigpy.device try: @@ -373,6 +374,19 @@ def inner(function): return inner +def serialize_zdo_command(command_id, **kwargs): + field_names, field_types = zdo_t.CLUSTERS[command_id] + + return t.Bytes(zigpy.types.serialize(kwargs.values(), field_types)) + + +def deserialize_zdo_command(command_id, data): + field_names, field_types = zdo_t.CLUSTERS[command_id] + args, data = zigpy.types.deserialize(data, field_types) + + return dict(zip(field_names, args)) + + class BaseZStackDevice(BaseServerZNP): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -381,6 +395,7 @@ def __init__(self, *args, **kwargs): self._nvram = {} self.device_state = t.DeviceState.InitializedNotStarted + self.zdo_callbacks = set() # Handle the decorators for name in dir(self): @@ -531,18 +546,79 @@ def _default_nib(self): nwkUpdateId=0, ) - @reply_to(c.ZDO.ActiveEpReq.Req(DstAddr=0x0000, NWKAddrOfInterest=0x0000)) - def active_endpoints_request(self, req): - return [ - c.ZDO.ActiveEpReq.Rsp(Status=t.Status.SUCCESS), + @reply_to(c.AF.DataRequestExt.Req(partial=True, DstEndpoint=0)) + def on_zdo_request(self, req): + kwargs = deserialize_zdo_command(req.ClusterId, req.Data[1:]) + handler_name = f"on_zdo_{zdo_t.ZDOCmd(req.ClusterId).name.lower()}" + handler = getattr(self, handler_name, None) + + if handler is None: + LOGGER.warning("No ZDO handler %s, kwargs: %s", handler_name, kwargs) + return + + responses = handler(req=req, **kwargs) or [] + + return [c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)] + responses + + def on_zdo_mgmt_permit_joining_req(self, req, PermitDuration, TC_Significant): + if req.DstAddrModeAddress.address != 0x0000: + return + + responses = [ + c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS) + ] + + if zdo_t.ZDOCmd.Mgmt_Permit_Joining_rsp in self.zdo_callbacks: + responses.append( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Mgmt_Permit_Joining_rsp, + SecurityUse=0, + TSN=req.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_rsp, + Status=t.ZDOStatus.SUCCESS, + ), + ) + ) + + return responses + + def on_zdo_active_ep_req(self, req, NWKAddrOfInterest): + if NWKAddrOfInterest != 0x0000: + return + + responses = [ c.ZDO.ActiveEpRsp.Callback( Src=0x0000, Status=t.ZDOStatus.SUCCESS, NWK=0x0000, ActiveEndpoints=[ep.Endpoint for ep in self.active_endpoints], - ), + ) ] + if zdo_t.ZDOCmd.Active_EP_rsp in self.zdo_callbacks: + responses.append( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Active_EP_rsp, + SecurityUse=0, + TSN=req.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Active_EP_rsp, + Status=t.ZDOStatus.SUCCESS, + NWKAddrOfInterest=0x0000, + ActiveEPList=[ep.Endpoint for ep in self.active_endpoints], + ), + ) + ) + + return responses + @reply_to(c.AF.Register.Req(partial=True)) def on_endpoint_registration(self, req): self.active_endpoints.insert(0, req) @@ -555,31 +631,53 @@ def on_endpoint_deletion(self, req): return c.AF.Delete.Rsp(Status=t.Status.SUCCESS) - @reply_to( - c.ZDO.SimpleDescReq.Req(DstAddr=0x0000, NWKAddrOfInterest=0x0000, partial=True) - ) - def on_simple_desc_req(self, req): + def on_zdo_simple_desc_req(self, req, NWKAddrOfInterest, EndPoint): + if NWKAddrOfInterest != 0x0000: + return + for ep in self.active_endpoints: - if ep.Endpoint == req.Endpoint: - return [ - c.ZDO.SimpleDescReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.SimpleDescRsp.Callback( - Src=0x0000, + if ep.Endpoint == EndPoint: + break + else: + # Bad things happen when an invalid endpoint ID is passed in + pytest.fail("Simple descriptor request to invalid endpoint breaks Z-Stack") + return + + responses = [ + c.ZDO.SimpleDescRsp.Callback( + Src=0x0000, + Status=t.ZDOStatus.SUCCESS, + NWK=0x0000, + SimpleDescriptor=zdo_t.SizePrefixedSimpleDescriptor( + endpoint=ep.Endpoint, + profile=ep.ProfileId, + device_type=ep.DeviceId, + device_version=ep.DeviceVersion, + input_clusters=ep.InputClusters, + output_clusters=ep.OutputClusters, + ), + ), + ] + + if zdo_t.ZDOCmd.Simple_Desc_rsp in self.zdo_callbacks: + responses.append( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Simple_Desc_rsp, + SecurityUse=0, + TSN=req.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Simple_Desc_rsp, Status=t.ZDOStatus.SUCCESS, - NWK=0x0000, - SimpleDescriptor=zdo_t.SizePrefixedSimpleDescriptor( - endpoint=ep.Endpoint, - profile=ep.ProfileId, - device_type=ep.DeviceId, - device_version=ep.DeviceVersion, - input_clusters=ep.InputClusters, - output_clusters=ep.OutputClusters, - ), + NWKAddrOfInterest=0x0000, + SimpleDescriptor=responses[0].SimpleDescriptor, ), - ] + ) + ) - # Bad things happen when an invalid endpoint ID is passed in - pytest.fail("Simple descriptor request to an invalid endpoint breaks Z-Stack") + return responses @reply_to(c.SYS.OSALNVWrite.Req(partial=True)) @reply_to(c.SYS.OSALNVWriteExt.Req(partial=True)) @@ -714,30 +812,15 @@ def util_device_info(self, request): AssociatedDevices=[], ) - @reply_to( - c.ZDO.MgmtNWKUpdateReq.Req(Dst=0x0000, DstAddrMode=t.AddrMode.NWK, partial=True) - ) - def nwk_update_req(self, request): - valid_channels = [t.Channels.from_channel_list([i]) for i in range(11, 26 + 1)] - - if request.ScanDuration == 0xFE: - assert request.Channels in valid_channels - - def update_channel(): - nib = self.nib - nib.nwkLogicalChannel = 11 + valid_channels.index(request.Channels) - nib.nwkUpdateId += 1 - - self.nib = nib - - asyncio.get_running_loop().call_later(0.1, update_channel) - - return c.ZDO.MgmtNWKUpdateReq.Rsp(Status=t.Status.SUCCESS) - @reply_to(c.ZDO.ExtRouteChk.Req(partial=True)) def zdo_route_check(self, request): return c.ZDO.ExtRouteChk.Rsp(Status=c.zdo.RoutingStatus.SUCCESS) + @reply_to(c.ZDO.MsgCallbackRegister.Req(partial=True)) + def register_zdo_callback(self, request): + self.zdo_callbacks.add(request.ClusterId) + return c.ZDO.MsgCallbackRegister.Rsp(Status=t.Status.SUCCESS) + @reply_to(c.UTIL.AssocFindDevice.Req(Index=0)) def assoc_find_dev_responder(self, req): return req.Rsp(Device=t.Bytes(b"\xFF" * (36 if self.align_structs else 28))) @@ -822,10 +905,11 @@ def startup_from_app(self, req): self.create_nib, ] - @reply_to(c.ZDO.NodeDescReq.Req(DstAddr=0x0000, NWKAddrOfInterest=0x0000)) - def node_desc_responder(self, req): - return [ - c.ZDO.NodeDescReq.Rsp(Status=t.Status.SUCCESS), + def on_zdo_node_desc_req(self, req, NWKAddrOfInterest): + if NWKAddrOfInterest != 0x0000: + return + + responses = [ c.ZDO.NodeDescRsp.Callback( Src=0x0000, Status=t.ZDOStatus.SUCCESS, @@ -844,25 +928,40 @@ def node_desc_responder(self, req): ), ] - @reply_to( - c.ZDO.MgmtPermitJoinReq.Req(AddrMode=t.AddrMode.NWK, Dst=0x0000, partial=True) - ) - @reply_to( - c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, partial=True + if zdo_t.ZDOCmd.Node_Desc_rsp in self.zdo_callbacks: + responses.append( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Node_Desc_rsp, + SecurityUse=0, + TSN=req.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Node_Desc_rsp, + Status=t.ZDOStatus.SUCCESS, + NWKAddrOfInterest=0x0000, + NodeDescriptor=zdo_t.NodeDescriptor( + **responses[0].NodeDescriptor.as_dict() + ), + ), + ) + ) + + return responses + + def on_zdo_mgmt_permit_joining_req(self, req, PermitDuration, TC_Significant): + result = super().on_zdo_mgmt_permit_joining_req( + req, PermitDuration, TC_Significant ) - ) - def permit_join(self, request): - if request.Duration != 0: - rsp = [c.ZDO.PermitJoinInd.Callback(Duration=request.Duration)] - else: - rsp = [] - return rsp + [ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), - c.ZDO.PermitJoinInd.Callback(Duration=0), - ] + if not result: + return + + if PermitDuration != 0: + result = [c.ZDO.PermitJoinInd.Callback(Duration=req.Duration)] + result + + return result + [c.ZDO.PermitJoinInd.Callback(Duration=0)] @reply_to(c.UTIL.LEDControl.Req(partial=True)) def led_responder(self, req): @@ -888,22 +987,6 @@ def handle_bdb_set_primary_channel(self, request): return c.AppConfig.BDBSetChannel.Rsp(Status=t.Status.SUCCESS) - @reply_to( - c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=0, partial=True - ) - ) - @reply_to( - c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, Duration=0, partial=True - ) - ) - def permit_join(self, request): - return [ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), - ] - def create_nib(self, _=None): super().create_nib() @@ -1077,10 +1160,11 @@ def version_replier(self, request): BootloaderRevision=0xFFFFFFFF, ) - @reply_to(c.ZDO.NodeDescReq.Req(DstAddr=0x0000, NWKAddrOfInterest=0x0000)) - def node_desc_responder(self, req): - return [ - c.ZDO.NodeDescReq.Rsp(Status=t.Status.SUCCESS), + def on_zdo_node_desc_req(self, req, NWKAddrOfInterest): + if NWKAddrOfInterest != 0x0000: + return + + responses = [ c.ZDO.NodeDescRsp.Callback( Src=0x0000, Status=t.ZDOStatus.SUCCESS, @@ -1099,6 +1183,28 @@ def node_desc_responder(self, req): ), ] + if zdo_t.ZDOCmd.Node_Desc_rsp in self.zdo_callbacks: + responses.append( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Node_Desc_rsp, + SecurityUse=0, + TSN=req.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Node_Desc_rsp, + Status=t.ZDOStatus.SUCCESS, + NWKAddrOfInterest=0x0000, + NodeDescriptor=zdo_t.NodeDescriptor.replace( + responses[0].NodeDescriptor + ), + ), + ) + ) + + return responses + @reply_to(c.UTIL.LEDControl.Req(partial=True)) def led_responder(self, req): # XXX: Yes, there is *no response* @@ -1151,7 +1257,7 @@ def version_replier(self, request): BootloaderRevision=0, ) - node_desc_responder = BaseZStack1CC2531.node_desc_responder + on_zdo_node_desc_req = BaseZStack1CC2531.on_zdo_node_desc_req @reply_to(c.UTIL.LEDControl.Req(partial=True)) def led_responder(self, req): From 4d342e181a4e5d243a37155e11a3e36df252ace0 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Sat, 8 Jan 2022 12:06:43 -0500 Subject: [PATCH 07/19] Update joining unit tests --- tests/application/test_joining.py | 423 +++++++----------------- tests/conftest.py | 23 +- tests/nvram/CC2652R-ZStack4.formed.json | 1 + zigpy_znp/zigbee/application.py | 86 +++-- 4 files changed, 202 insertions(+), 331 deletions(-) diff --git a/tests/application/test_joining.py b/tests/application/test_joining.py index 11cbac4c..3f0fdabe 100644 --- a/tests/application/test_joining.py +++ b/tests/application/test_joining.py @@ -14,6 +14,8 @@ FORMED_ZSTACK3_DEVICES, CoroutineMock, FormedLaunchpadCC26X2R1, + zdo_request_matcher, + serialize_zdo_command, ) @@ -29,22 +31,30 @@ async def test_permit_join(device, fixed_joining_bug, mocker, make_application): # Handle us opening joins on the coordinator permit_join_coordinator = znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=10, partial=True + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.NWK, 0x0000), + command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, + TSN=7, + zdo_PermitDuration=10, + zdo_TC_Significant=0, ), responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), + c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) # Handle the ZDO broadcast sent by Zigpy permit_join_broadcast = znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, Duration=10, partial=True + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.Broadcast, 0xFFFC), + command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, + TSN=8 if not fixed_joining_bug else 7, + zdo_PermitDuration=10, + zdo_TC_Significant=0, ), responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), + c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) @@ -70,11 +80,15 @@ async def test_join_coordinator(device, make_application): # Handle us opening joins on the coordinator permit_join_coordinator = znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=60, partial=True + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.NWK, 0x0000), + command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, + TSN=7, + zdo_PermitDuration=60, + zdo_TC_Significant=0, ), responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), + c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) @@ -194,6 +208,24 @@ async def test_on_zdo_device_join_and_announce_fast(device, make_application, mo await asyncio.sleep(0.1) + znp_server.send( + c.ZDO.MsgCbIncoming.Callback( + Src=nwk, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Device_annce, + SecurityUse=0, + TSN=123, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Device_annce, + NWKAddr=nwk, + IEEEAddr=ieee, + Capability=c.zdo.MACCapabilities.AllocateShortAddrDuringAssocNeeded, + Status=t.ZDOStatus.SUCCESS, + ), + ) + ) + znp_server.send( c.ZDO.EndDeviceAnnceInd.Callback( Src=nwk, @@ -203,12 +235,14 @@ async def test_on_zdo_device_join_and_announce_fast(device, make_application, mo ) ) + await asyncio.sleep(0.1) + app.handle_join.assert_called_once_with(nwk=nwk, ieee=ieee, parent_nwk=None) # Everything is cleaned up assert not app._join_announce_tasks - await app.shutdown() + await app.pre_shutdown() @pytest.mark.parametrize("device", FORMED_DEVICES) @@ -216,6 +250,11 @@ async def test_on_zdo_device_join_and_announce_slow(device, make_application, mo app, znp_server = make_application(server_cls=device) await app.startup(auto_form=False) + znp_server.reply_to( + c.ZDO.ExtRouteDisc.Req(partial=True), + responses=[c.ZDO.ExtRouteDisc.Rsp(Status=t.Status.SUCCESS)], + ) + mocker.patch.object(app, "handle_join") mocker.patch("zigpy_znp.zigbee.application.DEVICE_JOIN_MAX_DELAY", new=0.1) @@ -235,295 +274,38 @@ async def test_on_zdo_device_join_and_announce_slow(device, make_application, mo app.handle_join.assert_called_once_with(nwk=nwk, ieee=ieee, parent_nwk=0x0001) znp_server.send( - c.ZDO.EndDeviceAnnceInd.Callback( + c.ZDO.MsgCbIncoming.Callback( Src=nwk, - NWK=nwk, - IEEE=ieee, - Capabilities=c.zdo.MACCapabilities.AllocateShortAddrDuringAssocNeeded, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Device_annce, + SecurityUse=0, + TSN=123, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Device_annce, + NWKAddr=nwk, + IEEEAddr=ieee, + Capability=c.zdo.MACCapabilities.AllocateShortAddrDuringAssocNeeded, + Status=t.ZDOStatus.SUCCESS, + ), ) ) - # The announcement will trigger another join indication - assert app.handle_join.call_count == 2 - - await app.shutdown() - - -@pytest.mark.parametrize("device", FORMED_DEVICES) -async def test_new_device_join_and_bind_complex(device, make_application, mocker): - app, znp_server = make_application(server_cls=device) - await app.startup(auto_form=False) - - nwk = 0x6A7C - ieee = t.EUI64.convert("00:17:88:01:08:64:6C:81") - - # Handle the startup permit join clear - znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, Duration=0, partial=True - ), - responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), - ], - override=True, - ) - - # Handle the permit join request sent by us - znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=60, partial=True - ), - responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), - ], - ) - - # Handle the ZDO broadcast sent by Zigpy - znp_server.reply_once_to( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, Duration=60, partial=True - ), - responses=[ - c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), - c.ZDO.TCDevInd.Callback(SrcNwk=nwk, SrcIEEE=ieee, ParentNwk=0x0000), - ], - ) - - # Handle the route-discovery-upon-join request - znp_server.reply_once_to( - request=c.ZDO.ExtRouteDisc.Req(Dst=nwk, partial=True), - responses=[ - c.ZDO.ExtRouteDisc.Rsp(Status=t.Status.SUCCESS), - ], - ) - - node_desc = c.zdo.NullableNodeDescriptor(2, 64, 128, 4107, 89, 63, 0, 63, 0) - - num_node_desc_reqs = 0 - - # Some devices join once, wait a bit, and re-join again - def poorly_timed_announce_replier(req): - nonlocal num_node_desc_reqs - num_node_desc_reqs += 1 - - if num_node_desc_reqs > 1: - return - - return c.ZDO.EndDeviceAnnceInd.Callback( + znp_server.send( + c.ZDO.EndDeviceAnnceInd.Callback( Src=nwk, NWK=nwk, IEEE=ieee, Capabilities=c.zdo.MACCapabilities.AllocateShortAddrDuringAssocNeeded, ) - - znp_server.reply_to( - request=c.ZDO.NodeDescReq.Req(DstAddr=nwk, NWKAddrOfInterest=nwk), - responses=[ - c.ZDO.NodeDescReq.Rsp(Status=t.Status.SUCCESS), - poorly_timed_announce_replier, - c.ZDO.NodeDescRsp.Callback( - Src=nwk, Status=t.ZDOStatus.SUCCESS, NWK=nwk, NodeDescriptor=node_desc - ), - ], - ) - - znp_server.reply_to( - request=c.ZDO.ActiveEpReq.Req(DstAddr=nwk, NWKAddrOfInterest=nwk), - responses=[ - c.ZDO.ActiveEpReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.ActiveEpRsp.Callback( - Src=nwk, Status=t.ZDOStatus.SUCCESS, NWK=nwk, ActiveEndpoints=[2, 1] - ), - ], - ) - - znp_server.reply_to( - request=c.ZDO.SimpleDescReq.Req(DstAddr=nwk, NWKAddrOfInterest=nwk, Endpoint=2), - responses=[ - c.ZDO.SimpleDescReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.SimpleDescRsp.Callback( - Src=nwk, - Status=t.ZDOStatus.SUCCESS, - NWK=nwk, - SimpleDescriptor=zdo_t.SizePrefixedSimpleDescriptor( - 2, 260, 263, 0, [0, 1, 3, 1030, 1024, 1026], [25] - ), - ), - ], - ) - - znp_server.reply_to( - request=c.ZDO.SimpleDescReq.Req(DstAddr=nwk, NWKAddrOfInterest=nwk, Endpoint=1), - responses=[ - c.ZDO.SimpleDescReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.SimpleDescRsp.Callback( - Src=nwk, - Status=t.ZDOStatus.SUCCESS, - NWK=nwk, - SimpleDescriptor=zdo_t.SizePrefixedSimpleDescriptor( - 1, 49246, 2128, 2, [0], [0, 3, 4, 6, 8, 768, 5] - ), - ), - ], - ) - - def data_req_callback(request): - if request.Data == bytes([0x00, request.TSN]) + b"\x00\x04\x00\x05\x00": - # Manufacturer + model - znp_server.send(c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)) - znp_server.send( - c.AF.DataConfirm.Callback( - Status=t.Status.SUCCESS, - Endpoint=request.SrcEndpoint, - TSN=request.TSN, - ) - ) - znp_server.send( - c.AF.IncomingMsg.Callback( - GroupId=0x0000, - ClusterId=request.ClusterId, - SrcAddr=nwk, - SrcEndpoint=request.DstEndpoint, - DstEndpoint=request.SrcEndpoint, - WasBroadcast=t.Bool.false, - LQI=156, - SecurityUse=t.Bool.false, - TimeStamp=2123652, - TSN=0, - Data=b"\x18" - + bytes([request.TSN]) - + b"\x01\x04\x00\x00\x42\x07\x50\x68\x69\x6C\x69\x70\x73\x05\x00" - + b"\x00\x42\x06\x53\x4D\x4C\x30\x30\x31", - MacSrcAddr=nwk, - MsgResultRadius=29, - ) - ) - elif request.Data == bytes([0x00, request.TSN]) + b"\x00\x04\x00": - # Manufacturer - znp_server.send(c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)) - znp_server.send( - c.AF.DataConfirm.Callback( - Status=t.Status.SUCCESS, - Endpoint=request.SrcEndpoint, - TSN=request.TSN, - ) - ) - znp_server.send( - c.AF.IncomingMsg.Callback( - GroupId=0x0000, - ClusterId=request.ClusterId, - SrcAddr=nwk, - SrcEndpoint=request.DstEndpoint, - DstEndpoint=request.SrcEndpoint, - WasBroadcast=t.Bool.false, - LQI=156, - SecurityUse=t.Bool.false, - TimeStamp=2123652, - TSN=0, - Data=b"\x18" - + bytes([request.TSN]) - + b"\x01\x04\x00\x00\x42\x07\x50\x68\x69\x6C\x69\x70\x73", - MacSrcAddr=nwk, - MsgResultRadius=29, - ) - ) - elif request.Data == bytes([0x00, request.TSN]) + b"\x00\x05\x00": - # Model - znp_server.send(c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)) - znp_server.send( - c.AF.DataConfirm.Callback( - Status=t.Status.SUCCESS, - Endpoint=request.SrcEndpoint, - TSN=request.TSN, - ) - ) - znp_server.send( - c.AF.IncomingMsg.Callback( - GroupId=0x0000, - ClusterId=request.ClusterId, - SrcAddr=nwk, - SrcEndpoint=request.DstEndpoint, - DstEndpoint=request.SrcEndpoint, - WasBroadcast=t.Bool.false, - LQI=156, - SecurityUse=t.Bool.false, - TimeStamp=2123652, - TSN=0, - Data=b"\x18" - + bytes([request.TSN]) - + b"\x01\x05\x00\x00\x42\x06\x53\x4D\x4C\x30\x30\x31", - MacSrcAddr=nwk, - MsgResultRadius=29, - ) - ) - - znp_server.callback_for_response( - c.AF.DataRequestExt.Req( - partial=True, - DstAddrModeAddress=t.AddrModeAddress(mode=t.AddrMode.NWK, address=nwk), - ), - data_req_callback, ) - device_future = asyncio.get_running_loop().create_future() - - class TestListener: - def device_initialized(self, device): - device_future.set_result(device) - - app.add_listener(TestListener()) - - await app.permit(time_s=60) # duration is sent as byte 0x3C in first ZDO broadcast - - # The device has finally joined and been initialized - device = await device_future - - assert not device.initializing - assert device.model == "SML001" - assert device.manufacturer == "Philips" - assert set(device.endpoints.keys()) == {0, 1, 2} - - assert set(device.endpoints[1].in_clusters.keys()) == {0} - assert set(device.endpoints[1].out_clusters.keys()) == {0, 3, 4, 6, 8, 768, 5} - - assert set(device.endpoints[2].in_clusters.keys()) == {0, 1, 3, 1030, 1024, 1026} - assert set(device.endpoints[2].out_clusters.keys()) == {25} - - # Once we've confirmed the device is good, start testing binds - def bind_req_callback(request): - assert request.Dst == nwk - assert request.Src == ieee - assert request.SrcEndpoint in device.endpoints - - cluster = request.ClusterId - ep = device.endpoints[request.SrcEndpoint] - assert cluster in ep.in_clusters or cluster in ep.out_clusters - - assert request.Address.ieee == app.ieee - assert request.Address.addrmode == 0x03 - - # Make sure the endpoint profiles match up - our_ep = request.Address.endpoint - assert app.get_device(nwk=0x0000).endpoints[our_ep].profile_id == ep.profile_id - - znp_server.send(c.ZDO.BindReq.Rsp(Status=t.Status.SUCCESS)) - znp_server.send(c.ZDO.BindRsp.Callback(Src=nwk, Status=t.ZDOStatus.SUCCESS)) - - znp_server.callback_for_response( - c.ZDO.BindReq.Req(Dst=nwk, Src=ieee, partial=True), bind_req_callback - ) - - for ep_id, endpoint in device.endpoints.items(): - if ep_id == 0: - continue + await asyncio.sleep(0.1) - for cluster in endpoint.in_clusters.values(): - await cluster.bind() + # The announcement will trigger another join indication + assert app.handle_join.call_count == 2 - await app.shutdown() + await app.pre_shutdown() @pytest.mark.parametrize("device", FORMED_DEVICES) @@ -543,13 +325,33 @@ async def test_unknown_device_discovery(device, make_application, mocker): # If the device changes its NWK but doesn't tell zigpy, it will be re-discovered did_ieee_addr_req1 = znp_server.reply_once_to( - request=c.ZDO.IEEEAddrReq.Req( - NWK=existing_nwk + 1, - RequestType=c.zdo.AddrRequestType.SINGLE, - StartIndex=0, + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.NWK, existing_nwk + 1), + command_id=zdo_t.ZDOCmd.IEEE_addr_req, + TSN=7, + zdo_NWKAddrOfInterest=existing_nwk + 1, + zdo_RequestType=c.zdo.AddrRequestType.SINGLE, + zdo_StartIndex=0, ), responses=[ - c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), + c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), + c.ZDO.MsgCbIncoming.Callback( + Src=existing_nwk + 1, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, + SecurityUse=0, + TSN=7, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, + Status=zdo_t.Status.SUCCESS, + IEEEAddr=existing_ieee, + NWKAddr=existing_nwk + 1, + NumAssocDev=0, + StartIndex=0, + NWKAddrAssocDevList=[], + ), + ), c.ZDO.IEEEAddrRsp.Callback( Status=t.ZDOStatus.SUCCESS, IEEE=existing_ieee, @@ -580,14 +382,35 @@ async def test_unknown_device_discovery(device, make_application, mocker): # If a completely unknown device joins the network, it will be treated as a new join new_nwk = 0x5678 new_ieee = t.EUI64(range(1, 9)) + did_ieee_addr_req2 = znp_server.reply_once_to( - request=c.ZDO.IEEEAddrReq.Req( - NWK=new_nwk, - RequestType=c.zdo.AddrRequestType.SINGLE, - StartIndex=0, + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.NWK, new_nwk), + command_id=zdo_t.ZDOCmd.IEEE_addr_req, + TSN=8, + zdo_NWKAddrOfInterest=new_nwk, + zdo_RequestType=c.zdo.AddrRequestType.SINGLE, + zdo_StartIndex=0, ), responses=[ - c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), + c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), + c.ZDO.MsgCbIncoming.Callback( + Src=new_nwk, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, + SecurityUse=0, + TSN=8, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, + Status=zdo_t.Status.SUCCESS, + IEEEAddr=new_ieee, + NWKAddr=new_nwk, + NumAssocDev=0, + StartIndex=0, + NWKAddrAssocDevList=[], + ), + ), c.ZDO.IEEEAddrRsp.Callback( Status=t.ZDOStatus.SUCCESS, IEEE=new_ieee, @@ -598,6 +421,7 @@ async def test_unknown_device_discovery(device, make_application, mocker): ), ], ) + new_dev = await app._get_or_discover_device(nwk=new_nwk) await did_ieee_addr_req2 assert app.handle_join.call_count == 2 @@ -614,13 +438,6 @@ async def test_unknown_device_discovery_failure(device, make_application, mocker app, znp_server = make_application(server_cls=device) await app.startup(auto_form=False) - znp_server.reply_once_to( - request=c.ZDO.IEEEAddrReq.Req(partial=True), - responses=[ - c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), - ], - ) - # Discovery will throw an exception when the device cannot be found with pytest.raises(KeyError): await app._get_or_discover_device(nwk=0x3456) diff --git a/tests/conftest.py b/tests/conftest.py index c7a6f9a5..7cc19129 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -273,6 +273,27 @@ def add_initialized_device(self, *args, **kwargs): return inner +def zdo_request_matcher( + dst_addr: t.AddrModeAddress, command_id: t.uint16_t, **kwargs +) -> c.AF.DataRequestExt.Req: + zdo_kwargs = {k: v for k, v in kwargs.items() if k.startswith("zdo_")} + + kwargs = {k: v for k, v in kwargs.items() if not k.startswith("zdo_")} + kwargs.setdefault("DstEndpoint", 0x00) + kwargs.setdefault("DstPanId", 0x0000) + kwargs.setdefault("SrcEndpoint", 0x00) + kwargs.setdefault("Radius", None) + kwargs.setdefault("Options", None) + + return c.AF.DataRequestExt.Req( + DstAddrModeAddress=dst_addr, + ClusterId=command_id, + Data=bytes([kwargs["TSN"]]) + serialize_zdo_command(command_id, **zdo_kwargs), + **kwargs, + partial=True, + ) + + class BaseServerZNP(ZNP): align_structs = False version = None @@ -959,7 +980,7 @@ def on_zdo_mgmt_permit_joining_req(self, req, PermitDuration, TC_Significant): return if PermitDuration != 0: - result = [c.ZDO.PermitJoinInd.Callback(Duration=req.Duration)] + result + result = [c.ZDO.PermitJoinInd.Callback(Duration=PermitDuration)] + result return result + [c.ZDO.PermitJoinInd.Callback(Duration=0)] diff --git a/tests/nvram/CC2652R-ZStack4.formed.json b/tests/nvram/CC2652R-ZStack4.formed.json index 3f2d9bbe..2c2101e6 100644 --- a/tests/nvram/CC2652R-ZStack4.formed.json +++ b/tests/nvram/CC2652R-ZStack4.formed.json @@ -1,6 +1,7 @@ { "LEGACY": { "HAS_CONFIGURED_ZSTACK3": "55", + "ZIGPY_ZNP_MIGRATION_ID": "01", "EXTADDR": "a8ef171e004b1200", "STARTUP_OPTION": "00", "START_DELAY": "0a", diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index aebf76fb..312e1012 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -21,7 +21,7 @@ import zigpy.zdo.types as zdo_t import zigpy.application from zigpy.zcl import clusters -from zigpy.types import ExtendedPanId +from zigpy.types import ExtendedPanId, deserialize as list_deserialize from zigpy.exceptions import DeliveryError import zigpy_znp.const as const @@ -95,6 +95,17 @@ def model(self): return f"{model}, Z-Stack {version} (build {self.application._zstack_build_id})" +class InitializedDevice(zigpy.device.Device): + """ + Device that does not need to be initialized, since we just need it for addressing + purposes. + """ + + @property + def is_initialized(self): + return True + + class ControllerApplication(zigpy.application.ControllerApplication): SCHEMA = conf.CONFIG_SCHEMA SCHEMA_DEVICE = conf.SCHEMA_DEVICE @@ -116,6 +127,7 @@ def __init__(self, config: conf.ConfigType): self._currently_waiting_requests = 0 self._join_announce_tasks = {} + self._temp_devices = [] ################################################################## # Implementation of the core zigpy ControllerApplication methods # @@ -536,8 +548,8 @@ async def permit(self, time_s=60, node=None): # through the coordinator itself. # # Fixed in https://github.com/Koenkk/Z-Stack-firmware/commit/efac5ee46b9b437 - if time_s == 0 or self._zstack_build_id < 20210708 or node in (None, self.ieee): - response = await self.zigpy_device.zdo.Mgmt_Permit_Joining_req(time_s, 1) + if time_s == 0 or self._zstack_build_id < 20210708 or node == self.ieee: + response = await self.zigpy_device.zdo.Mgmt_Permit_Joining_req(time_s, 0) if response[0] != t.Status.SUCCESS: raise RuntimeError(f"Failed to permit joins on coordinator: {response}") @@ -680,27 +692,40 @@ async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: Global callback for all ZDO messages. """ - device = await self._get_or_discover_device(nwk=msg.Src) + message = t.uint8_t(msg.TSN).serialize() + msg.Data + hdr, data = zdo_t.ZDOHeader.deserialize(msg.ClusterId, message) + names, types = zdo_t.CLUSTERS[msg.ClusterId] + args, data = list_deserialize(data, types) + # kwargs = dict(zip(names, args)) + + if msg.ClusterId == zdo_t.ZDOCmd.Device_annce: + self.on_zdo_device_announce(*args) + elif msg.ClusterId == zdo_t.ZDOCmd.IEEE_addr_rsp: + device = next((d for d in self._temp_devices if d.nwk == msg.Src), None) + else: + device = await self._get_or_discover_device(nwk=msg.Src) if device is None: LOGGER.warning("Received a ZDO message from an unknown device: %s", msg.Src) return - message = t.uint8_t(msg.TSN).serialize() + msg.Data - - self.handle_message( - sender=device, - profile=zigpy.profiles.zha.PROFILE_ID, - cluster=msg.ClusterId, - src_ep=ZDO_ENDPOINT, - dst_ep=ZDO_ENDPOINT, - message=message, - ) - - hdr, args = device.zdo.deserialize(msg.ClusterId, message) - - if msg.ClusterId == zdo_t.ZDOCmd.Device_annce: - self.on_zdo_device_announce(*args) + if isinstance(device, InitializedDevice): + device.handle_message( + profile=ZDO_PROFILE, + cluster=msg.ClusterId, + src_ep=ZDO_ENDPOINT, + dst_ep=ZDO_ENDPOINT, + message=message, + ) + else: + self.handle_message( + sender=device, + profile=ZDO_PROFILE, + cluster=msg.ClusterId, + src_ep=ZDO_ENDPOINT, + dst_ep=ZDO_ENDPOINT, + message=message, + ) def on_zdo_permit_join_message(self, msg: c.ZDO.PermitJoinInd.Callback) -> None: """ @@ -896,15 +921,22 @@ async def _get_or_discover_device(self, nwk: t.NWK) -> zigpy.device.Device: LOGGER.debug("Device with NWK 0x%04X not in database", nwk) try: - # XXX: Multiple responses may arrive but we only use the first one async with async_timeout.timeout(IEEE_ADDR_DISCOVERY_TIMEOUT): - _, ieee, _, _, _, _ = await self.zigpy_device.zdo.IEEE_addr_req( - *{ - "NWKAddrOfInterest": nwk, - "RequestType": c.zdo.AddrRequestType.SINGLE, - "StartIndex": 0, - }.values() - ) + temp_device = InitializedDevice(application=self, ieee=None, nwk=nwk) + self._temp_devices.append(temp_device) + + try: + status, ieee, *_ = await temp_device.zdo.IEEE_addr_req( + *{ + "NWKAddrOfInterest": nwk, + "RequestType": c.zdo.AddrRequestType.SINGLE, + "StartIndex": 0, + }.values() + ) + finally: + self._temp_devices.remove(temp_device) + + assert status == zdo_t.Status.SUCCESS except asyncio.TimeoutError: raise KeyError(f"Unknown device: 0x{nwk:04X}") From 201f4ea164125db2827932975747f9ab7d520292 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Mon, 10 Jan 2022 14:55:51 -0500 Subject: [PATCH 08/19] Fix final unit tests --- tests/application/test_requests.py | 101 +++++++--------------- tests/application/test_zigpy_callbacks.py | 26 +++++- tests/tools/test_energy_scan.py | 64 ++++++++++---- zigpy_znp/zigbee/application.py | 10 +-- 4 files changed, 100 insertions(+), 101 deletions(-) diff --git a/tests/application/test_requests.py b/tests/application/test_requests.py index 979c629a..08fa9a3b 100644 --- a/tests/application/test_requests.py +++ b/tests/application/test_requests.py @@ -4,7 +4,7 @@ import zigpy.zdo import zigpy.endpoint import zigpy.profiles -from zigpy.zdo.types import ZDOCmd +import zigpy.zdo.types as zdo_t from zigpy.exceptions import DeliveryError import zigpy_znp.types as t @@ -12,7 +12,13 @@ import zigpy_znp.commands as c from zigpy_znp.exceptions import InvalidCommandResponse -from ..conftest import FORMED_DEVICES, CoroutineMock, FormedLaunchpadCC26X2R1 +from ..conftest import ( + FORMED_DEVICES, + CoroutineMock, + FormedLaunchpadCC26X2R1, + zdo_request_matcher, + serialize_zdo_command, +) @pytest.mark.parametrize("device", FORMED_DEVICES) @@ -177,52 +183,6 @@ async def test_request_addr_mode(device, addr, make_application, mocker): await app.shutdown() -@pytest.mark.parametrize("device", FORMED_DEVICES) -@pytest.mark.parametrize("status", [t.ZDOStatus.SUCCESS, t.ZDOStatus.TIMEOUT, None]) -async def test_remove(device, make_application, status, mocker): - app, znp_server = make_application(server_cls=device) - app._config[conf.CONF_ZNP_CONFIG][conf.CONF_ARSP_TIMEOUT] = 0.1 - - # Only zigpy>=0.29.0 has this method - if hasattr(app, "_remove_device"): - mocker.spy(app, "_remove_device") - - await app.startup(auto_form=False) - device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xAABB) - - responses = [c.ZDO.MgmtLeaveReq.Rsp(Status=t.Status.SUCCESS)] - - if status is not None: - responses.append(c.ZDO.MgmtLeaveRsp.Callback(Src=0x0000, Status=status)) - - # Normal ZDO leave must fail - normal_remove_req = znp_server.reply_once_to( - request=c.ZDO.MgmtLeaveReq.Req( - DstAddr=device.nwk, IEEE=device.ieee, partial=True - ), - responses=[ - c.ZDO.MgmtLeaveReq.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MgmtLeaveRsp.Callback(Src=device.nwk, Status=t.ZDOStatus.TIMEOUT), - ], - ) - - # Make sure the device exists - assert app.get_device(nwk=device.nwk) is device - - await app.remove(device.ieee) - await normal_remove_req - - if hasattr(app, "_remove_device"): - # Make sure the device is going to be removed - assert app._remove_device.call_count == 1 - else: - # Make sure the device is gone - with pytest.raises(KeyError): - app.get_device(ieee=device.ieee) - - await app.shutdown() - - @pytest.mark.parametrize("device", FORMED_DEVICES) async def test_mrequest(device, make_application, mocker): app, znp_server = make_application(server_cls=device) @@ -274,25 +234,6 @@ async def test_mrequest_doesnt_block(device, make_application, event_loop): await app.shutdown() -@pytest.mark.parametrize("device", [FormedLaunchpadCC26X2R1]) -async def test_unimplemented_zdo_converter(device, make_application, mocker): - app, znp_server = make_application(server_cls=device) - await app.startup() - - with pytest.raises(RuntimeError): - await zigpy.zdo.broadcast( - app, - ZDOCmd.Remove_node_cache_req, - 0x0000, - 0x00, - t.NWK(0x1234), - t.EUI64.convert("11:22:33:44:55:66:77:88"), - broadcast_address=0xFFFC, - ) - - await app.shutdown() - - @pytest.mark.parametrize("device", [FormedLaunchpadCC26X2R1]) async def test_broadcast(device, make_application, mocker): app, znp_server = make_application(server_cls=device) @@ -595,28 +536,46 @@ def set_route_discovered(req): return c.ZDO.ExtRouteDisc.Rsp(Status=t.Status.SUCCESS) znp_server.reply_to( - c.ZDO.ExtRouteChk.Req(Dst=device.nwk, partial=True), + request=c.ZDO.ExtRouteChk.Req(Dst=device.nwk, partial=True), responses=[route_replier], override=True, ) was_route_discovered = znp_server.reply_once_to( - c.ZDO.ExtRouteDisc.Req( + request=c.ZDO.ExtRouteDisc.Req( Dst=device.nwk, Options=c.zdo.RouteDiscoveryOptions.UNICAST, partial=True ), responses=[set_route_discovered], ) zdo_req = znp_server.reply_once_to( - c.ZDO.ActiveEpReq.Req(DstAddr=device.nwk, NWKAddrOfInterest=device.nwk), + request=zdo_request_matcher( + dst_addr=t.AddrModeAddress(t.AddrMode.NWK, device.nwk), + command_id=zdo_t.ZDOCmd.Active_EP_req, + TSN=7, + zdo_NWKAddrOfInterest=device.nwk, + ), responses=[ - c.ZDO.ActiveEpReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.ActiveEpRsp.Callback( Src=device.nwk, Status=t.ZDOStatus.SUCCESS, NWK=device.nwk, ActiveEndpoints=[], ), + c.ZDO.MsgCbIncoming.Callback( + Src=device.nwk, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Active_EP_rsp, + SecurityUse=0, + TSN=7, + MacDst=device.nwk, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Active_EP_rsp, + Status=t.ZDOStatus.SUCCESS, + NWKAddrOfInterest=device.nwk, + ActiveEPList=[], + ), + ), ], ) diff --git a/tests/application/test_zigpy_callbacks.py b/tests/application/test_zigpy_callbacks.py index a8322aea..21d2d250 100644 --- a/tests/application/test_zigpy_callbacks.py +++ b/tests/application/test_zigpy_callbacks.py @@ -2,12 +2,12 @@ import logging import pytest -from zigpy.zdo.types import ZDOCmd +import zigpy.zdo.types as zdo_t import zigpy_znp.types as t import zigpy_znp.commands as c -from ..conftest import FORMED_DEVICES, CoroutineMock +from ..conftest import FORMED_DEVICES, CoroutineMock, serialize_zdo_command def awaitable_mock(*, return_value=None, side_effect=None): @@ -74,6 +74,24 @@ async def test_on_zdo_device_announce_nwk_change(device, make_application, mocke new_nwk = device.nwk + 1 # Assume its NWK changed and we're just finding out + znp_server.send( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0001, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Device_annce, + SecurityUse=0, + TSN=123, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Device_annce, + NWKAddr=new_nwk, + IEEEAddr=device.ieee, + Capability=c.zdo.MACCapabilities.Router, + Status=t.ZDOStatus.SUCCESS, + ), + ) + ) + znp_server.send( c.ZDO.EndDeviceAnnceInd.Callback( Src=0x0001, @@ -83,11 +101,13 @@ async def test_on_zdo_device_announce_nwk_change(device, make_application, mocke ) ) + await asyncio.sleep(0.1) + app.handle_join.assert_called_once_with( nwk=new_nwk, ieee=device.ieee, parent_nwk=None ) assert app.handle_message.call_count == 1 - assert app.handle_message.mock_calls[0][2]["cluster"] == ZDOCmd.Device_annce + assert app.handle_message.mock_calls[0][2]["cluster"] == zdo_t.ZDOCmd.Device_annce # The device's NWK updated assert device.nwk == new_nwk diff --git a/tests/tools/test_energy_scan.py b/tests/tools/test_energy_scan.py index 69be7549..4a284593 100644 --- a/tests/tools/test_energy_scan.py +++ b/tests/tools/test_energy_scan.py @@ -1,12 +1,18 @@ import asyncio import pytest +import zigpy.zdo.types as zdo_t import zigpy_znp.types as t import zigpy_znp.commands as c from zigpy_znp.tools.energy_scan import main as energy_scan -from ..conftest import EMPTY_DEVICES, FORMED_DEVICES +from ..conftest import ( + EMPTY_DEVICES, + FORMED_DEVICES, + serialize_zdo_command, + deserialize_zdo_command, +) @pytest.mark.parametrize("device", EMPTY_DEVICES) @@ -23,32 +29,52 @@ async def test_energy_scan_formed(device, make_znp_server, capsys): def fake_scanner(request): async def response(request): - znp_server.send(c.ZDO.MgmtNWKUpdateReq.Rsp(Status=t.Status.SUCCESS)) - - delay = 2 ** request.ScanDuration - num_channels = len(list(request.Channels)) - - for i in range(request.ScanCount): - await asyncio.sleep(delay / 100) - - znp_server.send( - c.ZDO.MgmtNWKUpdateNotify.Callback( - Src=0x0000, + znp_server.send(c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)) + + params = deserialize_zdo_command(request.ClusterId, request.Data[1:]) + channels = params["NwkUpdate"].ScanChannels + num_channels = len(list(channels)) + + await asyncio.sleep(0.1) + + znp_server.send( + c.ZDO.MsgCbIncoming.Callback( + Src=0x0000, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Mgmt_NWK_Update_rsp, + SecurityUse=0, + TSN=request.TSN, + MacDst=0x0000, + Data=serialize_zdo_command( + command_id=zdo_t.ZDOCmd.Mgmt_NWK_Update_rsp, Status=t.ZDOStatus.SUCCESS, - ScannedChannels=request.Channels, + ScannedChannels=channels, TotalTransmissions=998, TransmissionFailures=2, - EnergyValues=list(range(num_channels)), - ) + EnergyValues=list(range(11, 26 + 1))[:num_channels], + ), + ) + ) + + znp_server.send( + c.ZDO.MgmtNWKUpdateNotify.Callback( + Src=0x0000, + Status=t.ZDOStatus.SUCCESS, + ScannedChannels=channels, + TotalTransmissions=998, + TransmissionFailures=2, + EnergyValues=list(range(11, 26 + 1))[:num_channels], ) + ) asyncio.create_task(response(request)) znp_server.callback_for_response( - c.ZDO.MgmtNWKUpdateReq.Req( - Dst=0x0000, - DstAddrMode=t.AddrMode.NWK, - NwkManagerAddr=0x0000, + c.AF.DataRequestExt.Req( + DstAddrModeAddress=t.AddrModeAddress(mode=t.AddrMode.NWK, address=0x0000), + DstEndpoint=0, + SrcEndpoint=0, + ClusterId=zdo_t.ZDOCmd.Mgmt_NWK_Update_req, partial=True, ), fake_scanner, diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 312e1012..f9e97466 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -528,13 +528,6 @@ async def mrequest( data=data, ) - async def force_remove(self, device: zigpy.device.Device) -> None: - """ - Attempts to forcibly remove a device from the network. - """ - - LOGGER.warning("Z-Stack does not support force remove") - async def permit(self, time_s=60, node=None): """ Permit joining the network via a specific node or via all router nodes. @@ -696,10 +689,11 @@ async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: hdr, data = zdo_t.ZDOHeader.deserialize(msg.ClusterId, message) names, types = zdo_t.CLUSTERS[msg.ClusterId] args, data = list_deserialize(data, types) - # kwargs = dict(zip(names, args)) + kwargs = dict(zip(names, args)) if msg.ClusterId == zdo_t.ZDOCmd.Device_annce: self.on_zdo_device_announce(*args) + device = self.get_device(ieee=kwargs["IEEEAddr"]) elif msg.ClusterId == zdo_t.ZDOCmd.IEEE_addr_rsp: device = next((d for d in self._temp_devices if d.nwk == msg.Src), None) else: From 13781eadcc58db62f90293f8918ac179e41ac786 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Mon, 10 Jan 2022 16:23:16 -0500 Subject: [PATCH 09/19] Always send `ZDO.MgmtPermitJoinReq.Req` when permitting joins --- tests/application/test_joining.py | 52 +++++++++++++++--------------- tests/application/test_requests.py | 8 ++--- tests/conftest.py | 14 ++++++++ zigpy_znp/zigbee/application.py | 30 +++++++++++++++-- 4 files changed, 71 insertions(+), 33 deletions(-) diff --git a/tests/application/test_joining.py b/tests/application/test_joining.py index 3f0fdabe..982b4a2b 100644 --- a/tests/application/test_joining.py +++ b/tests/application/test_joining.py @@ -29,32 +29,37 @@ async def test_permit_join(device, fixed_joining_bug, mocker, make_application): app, znp_server = make_application(server_cls=device) - # Handle us opening joins on the coordinator permit_join_coordinator = znp_server.reply_once_to( - request=zdo_request_matcher( - dst_addr=t.AddrModeAddress(t.AddrMode.NWK, 0x0000), - command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, - TSN=7, - zdo_PermitDuration=10, - zdo_TC_Significant=0, + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=10, partial=True ), responses=[ - c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), + c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) # Handle the ZDO broadcast sent by Zigpy - permit_join_broadcast = znp_server.reply_once_to( + permit_join_broadcast_raw = znp_server.reply_once_to( request=zdo_request_matcher( dst_addr=t.AddrModeAddress(t.AddrMode.Broadcast, 0xFFFC), command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, - TSN=8 if not fixed_joining_bug else 7, + TSN=6, zdo_PermitDuration=10, zdo_TC_Significant=0, ), responses=[ c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), + ], + ) + + # And the duplicate one using the MT command + permit_join_broadcast = znp_server.reply_once_to( + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, Duration=10, partial=True + ), + responses=[ + c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) @@ -62,14 +67,13 @@ async def test_permit_join(device, fixed_joining_bug, mocker, make_application): await app.startup(auto_form=False) await app.permit(time_s=10) - if fixed_joining_bug: - await permit_join_broadcast + await permit_join_broadcast + await permit_join_broadcast_raw - # Joins should not have been opened on the coordinator + if fixed_joining_bug: assert not permit_join_coordinator.done() else: - await permit_join_coordinator - await permit_join_broadcast + assert permit_join_coordinator.done() await app.shutdown() @@ -80,15 +84,11 @@ async def test_join_coordinator(device, make_application): # Handle us opening joins on the coordinator permit_join_coordinator = znp_server.reply_once_to( - request=zdo_request_matcher( - dst_addr=t.AddrModeAddress(t.AddrMode.NWK, 0x0000), - command_id=zdo_t.ZDOCmd.Mgmt_Permit_Joining_req, - TSN=7, - zdo_PermitDuration=60, - zdo_TC_Significant=0, + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.NWK, Dst=0x0000, Duration=60, partial=True ), responses=[ - c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), + c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), ], ) @@ -328,7 +328,7 @@ async def test_unknown_device_discovery(device, make_application, mocker): request=zdo_request_matcher( dst_addr=t.AddrModeAddress(t.AddrMode.NWK, existing_nwk + 1), command_id=zdo_t.ZDOCmd.IEEE_addr_req, - TSN=7, + TSN=6, zdo_NWKAddrOfInterest=existing_nwk + 1, zdo_RequestType=c.zdo.AddrRequestType.SINGLE, zdo_StartIndex=0, @@ -340,7 +340,7 @@ async def test_unknown_device_discovery(device, make_application, mocker): IsBroadcast=t.Bool.false, ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, SecurityUse=0, - TSN=7, + TSN=6, MacDst=0x0000, Data=serialize_zdo_command( command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, @@ -387,7 +387,7 @@ async def test_unknown_device_discovery(device, make_application, mocker): request=zdo_request_matcher( dst_addr=t.AddrModeAddress(t.AddrMode.NWK, new_nwk), command_id=zdo_t.ZDOCmd.IEEE_addr_req, - TSN=8, + TSN=7, zdo_NWKAddrOfInterest=new_nwk, zdo_RequestType=c.zdo.AddrRequestType.SINGLE, zdo_StartIndex=0, @@ -399,7 +399,7 @@ async def test_unknown_device_discovery(device, make_application, mocker): IsBroadcast=t.Bool.false, ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, SecurityUse=0, - TSN=8, + TSN=7, MacDst=0x0000, Data=serialize_zdo_command( command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, diff --git a/tests/application/test_requests.py b/tests/application/test_requests.py index 08fa9a3b..7d024f15 100644 --- a/tests/application/test_requests.py +++ b/tests/application/test_requests.py @@ -48,7 +48,7 @@ async def test_zigpy_request(device, make_application): app, znp_server = make_application(device) await app.startup(auto_form=False) - TSN = 7 + TSN = 6 device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xAABB) @@ -108,7 +108,7 @@ async def test_zigpy_request_failure(device, make_application, mocker): app, znp_server = make_application(device) await app.startup(auto_form=False) - TSN = 7 + TSN = 6 device = app.add_initialized_device(ieee=t.EUI64(range(8)), nwk=0xAABB) @@ -552,7 +552,7 @@ def set_route_discovered(req): request=zdo_request_matcher( dst_addr=t.AddrModeAddress(t.AddrMode.NWK, device.nwk), command_id=zdo_t.ZDOCmd.Active_EP_req, - TSN=7, + TSN=6, zdo_NWKAddrOfInterest=device.nwk, ), responses=[ @@ -567,7 +567,7 @@ def set_route_discovered(req): IsBroadcast=t.Bool.false, ClusterId=zdo_t.ZDOCmd.Active_EP_rsp, SecurityUse=0, - TSN=7, + TSN=6, MacDst=device.nwk, Data=serialize_zdo_command( command_id=zdo_t.ZDOCmd.Active_EP_rsp, diff --git a/tests/conftest.py b/tests/conftest.py index 7cc19129..ab8080a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -567,6 +567,20 @@ def _default_nib(self): nwkUpdateId=0, ) + @reply_to( + c.ZDO.MgmtPermitJoinReq.Req(AddrMode=t.AddrMode.NWK, Dst=0x0000, partial=True) + ) + @reply_to( + c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.Broadcast, Dst=0xFFFC, partial=True + ) + ) + def permit_join(self, request): + return [ + c.ZDO.MgmtPermitJoinReq.Rsp(Status=t.Status.SUCCESS), + c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, Status=t.ZDOStatus.SUCCESS), + ] + @reply_to(c.AF.DataRequestExt.Req(partial=True, DstEndpoint=0)) def on_zdo_request(self, req): kwargs = deserialize_zdo_command(req.ClusterId, req.Data[1:]) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index f9e97466..43ff68e1 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -528,7 +528,7 @@ async def mrequest( data=data, ) - async def permit(self, time_s=60, node=None): + async def permit(self, time_s: int = 60, node: t.EUI64 = None): """ Permit joining the network via a specific node or via all router nodes. """ @@ -542,9 +542,18 @@ async def permit(self, time_s=60, node=None): # # Fixed in https://github.com/Koenkk/Z-Stack-firmware/commit/efac5ee46b9b437 if time_s == 0 or self._zstack_build_id < 20210708 or node == self.ieee: - response = await self.zigpy_device.zdo.Mgmt_Permit_Joining_req(time_s, 0) + response = await self._znp.request_callback_rsp( + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.NWK, + Dst=0x0000, + Duration=time_s, + TCSignificance=1, + ), + RspStatus=t.Status.SUCCESS, + callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), + ) - if response[0] != t.Status.SUCCESS: + if response.Status != t.Status.SUCCESS: raise RuntimeError(f"Failed to permit joins on coordinator: {response}") await super().permit(time_s=time_s, node=node) @@ -660,6 +669,7 @@ def _bind_callbacks(self) -> None: c.ZDO.NodeDescRsp, c.ZDO.SimpleDescRsp, c.ZDO.ActiveEpRsp, + c.ZDO.MgmtLqiRsp, ]: self._znp.callback_for_response( ignored_msg.Callback(partial=True), @@ -1283,6 +1293,20 @@ async def _send_request_raw( Data=data, ) + # XXX: Joins *must* be sent via a ZDO command. Otherwise, Z-Stack will not + # actually permit the coordinator to send the network key. + if dst_ep == ZDO_ENDPOINT and cluster == zdo_t.ZDOCmd.Mgmt_Permit_Joining_req: + await self._znp.request_callback_rsp( + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=dst_addr.mode, + Dst=dst_addr.address, + Duration=data[1], + TCSignificance=data[2], + ), + RspStatus=t.Status.SUCCESS, + callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), + ) + if dst_addr.mode == t.AddrMode.Broadcast or dst_ep == ZDO_ENDPOINT: # Broadcasts and ZDO requests will not receive a confirmation response = await self._znp.request( From a1edd166806e95cc60c096cc477911d207a5b1d1 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 11 Jan 2022 11:12:16 -0500 Subject: [PATCH 10/19] Directly use Z-Stack `c.ZDO.IEEEAddrReq.Req` again, for now Once the required changes to zigpy are made, `_get_or_discover_device` can be shared across other libraries. --- tests/application/test_joining.py | 67 +++++++------------------ tests/nvram/CC2652R-ZStack4.formed.json | 1 - zigpy_znp/zigbee/application.py | 65 +++++++----------------- 3 files changed, 36 insertions(+), 97 deletions(-) diff --git a/tests/application/test_joining.py b/tests/application/test_joining.py index 982b4a2b..db5aadc4 100644 --- a/tests/application/test_joining.py +++ b/tests/application/test_joining.py @@ -325,33 +325,13 @@ async def test_unknown_device_discovery(device, make_application, mocker): # If the device changes its NWK but doesn't tell zigpy, it will be re-discovered did_ieee_addr_req1 = znp_server.reply_once_to( - request=zdo_request_matcher( - dst_addr=t.AddrModeAddress(t.AddrMode.NWK, existing_nwk + 1), - command_id=zdo_t.ZDOCmd.IEEE_addr_req, - TSN=6, - zdo_NWKAddrOfInterest=existing_nwk + 1, - zdo_RequestType=c.zdo.AddrRequestType.SINGLE, - zdo_StartIndex=0, + request=c.ZDO.IEEEAddrReq.Req( + NWK=existing_nwk + 1, + RequestType=c.zdo.AddrRequestType.SINGLE, + StartIndex=0, ), responses=[ - c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MsgCbIncoming.Callback( - Src=existing_nwk + 1, - IsBroadcast=t.Bool.false, - ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, - SecurityUse=0, - TSN=6, - MacDst=0x0000, - Data=serialize_zdo_command( - command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, - Status=zdo_t.Status.SUCCESS, - IEEEAddr=existing_ieee, - NWKAddr=existing_nwk + 1, - NumAssocDev=0, - StartIndex=0, - NWKAddrAssocDevList=[], - ), - ), + c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.IEEEAddrRsp.Callback( Status=t.ZDOStatus.SUCCESS, IEEE=existing_ieee, @@ -384,33 +364,13 @@ async def test_unknown_device_discovery(device, make_application, mocker): new_ieee = t.EUI64(range(1, 9)) did_ieee_addr_req2 = znp_server.reply_once_to( - request=zdo_request_matcher( - dst_addr=t.AddrModeAddress(t.AddrMode.NWK, new_nwk), - command_id=zdo_t.ZDOCmd.IEEE_addr_req, - TSN=7, - zdo_NWKAddrOfInterest=new_nwk, - zdo_RequestType=c.zdo.AddrRequestType.SINGLE, - zdo_StartIndex=0, + request=c.ZDO.IEEEAddrReq.Req( + NWK=new_nwk, + RequestType=c.zdo.AddrRequestType.SINGLE, + StartIndex=0, ), responses=[ - c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS), - c.ZDO.MsgCbIncoming.Callback( - Src=new_nwk, - IsBroadcast=t.Bool.false, - ClusterId=zdo_t.ZDOCmd.IEEE_addr_rsp, - SecurityUse=0, - TSN=7, - MacDst=0x0000, - Data=serialize_zdo_command( - command_id=zdo_t.ZDOCmd.IEEE_addr_rsp, - Status=zdo_t.Status.SUCCESS, - IEEEAddr=new_ieee, - NWKAddr=new_nwk, - NumAssocDev=0, - StartIndex=0, - NWKAddrAssocDevList=[], - ), - ), + c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), c.ZDO.IEEEAddrRsp.Callback( Status=t.ZDOStatus.SUCCESS, IEEE=new_ieee, @@ -438,6 +398,13 @@ async def test_unknown_device_discovery_failure(device, make_application, mocker app, znp_server = make_application(server_cls=device) await app.startup(auto_form=False) + znp_server.reply_once_to( + request=c.ZDO.IEEEAddrReq.Req(partial=True), + responses=[ + c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS), + ], + ) + # Discovery will throw an exception when the device cannot be found with pytest.raises(KeyError): await app._get_or_discover_device(nwk=0x3456) diff --git a/tests/nvram/CC2652R-ZStack4.formed.json b/tests/nvram/CC2652R-ZStack4.formed.json index 2c2101e6..3f2d9bbe 100644 --- a/tests/nvram/CC2652R-ZStack4.formed.json +++ b/tests/nvram/CC2652R-ZStack4.formed.json @@ -1,7 +1,6 @@ { "LEGACY": { "HAS_CONFIGURED_ZSTACK3": "55", - "ZIGPY_ZNP_MIGRATION_ID": "01", "EXTADDR": "a8ef171e004b1200", "STARTUP_OPTION": "00", "START_DELAY": "0a", diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 43ff68e1..cd611533 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -95,17 +95,6 @@ def model(self): return f"{model}, Z-Stack {version} (build {self.application._zstack_build_id})" -class InitializedDevice(zigpy.device.Device): - """ - Device that does not need to be initialized, since we just need it for addressing - purposes. - """ - - @property - def is_initialized(self): - return True - - class ControllerApplication(zigpy.application.ControllerApplication): SCHEMA = conf.CONFIG_SCHEMA SCHEMA_DEVICE = conf.SCHEMA_DEVICE @@ -127,7 +116,6 @@ def __init__(self, config: conf.ConfigType): self._currently_waiting_requests = 0 self._join_announce_tasks = {} - self._temp_devices = [] ################################################################## # Implementation of the core zigpy ControllerApplication methods # @@ -704,8 +692,6 @@ async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: if msg.ClusterId == zdo_t.ZDOCmd.Device_annce: self.on_zdo_device_announce(*args) device = self.get_device(ieee=kwargs["IEEEAddr"]) - elif msg.ClusterId == zdo_t.ZDOCmd.IEEE_addr_rsp: - device = next((d for d in self._temp_devices if d.nwk == msg.Src), None) else: device = await self._get_or_discover_device(nwk=msg.Src) @@ -713,23 +699,14 @@ async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: LOGGER.warning("Received a ZDO message from an unknown device: %s", msg.Src) return - if isinstance(device, InitializedDevice): - device.handle_message( - profile=ZDO_PROFILE, - cluster=msg.ClusterId, - src_ep=ZDO_ENDPOINT, - dst_ep=ZDO_ENDPOINT, - message=message, - ) - else: - self.handle_message( - sender=device, - profile=ZDO_PROFILE, - cluster=msg.ClusterId, - src_ep=ZDO_ENDPOINT, - dst_ep=ZDO_ENDPOINT, - message=message, - ) + self.handle_message( + sender=device, + profile=ZDO_PROFILE, + cluster=msg.ClusterId, + src_ep=ZDO_ENDPOINT, + dst_ep=ZDO_ENDPOINT, + message=message, + ) def on_zdo_permit_join_message(self, msg: c.ZDO.PermitJoinInd.Callback) -> None: """ @@ -926,23 +903,19 @@ async def _get_or_discover_device(self, nwk: t.NWK) -> zigpy.device.Device: try: async with async_timeout.timeout(IEEE_ADDR_DISCOVERY_TIMEOUT): - temp_device = InitializedDevice(application=self, ieee=None, nwk=nwk) - self._temp_devices.append(temp_device) - - try: - status, ieee, *_ = await temp_device.zdo.IEEE_addr_req( - *{ - "NWKAddrOfInterest": nwk, - "RequestType": c.zdo.AddrRequestType.SINGLE, - "StartIndex": 0, - }.values() - ) - finally: - self._temp_devices.remove(temp_device) - - assert status == zdo_t.Status.SUCCESS + ieee_addr_rsp = await self._znp.request_callback_rsp( + request=c.ZDO.IEEEAddrReq.Req( + NWK=nwk, + RequestType=c.zdo.AddrRequestType.SINGLE, + StartIndex=0, + ), + RspStatus=t.Status.SUCCESS, + callback=c.ZDO.IEEEAddrRsp.Callback(partial=True, NWK=nwk), + ) except asyncio.TimeoutError: raise KeyError(f"Unknown device: 0x{nwk:04X}") + else: + ieee = ieee_addr_rsp.IEEE try: device = self.get_device(ieee=ieee) From 4311c2ab03a1c4cf9537af3915db17ed72fe8920 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 11 Jan 2022 12:41:32 -0500 Subject: [PATCH 11/19] Improve test coverage --- tests/application/test_requests.py | 35 +++++++++++++++++++++++++++++- zigpy_znp/zigbee/application.py | 12 +++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/tests/application/test_requests.py b/tests/application/test_requests.py index 7d024f15..986d1b22 100644 --- a/tests/application/test_requests.py +++ b/tests/application/test_requests.py @@ -1,7 +1,7 @@ import asyncio +import logging import pytest -import zigpy.zdo import zigpy.endpoint import zigpy.profiles import zigpy.zdo.types as zdo_t @@ -967,3 +967,36 @@ async def test_route_discovery_concurrency(device, make_application): assert route_discovery2.call_count == 2 await app.shutdown() + + +@pytest.mark.parametrize("device", [FormedLaunchpadCC26X2R1]) +async def test_zdo_from_unknown(device, make_application, caplog, mocker): + mocker.patch("zigpy_znp.zigbee.application.IEEE_ADDR_DISCOVERY_TIMEOUT", new=0.1) + + app, znp_server = make_application(server_cls=device) + + znp_server.reply_once_to( + request=c.ZDO.IEEEAddrReq.Req(partial=True), + responses=[c.ZDO.IEEEAddrReq.Rsp(Status=t.Status.SUCCESS)], + ) + + await app.startup(auto_form=False) + + caplog.set_level(logging.WARNING) + + znp_server.send( + c.ZDO.MsgCbIncoming.Callback( + Src=0x1234, + IsBroadcast=t.Bool.false, + ClusterId=zdo_t.ZDOCmd.Mgmt_Leave_rsp, + SecurityUse=0, + TSN=123, + MacDst=0x0000, + Data=t.Bytes([123, 0x00]), + ) + ) + + await asyncio.sleep(0.5) + assert "unknown device" in caplog.text + + await app.shutdown() diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index cd611533..1f090270 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -693,11 +693,13 @@ async def on_zdo_message(self, msg: c.ZDO.MsgCbIncoming.Callback) -> None: self.on_zdo_device_announce(*args) device = self.get_device(ieee=kwargs["IEEEAddr"]) else: - device = await self._get_or_discover_device(nwk=msg.Src) - - if device is None: - LOGGER.warning("Received a ZDO message from an unknown device: %s", msg.Src) - return + try: + device = await self._get_or_discover_device(nwk=msg.Src) + except KeyError: + LOGGER.warning( + "Received a ZDO message from an unknown device: %s", msg.Src + ) + return self.handle_message( sender=device, From 2b83b5783ae893d5da2b09f76e59c8e32358443f Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 11 Jan 2022 13:15:39 -0500 Subject: [PATCH 12/19] Add a few more ignored ZDO callbacks --- zigpy_znp/zigbee/application.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 1f090270..829f5b77 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -658,6 +658,8 @@ def _bind_callbacks(self) -> None: c.ZDO.SimpleDescRsp, c.ZDO.ActiveEpRsp, c.ZDO.MgmtLqiRsp, + c.ZDO.BindRsp, + c.ZDO.UnBindRsp, ]: self._znp.callback_for_response( ignored_msg.Callback(partial=True), From 0d9837e72fa137d5fcd0af139ae82729db0f8a23 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Tue, 11 Jan 2022 13:31:56 -0500 Subject: [PATCH 13/19] Organize ignored callbacks into a top-level list --- zigpy_znp/zigbee/application.py | 34 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 829f5b77..a226c222 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -71,6 +71,22 @@ REQUEST_RETRYABLE_ERRORS = REQUEST_TRANSIENT_ERRORS | REQUEST_ROUTING_ERRORS +IGNORED_ZDO_CALLBACKS = [ + c.ZDO.EndDeviceAnnceInd, + c.ZDO.LeaveInd, + c.ZDO.PermitJoinInd, + c.ZDO.ParentAnnceRsp, + c.ZDO.ConcentratorInd, + c.ZDO.MgmtNWKUpdateNotify, + c.ZDO.MgmtPermitJoinRsp, + c.ZDO.NodeDescRsp, + c.ZDO.SimpleDescRsp, + c.ZDO.ActiveEpRsp, + c.ZDO.MgmtLqiRsp, + c.ZDO.BindRsp, + c.ZDO.UnBindRsp, +] + LOGGER = logging.getLogger(__name__) @@ -646,23 +662,9 @@ def _bind_callbacks(self) -> None: ) # Handle messages that we do not use to prevent unnecessary WARNINGs in logs - for ignored_msg in [ - c.ZDO.EndDeviceAnnceInd, - c.ZDO.LeaveInd, - c.ZDO.PermitJoinInd, - c.ZDO.ParentAnnceRsp, - c.ZDO.ConcentratorInd, - c.ZDO.MgmtNWKUpdateNotify, - c.ZDO.MgmtPermitJoinRsp, - c.ZDO.NodeDescRsp, - c.ZDO.SimpleDescRsp, - c.ZDO.ActiveEpRsp, - c.ZDO.MgmtLqiRsp, - c.ZDO.BindRsp, - c.ZDO.UnBindRsp, - ]: + for ignored_callback in IGNORED_ZDO_CALLBACKS: self._znp.callback_for_response( - ignored_msg.Callback(partial=True), + ignored_callback.Callback(partial=True), self.on_intentionally_unhandled_message, ) From cbecee4f9f918df429f861468d92f5f7a625c153 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 12 Jan 2022 11:09:06 -0500 Subject: [PATCH 14/19] Do not log warnings on unhandled commands --- zigpy_znp/api.py | 2 +- zigpy_znp/zigbee/application.py | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/zigpy_znp/api.py b/zigpy_znp/api.py index 022984d6..e7cd359e 100644 --- a/zigpy_znp/api.py +++ b/zigpy_znp/api.py @@ -718,7 +718,7 @@ def _unhandled_command(self, command: t.CommandBase): Called when a command that is not handled by any listener is received. """ - LOGGER.warning("Received an unhandled command: %s", command) + LOGGER.debug("Command was not handled: %s", command) @contextlib.asynccontextmanager async def capture_responses(self, responses): diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index a226c222..bfdc0f6a 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -71,22 +71,6 @@ REQUEST_RETRYABLE_ERRORS = REQUEST_TRANSIENT_ERRORS | REQUEST_ROUTING_ERRORS -IGNORED_ZDO_CALLBACKS = [ - c.ZDO.EndDeviceAnnceInd, - c.ZDO.LeaveInd, - c.ZDO.PermitJoinInd, - c.ZDO.ParentAnnceRsp, - c.ZDO.ConcentratorInd, - c.ZDO.MgmtNWKUpdateNotify, - c.ZDO.MgmtPermitJoinRsp, - c.ZDO.NodeDescRsp, - c.ZDO.SimpleDescRsp, - c.ZDO.ActiveEpRsp, - c.ZDO.MgmtLqiRsp, - c.ZDO.BindRsp, - c.ZDO.UnBindRsp, -] - LOGGER = logging.getLogger(__name__) @@ -661,13 +645,6 @@ def _bind_callbacks(self) -> None: c.ZDO.MsgCbIncoming.Callback(partial=True), self.on_zdo_message ) - # Handle messages that we do not use to prevent unnecessary WARNINGs in logs - for ignored_callback in IGNORED_ZDO_CALLBACKS: - self._znp.callback_for_response( - ignored_callback.Callback(partial=True), - self.on_intentionally_unhandled_message, - ) - # These are responses to a broadcast but we ignore all but the first self._znp.callback_for_response( c.ZDO.IEEEAddrRsp.Callback(partial=True), From 5e14aef08b402aee0044c1545556621ea7ebfc22 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Wed, 12 Jan 2022 19:43:13 -0500 Subject: [PATCH 15/19] Create ZDO handlers for commands that require using the internal API --- zigpy_znp/zigbee/application.py | 53 +++++++----- zigpy_znp/zigbee/device.py | 145 ++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 21 deletions(-) create mode 100644 zigpy_znp/zigbee/device.py diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index bfdc0f6a..27a003e5 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -33,6 +33,7 @@ from zigpy_znp.utils import combine_concurrent_calls from zigpy_znp.exceptions import CommandNotRecognized, InvalidCommandResponse from zigpy_znp.types.nvids import OsalNvIds +from zigpy_znp.zigbee.device import ZNPCoordinator ZDO_ENDPOINT = 0 ZHA_ENDPOINT = 1 @@ -74,27 +75,6 @@ LOGGER = logging.getLogger(__name__) -class ZNPCoordinator(zigpy.device.Device): - """ - Coordinator zigpy device that keeps track of our endpoints and clusters. - """ - - @property - def manufacturer(self): - return "Texas Instruments" - - @property - def model(self): - if self.application._znp.version > 3.0: - model = "CC1352/CC2652" - version = "3.30+" - else: - model = "CC2538" if self.application._znp.nvram.align_structs else "CC2531" - version = "Home 1.2" if self.application._znp.version == 1.2 else "3.0.x" - - return f"{model}, Z-Stack {version} (build {self.application._zstack_build_id})" - - class ControllerApplication(zigpy.application.ControllerApplication): SCHEMA = conf.CONFIG_SCHEMA SCHEMA_DEVICE = conf.SCHEMA_DEVICE @@ -289,6 +269,10 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): # Receive a callback for every known ZDO command for cluster_id in zdo_t.ZDOCmd: + # Ignore ZDO requests + if cluster_id < 0x8000: + continue + await self._znp.request(c.ZDO.MsgCallbackRegister.Req(ClusterId=cluster_id)) # Setup the coordinator as a zigpy device and initialize it to request node info @@ -1262,6 +1246,33 @@ async def _send_request_raw( RspStatus=t.Status.SUCCESS, callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), ) + # Internally forward ZDO requests destined for the coordinator back to zigpy so + # we can send Z-Stack internal requests when necessary + elif dst_ep == ZDO_ENDPOINT and ( + # Broadcast that will reach the device + ( + dst_addr.mode == t.AddrMode.Broadcast + and dst_addr.address + in ( + zigpy.types.BroadcastAddress.ALL_DEVICES, + zigpy.types.BroadcastAddress.RX_ON_WHEN_IDLE, + zigpy.types.BroadcastAddress.ALL_ROUTERS_AND_COORDINATOR, + ) + ) + # Or a direct unicast request + or ( + dst_addr.mode == t.AddrMode.NWK + and dst_addr.address == self.zigpy_device.nwk + ) + ): + self.handle_message( + sender=self.zigpy_device, + profile=ZDO_PROFILE, + cluster=cluster, + src_ep=ZDO_ENDPOINT, + dst_ep=ZDO_ENDPOINT, + message=data, + ) if dst_addr.mode == t.AddrMode.Broadcast or dst_ep == ZDO_ENDPOINT: # Broadcasts and ZDO requests will not receive a confirmation diff --git a/zigpy_znp/zigbee/device.py b/zigpy_znp/zigbee/device.py new file mode 100644 index 00000000..20d3a292 --- /dev/null +++ b/zigpy_znp/zigbee/device.py @@ -0,0 +1,145 @@ +import asyncio +import logging + +import zigpy.zdo +import zigpy.device +import zigpy.zdo.types as zdo_t + +import zigpy_znp.types as t +import zigpy_znp.commands as c + +LOGGER = logging.getLogger(__name__) + + +class ZNPCoordinator(zigpy.device.Device): + """ + Coordinator zigpy device that keeps track of our endpoints and clusters. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + assert hasattr(self, "zdo") + self.zdo = ZNPZDOEndpoint(self) + self.endpoints[0] = self.zdo + + @property + def manufacturer(self): + return "Texas Instruments" + + @property + def model(self): + if self.application._znp.version > 3.0: + model = "CC1352/CC2652" + version = "3.30+" + else: + model = "CC2538" if self.application._znp.nvram.align_structs else "CC2531" + version = "Home 1.2" if self.application._znp.version == 1.2 else "3.0.x" + + return f"{model}, Z-Stack {version} (build {self.application._zstack_build_id})" + + +class ZNPZDOEndpoint(zigpy.zdo.ZDO): + @property + def app(self): + return self.device.application + + def handle_mgmt_permit_joining_req( + self, + hdr: zdo_t.ZDOHeader, + PermitDuration: t.uint8_t, + TC_Significant: t.Bool, + *, + dst_addressing, + ): + """ + Handles ZDO `Mgmt_Permit_Joining_req` sent to the coordinator. + """ + + self.create_catching_task( + self.async_handle_mgmt_permit_joining_req( + hdr, PermitDuration, TC_Significant, dst_addressing=dst_addressing + ) + ) + + async def async_handle_mgmt_permit_joining_req( + self, + hdr: zdo_t.ZDOHeader, + PermitDuration: t.uint8_t, + TC_Significant: t.Bool, + *, + dst_addressing, + ): + # Joins *must* be sent via a ZDO command. Otherwise, Z-Stack will not actually + # permit the coordinator to send the network key while routers will. + await self.app._znp.request_callback_rsp( + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=t.AddrMode.NWK, + Dst=0x0000, + Duration=PermitDuration, + TCSignificance=TC_Significant, + ), + RspStatus=t.Status.SUCCESS, + callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), + ) + + def handle_mgmt_nwk_update_req( + self, hdr: zdo_t.ZDOHeader, NwkUpdate: zdo_t.NwkUpdate, *, dst_addressing + ): + """ + Handles ZDO `Mgmt_NWK_Update_req` sent to the coordinator. + """ + + self.create_catching_task( + self.async_handle_mgmt_nwk_update_req( + hdr, NwkUpdate, dst_addressing=dst_addressing + ) + ) + + async def async_handle_mgmt_nwk_update_req( + self, hdr: zdo_t.ZDOHeader, NwkUpdate: zdo_t.NwkUpdate, *, dst_addressing + ): + # Energy scans are handled properly by Z-Stack + if NwkUpdate.ScanDuration not in ( + zdo_t.NwkUpdate.CHANNEL_CHANGE_REQ, + zdo_t.NwkUpdate.CHANNEL_MASK_MANAGER_ADDR_CHANGE_REQ, + ): + return + + old_network_info = self.app.state.network_information + + if ( + t.Channels.from_channel_list([old_network_info.channel]) + == NwkUpdate.ScanChannels + ): + LOGGER.info("NWK update request is ignored when channel does not change") + return + + await self.app._znp.request( + request=c.ZDO.MgmtNWKUpdateReq.Req( + Dst=0x0000, + DstAddrMode=t.AddrMode.NWK, + Channels=NwkUpdate.ScanChannels, + ScanDuration=NwkUpdate.ScanDuration, + ScanCount=NwkUpdate.ScanCount or 0, + NwkManagerAddr=NwkUpdate.nwkManagerAddr or 0x0000, + ), + RspStatus=t.Status.SUCCESS, + ) + + # Wait until the network info changes, it can take ~5s + while ( + self.app.state.network_information.nwk_update_id + == old_network_info.nwk_update_id + ): + await self.app.load_network_info(load_devices=False) + await asyncio.sleep(1) + + # Z-Stack automatically increments the NWK update ID instead of setting it + # TODO: Directly set it once radio settings API is finalized. + if NwkUpdate.nwkUpdateId != self.app.state.network_information.nwk_update_id: + LOGGER.warning( + f"`nwkUpdateId` was incremented to" + f" {self.app.state.network_information.nwk_update_id} instead of being" + f" set to {NwkUpdate.nwkUpdateId}" + ) From adc2b546902b239e3df18b8aa6e7bb28c2ca3619 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Thu, 13 Jan 2022 14:26:57 -0500 Subject: [PATCH 16/19] Explicitly ignore requests, some commands below 0x8000 are announcements https://github.com/zigpy/zigpy-znp/pull/109#issuecomment-1012415890 --- zigpy_znp/zigbee/application.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index 27a003e5..a52b8162 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -269,8 +269,8 @@ async def _startup(self, auto_form=False, force_form=False, read_only=False): # Receive a callback for every known ZDO command for cluster_id in zdo_t.ZDOCmd: - # Ignore ZDO requests - if cluster_id < 0x8000: + # Ignore outgoing ZDO requests, only receive announcements and responses + if cluster_id.name.endswith(("_req", "_set")): continue await self._znp.request(c.ZDO.MsgCallbackRegister.Req(ClusterId=cluster_id)) From 9d9df3abcd0be3b4bbbf798a8ecfc2711a7eee3b Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Thu, 13 Jan 2022 14:27:46 -0500 Subject: [PATCH 17/19] Clean up and better document ZDO exceptions --- zigpy_znp/zigbee/application.py | 64 ++++++++++++++++----------------- zigpy_znp/zigbee/device.py | 25 +++---------- 2 files changed, 37 insertions(+), 52 deletions(-) diff --git a/zigpy_znp/zigbee/application.py b/zigpy_znp/zigbee/application.py index a52b8162..80f4364a 100644 --- a/zigpy_znp/zigbee/application.py +++ b/zigpy_znp/zigbee/application.py @@ -1233,24 +1233,26 @@ async def _send_request_raw( Data=data, ) - # XXX: Joins *must* be sent via a ZDO command. Otherwise, Z-Stack will not - # actually permit the coordinator to send the network key. - if dst_ep == ZDO_ENDPOINT and cluster == zdo_t.ZDOCmd.Mgmt_Permit_Joining_req: - await self._znp.request_callback_rsp( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=dst_addr.mode, - Dst=dst_addr.address, - Duration=data[1], - TCSignificance=data[2], - ), - RspStatus=t.Status.SUCCESS, - callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), - ) - # Internally forward ZDO requests destined for the coordinator back to zigpy so - # we can send Z-Stack internal requests when necessary - elif dst_ep == ZDO_ENDPOINT and ( - # Broadcast that will reach the device - ( + # Z-Stack requires special treatment when sending ZDO requests + if dst_ep == ZDO_ENDPOINT: + # XXX: Joins *must* be sent via a ZDO command, even if they are directly + # addressing another device. The router will receive the ZDO request and a + # device will try to join, but Z-Stack will never send the network key. + if cluster == zdo_t.ZDOCmd.Mgmt_Permit_Joining_req: + await self._znp.request_callback_rsp( + request=c.ZDO.MgmtPermitJoinReq.Req( + AddrMode=dst_addr.mode, + Dst=dst_addr.address, + Duration=data[1], + TCSignificance=data[2], + ), + RspStatus=t.Status.SUCCESS, + callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), + ) + # Internally forward ZDO requests destined for the coordinator back to zigpy + # so we can send internal Z-Stack requests when necessary + elif ( + # Broadcast that will reach the device dst_addr.mode == t.AddrMode.Broadcast and dst_addr.address in ( @@ -1258,23 +1260,21 @@ async def _send_request_raw( zigpy.types.BroadcastAddress.RX_ON_WHEN_IDLE, zigpy.types.BroadcastAddress.ALL_ROUTERS_AND_COORDINATOR, ) - ) - # Or a direct unicast request - or ( + ) or ( + # Or a direct unicast request dst_addr.mode == t.AddrMode.NWK and dst_addr.address == self.zigpy_device.nwk - ) - ): - self.handle_message( - sender=self.zigpy_device, - profile=ZDO_PROFILE, - cluster=cluster, - src_ep=ZDO_ENDPOINT, - dst_ep=ZDO_ENDPOINT, - message=data, - ) + ): + self.handle_message( + sender=self.zigpy_device, + profile=profile, + cluster=cluster, + src_ep=src_ep, + dst_ep=dst_ep, + message=data, + ) - if dst_addr.mode == t.AddrMode.Broadcast or dst_ep == ZDO_ENDPOINT: + if dst_ep == ZDO_ENDPOINT or dst_addr.mode == t.AddrMode.Broadcast: # Broadcasts and ZDO requests will not receive a confirmation response = await self._znp.request( request=request, RspStatus=t.Status.SUCCESS diff --git a/zigpy_znp/zigbee/device.py b/zigpy_znp/zigbee/device.py index 20d3a292..5d6ba13f 100644 --- a/zigpy_znp/zigbee/device.py +++ b/zigpy_znp/zigbee/device.py @@ -10,6 +10,8 @@ LOGGER = logging.getLogger(__name__) +NWK_UPDATE_LOOP_DELAY = 1 + class ZNPCoordinator(zigpy.device.Device): """ @@ -44,24 +46,6 @@ class ZNPZDOEndpoint(zigpy.zdo.ZDO): def app(self): return self.device.application - def handle_mgmt_permit_joining_req( - self, - hdr: zdo_t.ZDOHeader, - PermitDuration: t.uint8_t, - TC_Significant: t.Bool, - *, - dst_addressing, - ): - """ - Handles ZDO `Mgmt_Permit_Joining_req` sent to the coordinator. - """ - - self.create_catching_task( - self.async_handle_mgmt_permit_joining_req( - hdr, PermitDuration, TC_Significant, dst_addressing=dst_addressing - ) - ) - async def async_handle_mgmt_permit_joining_req( self, hdr: zdo_t.ZDOHeader, @@ -112,7 +96,7 @@ async def async_handle_mgmt_nwk_update_req( t.Channels.from_channel_list([old_network_info.channel]) == NwkUpdate.ScanChannels ): - LOGGER.info("NWK update request is ignored when channel does not change") + LOGGER.warning("NWK update request is ignored when channel does not change") return await self.app._znp.request( @@ -121,6 +105,7 @@ async def async_handle_mgmt_nwk_update_req( DstAddrMode=t.AddrMode.NWK, Channels=NwkUpdate.ScanChannels, ScanDuration=NwkUpdate.ScanDuration, + # Missing fields in the request cannot be `None` in the Z-Stack command ScanCount=NwkUpdate.ScanCount or 0, NwkManagerAddr=NwkUpdate.nwkManagerAddr or 0x0000, ), @@ -133,7 +118,7 @@ async def async_handle_mgmt_nwk_update_req( == old_network_info.nwk_update_id ): await self.app.load_network_info(load_devices=False) - await asyncio.sleep(1) + await asyncio.sleep(NWK_UPDATE_LOOP_DELAY) # Z-Stack automatically increments the NWK update ID instead of setting it # TODO: Directly set it once radio settings API is finalized. From 4edfb135e5ba8c762cbfa5c2a50ebce1329ac91a Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 14 Jan 2022 10:46:07 -0500 Subject: [PATCH 18/19] Add unit tests and properly handle unicast ZDO requests to coordinator --- tests/application/test_zdo_requests.py | 100 +++++++++++++++++++++++++ zigpy_znp/zigbee/device.py | 93 ++++++++++++++++++----- 2 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 tests/application/test_zdo_requests.py diff --git a/tests/application/test_zdo_requests.py b/tests/application/test_zdo_requests.py new file mode 100644 index 00000000..fdfd34ba --- /dev/null +++ b/tests/application/test_zdo_requests.py @@ -0,0 +1,100 @@ +import asyncio + +import pytest +import zigpy.zdo +import zigpy.types as zigpy_t +import zigpy.zdo.types as zdo_t + +import zigpy_znp.types as t +import zigpy_znp.commands as c + +from tests.conftest import FormedLaunchpadCC26X2R1 + + +@pytest.mark.parametrize( + "broadcast,nwk_update_id,change_channel", + [ + (False, 1, False), + (False, 1, True), + (True, 1, False), + (False, 200, True), + ], +) +@pytest.mark.parametrize("device", [FormedLaunchpadCC26X2R1]) +async def test_mgmt_nwk_update_req( + device, broadcast, nwk_update_id, change_channel, make_application, mocker +): + mocker.patch("zigpy_znp.zigbee.device.NWK_UPDATE_LOOP_DELAY", 0.1) + + app, znp_server = make_application(server_cls=device) + + if change_channel: + new_channel = 11 + (26 - znp_server.nib.nwkLogicalChannel) + else: + new_channel = znp_server.nib.nwkLogicalChannel + + async def update_channel(req): + # Wait a bit before updating + await asyncio.sleep(0.5) + + znp_server.nib = znp_server.nib.replace( + nwkUpdateId=znp_server.nib.nwkUpdateId + 1, + nwkLogicalChannel=list(req.Channels)[0], + channelList=req.Channels, + ) + + yield + + znp_server.reply_once_to( + request=c.AF.DataRequestExt.Req( + DstEndpoint=0, + ClusterId=zdo_t.ZDOCmd.Mgmt_NWK_Update_req, + partial=True, + ), + responses=[c.AF.DataRequestExt.Rsp(Status=t.Status.SUCCESS)], + ) + + nwk_update_req = znp_server.reply_once_to( + request=c.ZDO.MgmtNWKUpdateReq.Req( + Dst=0x0000, + DstAddrMode=t.AddrMode.NWK, + Channels=t.Channels.from_channel_list([new_channel]), + ScanDuration=254, + # Missing fields in the request cannot be `None` in the Z-Stack command + ScanCount=0, + NwkManagerAddr=0x0000, + ), + responses=[ + c.ZDO.MgmtNWKUpdateReq.Rsp(Status=t.Status.SUCCESS), + update_channel, + ], + ) + + await app.startup(auto_form=False) + + update = zdo_t.NwkUpdate( + ScanChannels=t.Channels.from_channel_list([new_channel]), + ScanDuration=zdo_t.NwkUpdate.CHANNEL_CHANGE_REQ, + nwkUpdateId=nwk_update_id, + ) + + if broadcast: + await zigpy.zdo.broadcast( + app, + zdo_t.ZDOCmd.Mgmt_NWK_Update_req, + 0x0000, # group id (ignore) + 0, # radius + update, + broadcast_address=zigpy_t.BroadcastAddress.ALL_ROUTERS_AND_COORDINATOR, + ) + else: + await app.zigpy_device.zdo.Mgmt_NWK_Update_req(update) + + if change_channel: + await nwk_update_req + else: + assert not nwk_update_req.done() + + assert znp_server.nib.nwkLogicalChannel == list(update.ScanChannels)[0] + + await app.shutdown() diff --git a/zigpy_znp/zigbee/device.py b/zigpy_znp/zigbee/device.py index 5d6ba13f..d93f02e3 100644 --- a/zigpy_znp/zigbee/device.py +++ b/zigpy_znp/zigbee/device.py @@ -1,12 +1,16 @@ +from __future__ import annotations + import asyncio import logging import zigpy.zdo import zigpy.device import zigpy.zdo.types as zdo_t +import zigpy.application import zigpy_znp.types as t import zigpy_znp.commands as c +import zigpy_znp.zigbee.application as znp_app LOGGER = logging.getLogger(__name__) @@ -40,31 +44,61 @@ def model(self): return f"{model}, Z-Stack {version} (build {self.application._zstack_build_id})" + def request( + self, + profile, + cluster, + src_ep, + dst_ep, + sequence, + data, + expect_reply=True, + # Extend the default timeout + timeout=2 * zigpy.device.APS_REPLY_TIMEOUT, + use_ieee=False, + ): + """ + Normal `zigpy.device.Device:request` except its default timeout is longer. + """ + + return super().request( + profile, + cluster, + src_ep, + dst_ep, + sequence, + data, + expect_reply=expect_reply, + timeout=timeout, + use_ieee=use_ieee, + ) + class ZNPZDOEndpoint(zigpy.zdo.ZDO): @property - def app(self): + def app(self) -> zigpy.application.ControllerApplication: return self.device.application - async def async_handle_mgmt_permit_joining_req( - self, - hdr: zdo_t.ZDOHeader, - PermitDuration: t.uint8_t, - TC_Significant: t.Bool, - *, - dst_addressing, + def _send_loopback_reply( + self, command_id: zdo_t.ZDOCmd, *, tsn: t.uint8_t, **kwargs ): - # Joins *must* be sent via a ZDO command. Otherwise, Z-Stack will not actually - # permit the coordinator to send the network key while routers will. - await self.app._znp.request_callback_rsp( - request=c.ZDO.MgmtPermitJoinReq.Req( - AddrMode=t.AddrMode.NWK, - Dst=0x0000, - Duration=PermitDuration, - TCSignificance=TC_Significant, - ), - RspStatus=t.Status.SUCCESS, - callback=c.ZDO.MgmtPermitJoinRsp.Callback(Src=0x0000, partial=True), + """ + Constructs and sends back a loopback ZDO response. + """ + + message = t.uint8_t(tsn).serialize() + self._serialize( + command_id, *kwargs.values() + ) + + LOGGER.debug("Sending loopback reply %s (%s), tsn=%s", command_id, kwargs, tsn) + + self.app.handle_message( + sender=self.app.zigpy_device, + profile=znp_app.ZDO_PROFILE, + cluster=command_id, + src_ep=znp_app.ZDO_ENDPOINT, + dst_ep=znp_app.ZDO_ENDPOINT, + message=message, ) def handle_mgmt_nwk_update_req( @@ -83,7 +117,7 @@ def handle_mgmt_nwk_update_req( async def async_handle_mgmt_nwk_update_req( self, hdr: zdo_t.ZDOHeader, NwkUpdate: zdo_t.NwkUpdate, *, dst_addressing ): - # Energy scans are handled properly by Z-Stack + # Energy scans are handled properly by Z-Stack, no need to do anything if NwkUpdate.ScanDuration not in ( zdo_t.NwkUpdate.CHANNEL_CHANGE_REQ, zdo_t.NwkUpdate.CHANNEL_MASK_MANAGER_ADDR_CHANGE_REQ, @@ -97,6 +131,15 @@ async def async_handle_mgmt_nwk_update_req( == NwkUpdate.ScanChannels ): LOGGER.warning("NWK update request is ignored when channel does not change") + self._send_loopback_reply( + zdo_t.ZDOCmd.Mgmt_NWK_Update_rsp, + Status=zdo_t.Status.SUCCESS, + ScannedChannels=t.Channels.NO_CHANNELS, + TotalTransmissions=0, + TransmissionFailures=0, + EnergyValues=[], + tsn=hdr.tsn, + ) return await self.app._znp.request( @@ -128,3 +171,13 @@ async def async_handle_mgmt_nwk_update_req( f" {self.app.state.network_information.nwk_update_id} instead of being" f" set to {NwkUpdate.nwkUpdateId}" ) + + self._send_loopback_reply( + zdo_t.ZDOCmd.Mgmt_NWK_Update_rsp, + Status=zdo_t.Status.SUCCESS, + ScannedChannels=t.Channels.NO_CHANNELS, + TotalTransmissions=0, + TransmissionFailures=0, + EnergyValues=[], + tsn=hdr.tsn, + ) From 9ba02f1ec78713cd34743dc320ae3da31f367339 Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Fri, 14 Jan 2022 12:58:02 -0500 Subject: [PATCH 19/19] Reduce unnecessary logging verbosity Unhandled commands will be logged completely in the preceding line --- zigpy_znp/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zigpy_znp/api.py b/zigpy_znp/api.py index e7cd359e..f7e155c2 100644 --- a/zigpy_znp/api.py +++ b/zigpy_znp/api.py @@ -718,7 +718,7 @@ def _unhandled_command(self, command: t.CommandBase): Called when a command that is not handled by any listener is received. """ - LOGGER.debug("Command was not handled: %s", command) + LOGGER.debug("Command was not handled") @contextlib.asynccontextmanager async def capture_responses(self, responses):