Test failing backward any pending HTLCs

Upon channel failure, any pending HTLCs in a channel's holding cell must
be failed backward. The added test exercises this behavior and
demonstrates a deadlock triggered within the handle_error!() macro. The
deadlock occurs when the channel_state lock is already held and then
reacquired when finish_force_close_channel() is called.

Author: jkczyz

lightning/src/ln/functional_tests.rs

@@ -2988,6 +2988,77 @@ fn test_commitment_revoked_fail_backward_exhaustive_b() {
 	do_test_commitment_revoked_fail_backward_exhaustive(true, false, true);
 }
 
+#[test]
+fn fail_pending_htlc_backward_upon_channel_failure() {
+	let chanmon_cfgs = create_chanmon_cfgs(3);
+	let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+	let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+	let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+	let _chan_1 = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 500_000_000, InitFeatures::supported(), InitFeatures::supported());
+	let chan_2 = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 1_000_000, 500_000_000, InitFeatures::supported(), InitFeatures::supported());
+
+	// Alice -> Bob -> Chuck: Route a payment but without Chuck sending revoke_and_ack.
+	{
+		let (_, payment_hash) = route_payment(&nodes[0], &[&nodes[1]], 50_000);
+		let remaining_route = nodes[1].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 50_000, TEST_FINAL_CLTV).unwrap();
+		nodes[1].node.send_payment(remaining_route, payment_hash).unwrap();
+		check_added_monitors!(nodes[1], 1);
+
+		let payment_event = {
+			let mut events = nodes[1].node.get_and_clear_pending_msg_events();
+			assert_eq!(events.len(), 1);
+			SendEvent::from_event(events.remove(0))
+		};
+		assert_eq!(payment_event.node_id, nodes[2].node.get_our_node_id());
+		assert_eq!(payment_event.msgs.len(), 1);
+	}
+
+	// Alice -> Bob -> Chuck: Route another payment but now Bob waits for Chuck's earlier revoke_and_ack.
+	let (_, failed_payment_hash) = route_payment(&nodes[0], &[&nodes[1]], 50_000);
+	{
+		let remaining_route = nodes[1].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 50_000, TEST_FINAL_CLTV).unwrap();
+		nodes[1].node.send_payment(remaining_route, failed_payment_hash).unwrap();
+		check_added_monitors!(nodes[1], 0);
+
+		assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+	}
+
+	// Alice <- Bob <- Chuck: Send a malformed update_add_htlc so Bob fails the channel.
+	{
+		let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 50_000, TEST_FINAL_CLTV).unwrap();
+		let (_, payment_hash) = get_payment_preimage_hash!(nodes[2]);
+
+		let secp_ctx = Secp256k1::new();
+		let session_priv = {
+			let mut session_key = [0; 32];
+			let mut rng = thread_rng();
+			rng.fill_bytes(&mut session_key);
+			SecretKey::from_slice(&session_key).expect("RNG is bad!")
+		};
+
+		let current_height = nodes[2].node.latest_block_height.load(Ordering::Acquire) as u32 + 1;
+		let (onion_payloads, _amount_msat, cltv_expiry) = onion_utils::build_onion_payloads(&route, current_height).unwrap();
+		let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route, &session_priv).unwrap();
+		let onion_routing_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
+
+		// Send a 0-msat update_add_htlc to fail the channel.
+		let update_add_htlc = msgs::UpdateAddHTLC {
+			channel_id: chan_2.2,
+			htlc_id: 0,
+			amount_msat: 0,
+			payment_hash,
+			cltv_expiry,
+			onion_routing_packet,
+		};
+		nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_htlc);
+	}
+
+	// Check that Bob fails backward the pending HTLC from the second payment.
+	expect_payment_failed!(nodes[1], failed_payment_hash, true);
+	check_closed_broadcast!(nodes[1], true);
+	check_added_monitors!(nodes[1], 1);
+}
+
 #[test]
 fn test_htlc_ignore_latest_remote_commitment() {
 	// Test that HTLC transactions spending the latest remote commitment transaction are simply