From 2bc17e56331fae78111c91f7e8e8aece5b2b7ec2 Mon Sep 17 00:00:00 2001 From: SeongChan Lee Date: Thu, 26 Sep 2019 16:23:39 +0900 Subject: [PATCH 1/4] Cleanup vote functions --- core/src/consensus/tendermint/message.rs | 24 ---- core/src/consensus/tendermint/worker.rs | 149 ++++++++++++----------- 2 files changed, 76 insertions(+), 97 deletions(-) diff --git a/core/src/consensus/tendermint/message.rs b/core/src/consensus/tendermint/message.rs index a811e98da1..65cd5dfbcc 100644 --- a/core/src/consensus/tendermint/message.rs +++ b/core/src/consensus/tendermint/message.rs @@ -18,12 +18,10 @@ use std::cmp; use ccrypto::blake256; use ckey::{verify_schnorr, Error as KeyError, Public, SchnorrSignature}; -use ctypes::Header; use primitives::{Bytes, H256}; use rlp::{Decodable, DecoderError, Encodable, RlpStream, UntrustedRlp}; use snap; -use super::super::validator_set::DynamicValidator; use super::super::BitSet; use super::{BlockHash, Height, Step, View}; @@ -331,28 +329,6 @@ pub struct ConsensusMessage { } impl ConsensusMessage { - /// If a locked node re-proposes locked proposal, the proposed_view is different from the header's view. - pub fn new_proposal( - signature: SchnorrSignature, - validators: &DynamicValidator, - proposal_header: &Header, - proposed_view: View, - prev_proposer_idx: usize, - ) -> Result { - let height = proposal_header.number() as Height; - let signer_index = - validators.proposer_index(*proposal_header.parent_hash(), prev_proposer_idx, proposed_view as usize); - - Ok(ConsensusMessage { - signature, - signer_index, - on: VoteOn { - step: VoteStep::new(height, proposed_view, Step::Propose), - block_hash: Some(proposal_header.hash()), - }, - }) - } - pub fn signature(&self) -> SchnorrSignature { self.signature } diff --git a/core/src/consensus/tendermint/worker.rs b/core/src/consensus/tendermint/worker.rs index 0b714b576e..f4506d0749 100644 --- a/core/src/consensus/tendermint/worker.rs +++ b/core/src/consensus/tendermint/worker.rs @@ -821,7 +821,8 @@ impl Worker { } fn generate_and_broadcast_message(&mut self, block_hash: Option, is_restoring: bool) { - if let Some(message) = self.generate_message(block_hash, is_restoring) { + if let Some(message) = self.vote_on_block_hash(block_hash).expect("Error while vote") { + self.handle_valid_message(&message, is_restoring); if !is_restoring { self.backup(); } @@ -829,37 +830,6 @@ impl Worker { } } - fn generate_message(&mut self, block_hash: Option, is_restoring: bool) -> Option { - let height = self.height; - let r = self.view; - let on = VoteOn { - step: VoteStep::new(height, r, self.step.to_step()), - block_hash, - }; - let signer_index = self.signer_index().or_else(|| { - ctrace!(ENGINE, "No message, since there is no engine signer."); - None - })?; - let signature = self - .sign(&on) - .map_err(|error| { - ctrace!(ENGINE, "{}th validator could not sign the message {}", signer_index, error); - error - }) - .ok()?; - let message = ConsensusMessage { - signature, - signer_index, - on, - }; - self.votes_received.set(signer_index); - self.votes.vote(message.clone()); - cinfo!(ENGINE, "Generated {:?} as {}th validator.", message, signer_index); - self.handle_valid_message(&message, is_restoring); - - Some(message) - } - fn handle_valid_message(&mut self, message: &ConsensusMessage, is_restoring: bool) { let vote_step = &message.on.step; let is_newer_than_lock = match self.last_two_thirds_majority.view() { @@ -1117,7 +1087,6 @@ impl Worker { } let header = sealed_block.header(); - let hash = header.hash(); let parent_hash = header.parent_hash(); if let TendermintState::ProposeWaitBlockGeneration { @@ -1137,19 +1106,9 @@ impl Worker { ); return } - let prev_proposer_idx = self.block_proposer_idx(*parent_hash).expect("Prev block must exists"); - debug_assert_eq!(Ok(self.view), TendermintSealView::new(header.seal()).consensus_view()); - let vote_on = VoteOn { - step: VoteStep::new(header.number() as Height, self.view, Step::Propose), - block_hash: Some(hash), - }; - let signature = self.sign(&vote_on).expect("I am proposer"); - self.votes.vote( - ConsensusMessage::new_proposal(signature, &*self.validators, header, self.view, prev_proposer_idx) - .expect("I am proposer"), - ); + self.vote_on_header_for_proposal(&header).expect("I'm a proposer"); self.step = TendermintState::ProposeWaitImported { block: Box::new(sealed_block.clone()), @@ -1499,18 +1458,7 @@ impl Worker { fn repropose_block(&mut self, block: encoded::Block) { let header = block.decode_header(); - let vote_on = VoteOn { - step: VoteStep::new(header.number() as Height, self.view, Step::Propose), - block_hash: Some(header.hash()), - }; - let parent_hash = header.parent_hash(); - let prev_proposer_idx = self.block_proposer_idx(*parent_hash).expect("Prev block must exists"); - let signature = self.sign(&vote_on).expect("I am proposer"); - self.votes.vote( - ConsensusMessage::new_proposal(signature, &*self.validators, &header, self.view, prev_proposer_idx) - .expect("I am proposer"), - ); - + self.vote_on_header_for_proposal(&header).expect("I am proposer"); self.proposal = Proposal::new_imported(header.hash()); self.broadcast_proposal_block(self.view, block); } @@ -1538,8 +1486,76 @@ impl Worker { self.signer.set_to_keep_decrypted_account(ap, address); } - fn sign(&self, vote_on: &VoteOn) -> Result { - self.signer.sign(vote_on.hash()).map_err(Into::into) + fn vote_on_block_hash(&mut self, block_hash: Option) -> Result, Error> { + let signer_index = if let Some(signer_index) = self.signer_index() { + signer_index + } else { + ctrace!(ENGINE, "No message, since there is no engine signer."); + return Ok(None) + }; + + let on = VoteOn { + step: VoteStep::new(self.height, self.view, self.step.to_step()), + block_hash, + }; + let signature = self.signer.sign(on.hash())?; + + let vote = ConsensusMessage { + signature, + signer_index, + on, + }; + + self.votes_received.set(vote.signer_index); + self.votes.vote(vote.clone()); + cinfo!(ENGINE, "Voted {:?} as {}th validator.", vote, signer_index); + Ok(Some(vote)) + } + + fn vote_on_header_for_proposal(&mut self, header: &Header) -> Result { + assert!(header.number() == self.height); + + let parent_hash = header.parent_hash(); + let prev_proposer_idx = self.block_proposer_idx(*parent_hash).expect("Prev block must exists"); + let signer_index = self.validators.proposer_index(*parent_hash, prev_proposer_idx, self.view as usize); + + let on = VoteOn { + step: VoteStep::new(self.height, self.view, Step::Propose), + block_hash: Some(header.hash()), + }; + let signature = self.signer.sign(on.hash())?; + + let vote = ConsensusMessage { + signature, + signer_index, + on, + }; + + self.votes.vote(vote.clone()); + cinfo!(ENGINE, "Voted {:?} as {}th proposer.", vote, signer_index); + Ok(vote) + } + + fn recover_proposal_vote( + &self, + header: &Header, + proposed_view: View, + signature: SchnorrSignature, + ) -> Option { + let prev_proposer_idx = self.block_proposer_idx(*header.parent_hash())?; + let signer_index = + self.validators.proposer_index(*header.parent_hash(), prev_proposer_idx, proposed_view as usize); + + let on = VoteOn { + step: VoteStep::new(header.number(), proposed_view, Step::Propose), + block_hash: Some(header.hash()), + }; + + Some(ConsensusMessage { + signature, + signer_index, + on, + }) } fn signer_index(&self) -> Option { @@ -1703,27 +1719,14 @@ impl Worker { return None } } - - let prev_proposer_idx = match self.block_proposer_idx(*parent_hash) { - Some(idx) => idx, + let message = match self.recover_proposal_vote(&header_view, proposed_view, signature) { + Some(vote) => vote, None => { cwarn!(ENGINE, "Prev block proposer does not exist for height {}", number); return None } }; - let message = ConsensusMessage::new_proposal( - signature, - &*self.validators, - &header_view, - proposed_view, - prev_proposer_idx, - ) - .map_err(|err| { - cwarn!(ENGINE, "Invalid proposal received: {:?}", err); - }) - .ok()?; - // If the proposal's height is current height + 1 and the proposal has valid precommits, // we should import it and increase height if number > (self.height + 1) as u64 { From 57f220a202b9e5a4a67fb19f6430c6d2143308e2 Mon Sep 17 00:00:00 2001 From: SeongChan Lee Date: Thu, 26 Sep 2019 16:24:12 +0900 Subject: [PATCH 2/4] Add VoteRegressionChecker --- core/src/consensus/tendermint/mod.rs | 1 + .../tendermint/vote_regression_checker.rs | 189 ++++++++++++++++++ core/src/consensus/tendermint/worker.rs | 9 +- 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 core/src/consensus/tendermint/vote_regression_checker.rs diff --git a/core/src/consensus/tendermint/mod.rs b/core/src/consensus/tendermint/mod.rs index 997e80cc78..8366faf1ae 100644 --- a/core/src/consensus/tendermint/mod.rs +++ b/core/src/consensus/tendermint/mod.rs @@ -22,6 +22,7 @@ mod network; mod params; pub mod types; pub mod vote_collector; +mod vote_regression_checker; mod worker; use std::sync::atomic::AtomicBool; diff --git a/core/src/consensus/tendermint/vote_regression_checker.rs b/core/src/consensus/tendermint/vote_regression_checker.rs new file mode 100644 index 0000000000..fc37907afa --- /dev/null +++ b/core/src/consensus/tendermint/vote_regression_checker.rs @@ -0,0 +1,189 @@ +use consensus::{Step, VoteOn}; +use std::cmp::Ordering; + +pub struct VoteRegressionChecker { + last_vote: Option, +} + +impl VoteRegressionChecker { + pub fn new() -> VoteRegressionChecker { + VoteRegressionChecker { + last_vote: None, + } + } + + pub fn check(&mut self, vote_on: &VoteOn) -> bool { + assert!(match vote_on.step.step { + Step::Propose | Step::Prevote | Step::Precommit => true, + _ => false, + }); + + let monotonic = if let Some(last_vote) = &self.last_vote { + match last_vote.step.cmp(&vote_on.step) { + Ordering::Less => true, + Ordering::Greater => false, + Ordering::Equal => last_vote.block_hash == vote_on.block_hash, + } + } else { + true + }; + + if monotonic { + self.last_vote = Some(vote_on.clone()); + } + monotonic + } +} + +#[cfg(test)] +mod tests { + use super::*; + use consensus::VoteStep; + use primitives::H256; + + #[test] + fn test_initial_set() { + let mut checker = VoteRegressionChecker::new(); + + let random_step = VoteStep::new(100, 10, Step::Prevote); + let random_hash = Some(H256::random()); + assert!(checker.check(&VoteOn { + step: random_step, + block_hash: random_hash + })) + } + + #[test] + #[should_panic] + fn test_disallow_commit() { + let mut checker = VoteRegressionChecker::new(); + + let random_commit_step = VoteStep::new(100, 10, Step::Commit); + let random_hash = Some(H256::random()); + assert!(checker.check(&VoteOn { + step: random_commit_step, + block_hash: random_hash + })) + } + + #[test] + fn test_allow_height_increase() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(checker.check(&VoteOn { + step: VoteStep::new(101, 10, Step::Prevote), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_disallow_height_decrease() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(!checker.check(&VoteOn { + step: VoteStep::new(99, 10, Step::Prevote), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_allow_view_increase() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(checker.check(&VoteOn { + step: VoteStep::new(100, 11, Step::Prevote), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_disallow_view_decrease() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(!checker.check(&VoteOn { + step: VoteStep::new(100, 9, Step::Prevote), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_allow_step_increased() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Precommit), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_disallow_step_decreased() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(!checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Propose), + block_hash: Some(H256::from(2)) + })) + } + + #[test] + fn test_allow_same_hash() { + let mut checker = VoteRegressionChecker::new(); + + let block_hash = Some(H256::random()); + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash, + }); + + assert!(checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash, + })) + } + + #[test] + fn test_disallow_hash_change() { + let mut checker = VoteRegressionChecker::new(); + + checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(1)), + }); + + assert!(!checker.check(&VoteOn { + step: VoteStep::new(100, 10, Step::Prevote), + block_hash: Some(H256::from(2)) + })) + } +} diff --git a/core/src/consensus/tendermint/worker.rs b/core/src/consensus/tendermint/worker.rs index f4506d0749..05473563f1 100644 --- a/core/src/consensus/tendermint/worker.rs +++ b/core/src/consensus/tendermint/worker.rs @@ -37,6 +37,7 @@ use super::params::TimeGapParams; use super::stake::CUSTOM_ACTION_HANDLER_ID; use super::types::{Height, Proposal, Step, TendermintSealView, TendermintState, TwoThirdsMajority, View}; use super::vote_collector::{DoubleVote, VoteCollector}; +use super::vote_regression_checker::VoteRegressionChecker; use super::{ BlockHash, ENGINE_TIMEOUT_BROADCAST_STEP_STATE, ENGINE_TIMEOUT_EMPTY_PROPOSAL, ENGINE_TIMEOUT_TOKEN_NONCE_BASE, SEAL_FIELDS, @@ -93,6 +94,7 @@ struct Worker { extension: EventSender, time_gap_params: TimeGapParams, timeout_token_nonce: usize, + vote_regression_checker: VoteRegressionChecker, } pub enum Event { @@ -196,6 +198,7 @@ impl Worker { votes_received_changed: false, time_gap_params, timeout_token_nonce: ENGINE_TIMEOUT_TOKEN_NONCE_BASE, + vote_regression_checker: VoteRegressionChecker::new(), } } @@ -1087,12 +1090,12 @@ impl Worker { } let header = sealed_block.header(); - let parent_hash = header.parent_hash(); if let TendermintState::ProposeWaitBlockGeneration { parent_hash: expected_parent_hash, } = self.step { + let parent_hash = header.parent_hash(); assert_eq!( *parent_hash, expected_parent_hash, "Generated hash({:?}) is different from expected({:?})", @@ -1498,6 +1501,8 @@ impl Worker { step: VoteStep::new(self.height, self.view, self.step.to_step()), block_hash, }; + assert!(self.vote_regression_checker.check(&on), "Vote should not regress"); + let signature = self.signer.sign(on.hash())?; let vote = ConsensusMessage { @@ -1523,6 +1528,8 @@ impl Worker { step: VoteStep::new(self.height, self.view, Step::Propose), block_hash: Some(header.hash()), }; + assert!(self.vote_regression_checker.check(&on), "Vote should not regress"); + let signature = self.signer.sign(on.hash())?; let vote = ConsensusMessage { From 03bfe3bbca5aadb8d8ae5a1a61683824e9abf11e Mon Sep 17 00:00:00 2001 From: SeongChan Lee Date: Thu, 26 Sep 2019 16:43:52 +0900 Subject: [PATCH 3/4] Rename VoteCollector::vote to collect --- core/src/consensus/tendermint/vote_collector.rs | 2 +- core/src/consensus/tendermint/worker.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/consensus/tendermint/vote_collector.rs b/core/src/consensus/tendermint/vote_collector.rs index ad5391ec8c..ea3cef1572 100644 --- a/core/src/consensus/tendermint/vote_collector.rs +++ b/core/src/consensus/tendermint/vote_collector.rs @@ -120,7 +120,7 @@ impl Default for VoteCollector { impl VoteCollector { /// Insert vote if it is newer than the oldest one. - pub fn vote(&mut self, message: ConsensusMessage) -> Option { + pub fn collect(&mut self, message: ConsensusMessage) -> Option { self.votes.entry(*message.round()).or_insert_with(Default::default).insert(message) } diff --git a/core/src/consensus/tendermint/worker.rs b/core/src/consensus/tendermint/worker.rs index 05473563f1..638db0c0fc 100644 --- a/core/src/consensus/tendermint/worker.rs +++ b/core/src/consensus/tendermint/worker.rs @@ -927,7 +927,7 @@ impl Worker { on: on.clone(), }; if !self.votes.is_old_or_known(&message) { - self.votes.vote(message); + self.votes.collect(message); } } @@ -1394,7 +1394,7 @@ impl Worker { self.votes_received.set(vote_index); } - if let Some(double) = self.votes.vote(message.clone()) { + if let Some(double) = self.votes.collect(message.clone()) { cerror!(ENGINE, "Double vote found {:?}", double); self.report_double_vote(&double); return Err(EngineError::DoubleVote(sender)) @@ -1512,7 +1512,7 @@ impl Worker { }; self.votes_received.set(vote.signer_index); - self.votes.vote(vote.clone()); + self.votes.collect(vote.clone()); cinfo!(ENGINE, "Voted {:?} as {}th validator.", vote, signer_index); Ok(Some(vote)) } @@ -1538,7 +1538,7 @@ impl Worker { on, }; - self.votes.vote(vote.clone()); + self.votes.collect(vote.clone()); cinfo!(ENGINE, "Voted {:?} as {}th proposer.", vote, signer_index); Ok(vote) } @@ -1790,7 +1790,7 @@ impl Worker { ); } - if let Some(double) = self.votes.vote(message.clone()) { + if let Some(double) = self.votes.collect(message.clone()) { cerror!(ENGINE, "Double Vote found {:?}", double); self.report_double_vote(&double); return None @@ -2125,7 +2125,7 @@ impl Worker { cdebug!(ENGINE, "Commit message-{} is verified", commit_height); for vote in votes { if !self.votes.is_old_or_known(&vote) { - self.votes.vote(vote); + self.votes.collect(vote); } } From 27cd4c507721dbc8cd6001ef36fe51ed84105c2d Mon Sep 17 00:00:00 2001 From: SeongChan Lee Date: Thu, 26 Sep 2019 17:27:12 +0900 Subject: [PATCH 4/4] Fix to use Err(DoubleVote) from VoteCollector VoteCollector is changed to return Result. Rust compiler will force you to check whether there was a double vote. --- .../consensus/tendermint/vote_collector.rs | 41 ++++++++++--------- core/src/consensus/tendermint/worker.rs | 16 +++++--- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/core/src/consensus/tendermint/vote_collector.rs b/core/src/consensus/tendermint/vote_collector.rs index ea3cef1572..d93fef2e86 100644 --- a/core/src/consensus/tendermint/vote_collector.rs +++ b/core/src/consensus/tendermint/vote_collector.rs @@ -61,26 +61,29 @@ impl Encodable for DoubleVote { } impl StepCollector { - /// Returns Some(&Address) when validator is double voting. - fn insert(&mut self, message: ConsensusMessage) -> Option { + /// Some(true): a message is new + /// Some(false): a message is duplicated + /// Err(DoubleVote): a double vote + fn insert(&mut self, message: ConsensusMessage) -> Result { // Do nothing when message was seen. - if !self.messages.contains(&message) { - self.messages.push(message.clone()); - if let Some(previous) = self.voted.insert(message.signer_index(), message.clone()) { - // Bad validator sent a different message. - return Some(DoubleVote { - author_index: message.signer_index(), - vote_one: previous, - vote_two: message, - }) - } else { - self.block_votes - .entry(message.block_hash()) - .or_default() - .insert(message.signer_index(), message.signature()); - } + if self.messages.contains(&message) { + return Ok(false) + } + self.messages.push(message.clone()); + if let Some(previous) = self.voted.insert(message.signer_index(), message.clone()) { + // Bad validator sent a different message. + Err(DoubleVote { + author_index: message.signer_index(), + vote_one: previous, + vote_two: message, + }) + } else { + self.block_votes + .entry(message.block_hash()) + .or_default() + .insert(message.signer_index(), message.signature()); + Ok(true) } - None } /// Count all votes for the given block hash at this round. @@ -120,7 +123,7 @@ impl Default for VoteCollector { impl VoteCollector { /// Insert vote if it is newer than the oldest one. - pub fn collect(&mut self, message: ConsensusMessage) -> Option { + pub fn collect(&mut self, message: ConsensusMessage) -> Result { self.votes.entry(*message.round()).or_insert_with(Default::default).insert(message) } diff --git a/core/src/consensus/tendermint/worker.rs b/core/src/consensus/tendermint/worker.rs index 638db0c0fc..3565cf6e96 100644 --- a/core/src/consensus/tendermint/worker.rs +++ b/core/src/consensus/tendermint/worker.rs @@ -927,7 +927,9 @@ impl Worker { on: on.clone(), }; if !self.votes.is_old_or_known(&message) { - self.votes.collect(message); + if let Err(double_vote) = self.votes.collect(message) { + cerror!(ENGINE, "Double vote found on_commit_message: {:?}", double_vote); + } } } @@ -1394,7 +1396,7 @@ impl Worker { self.votes_received.set(vote_index); } - if let Some(double) = self.votes.collect(message.clone()) { + if let Err(double) = self.votes.collect(message.clone()) { cerror!(ENGINE, "Double vote found {:?}", double); self.report_double_vote(&double); return Err(EngineError::DoubleVote(sender)) @@ -1512,7 +1514,7 @@ impl Worker { }; self.votes_received.set(vote.signer_index); - self.votes.collect(vote.clone()); + self.votes.collect(vote.clone()).expect("Must not attempt double vote"); cinfo!(ENGINE, "Voted {:?} as {}th validator.", vote, signer_index); Ok(Some(vote)) } @@ -1538,7 +1540,7 @@ impl Worker { on, }; - self.votes.collect(vote.clone()); + self.votes.collect(vote.clone()).expect("Must not attempt double vote on proposal");; cinfo!(ENGINE, "Voted {:?} as {}th proposer.", vote, signer_index); Ok(vote) } @@ -1790,7 +1792,7 @@ impl Worker { ); } - if let Some(double) = self.votes.collect(message.clone()) { + if let Err(double) = self.votes.collect(message.clone()) { cerror!(ENGINE, "Double Vote found {:?}", double); self.report_double_vote(&double); return None @@ -2125,7 +2127,9 @@ impl Worker { cdebug!(ENGINE, "Commit message-{} is verified", commit_height); for vote in votes { if !self.votes.is_old_or_known(&vote) { - self.votes.collect(vote); + if let Err(double_vote) = self.votes.collect(vote) { + cerror!(ENGINE, "Double vote found on_commit_message: {:?}", double_vote); + } } }