Allow setting idle_timeout for server connections.

In postgres, you can specify an `idle_session_timeout` which will close
sessions idling for that amount of time. If a session is closed because
of a timeout, PgCat will erroneously mark the server as unhealthy as the next
health check will return an error because the connection was drop, if no
health check is to be executed, it will simply fail trying to send the query
to the server for the same reason, the conn was drop.

Given that bb8 allows configuring an idle_timeout for pools, it would be
nice to allow setting this parameter in the config file, this way you can
set it to something shorter than the server one. Also, server pool will be kept
smaller in moments of less traffic. Actually, currently this value is set as its
default in bb8, which is 10 minutes.

This changes allows setting the parameter using the config file. It can be set both
globally and per pool. When creating the pool, if the pool don't have it defined, global
value is used.

Author: magec

pgcat.toml

@@ -20,6 +20,9 @@ prometheus_exporter_port = 9930
 # How long to wait before aborting a server connection (ms).
 connect_timeout = 5000
 
+# How long an idle connection with a server is left open (ms).
+idle_timeout = 30000
+
 # How much time to give the health check query to return with a result (ms).
 healthcheck_timeout = 1000
 
@@ -92,6 +95,9 @@ sharding_function = "pg_bigint_hash"
 # Automatically parse this from queries and route queries to the right shard!
 automatic_sharding_key = "id"
 
+# Idle timeout can be overwritten in the pool
+idle_timeout = 40000
+
 # Credentials for users that may connect to this cluster
 [pools.sharded_db.users.0]
 username = "sharding_user"

src/config.rs

@@ -157,6 +157,9 @@ pub struct General {
     #[serde(default = "General::default_connect_timeout")]
     pub connect_timeout: u64,
 
+    #[serde(default = "General::default_idle_timeout")]
+    pub idle_timeout: u64,
+
     #[serde(default)] // False
     pub log_client_connections: bool,
 
@@ -197,6 +200,10 @@ impl General {
         1000
     }
 
+    pub fn default_idle_timeout() -> u64 {
+        60000 // 10 minutes
+    }
+
     pub fn default_shutdown_timeout() -> u64 {
         60000
     }
@@ -222,6 +229,7 @@ impl Default for General {
             enable_prometheus_exporter: Some(false),
             prometheus_exporter_port: 9930,
             connect_timeout: General::default_connect_timeout(),
+            idle_timeout: General::default_idle_timeout(),
             shutdown_timeout: Self::default_shutdown_timeout(),
             healthcheck_timeout: Self::default_healthcheck_timeout(),
             healthcheck_delay: Self::default_healthcheck_delay(),
@@ -273,6 +281,8 @@ pub struct Pool {
 
     pub connect_timeout: Option<u64>,
 
+    pub idle_timeout: Option<u64>,
+
     pub sharding_function: ShardingFunction,
 
     #[serde(default = "Pool::default_automatic_sharding_key")]
@@ -335,6 +345,7 @@ impl Default for Pool {
             sharding_function: ShardingFunction::PgBigintHash,
             automatic_sharding_key: None,
             connect_timeout: None,
+            idle_timeout: None,
         }
     }
 }
@@ -496,6 +507,10 @@ impl From<&Config> for std::collections::HashMap<String, String> {
                 "connect_timeout".to_string(),
                 config.general.connect_timeout.to_string(),
             ),
+            (
+                "idle_timeout".to_string(),
+                config.general.idle_timeout.to_string(),
+            ),
             (
                 "healthcheck_timeout".to_string(),
                 config.general.healthcheck_timeout.to_string(),
@@ -525,6 +540,7 @@ impl Config {
             self.general.healthcheck_timeout
         );
         info!("Connection timeout: {}ms", self.general.connect_timeout);
+        info!("Idle timeout: {}ms", self.general.idle_timeout);
         info!(
             "Log client connections: {}",
             self.general.log_client_connections
@@ -578,6 +594,11 @@ impl Config {
                 "[pool: {}] Connection timeout: {}ms",
                 pool_name, connect_timeout
             );
+            let idle_timeout = match pool_config.idle_timeout {
+                Some(idle_timeout) => idle_timeout,
+                None => self.general.idle_timeout,
+            };
+            info!("[pool: {}] Idle timeout: {}ms", pool_name, idle_timeout);
             info!(
                 "[pool: {}] Sharding function: {}",
                 pool_name,
@@ -732,8 +753,10 @@ mod test {
         assert_eq!(get_config().path, "pgcat.toml".to_string());
 
         assert_eq!(get_config().general.ban_time, 60);
+        assert_eq!(get_config().general.idle_timeout, 30000);
         assert_eq!(get_config().pools.len(), 2);
         assert_eq!(get_config().pools["sharded_db"].shards.len(), 3);
+        assert_eq!(get_config().pools["sharded_db"].idle_timeout, Some(40000));
         assert_eq!(get_config().pools["simple_db"].shards.len(), 1);
         assert_eq!(get_config().pools["sharded_db"].users.len(), 2);
         assert_eq!(get_config().pools["simple_db"].users.len(), 1);

src/pool.rs

@@ -224,9 +224,15 @@ impl ConnectionPool {
                             None => config.general.connect_timeout,
                         };
 
+                        let idle_timeout = match pool_config.idle_timeout {
+                            Some(idle_timeout) => idle_timeout,
+                            None => config.general.idle_timeout,
+                        };
+
                         let pool = Pool::builder()
                             .max_size(user.pool_size)
                             .connection_timeout(std::time::Duration::from_millis(connect_timeout))
+                            .idle_timeout(Some(std::time::Duration::from_millis(idle_timeout)))
                             .test_on_check_out(false)
                             .build(manager)
                             .await