cleans up config building

Author: JyotinderSingh

keras_hub/src/models/backbone.py

@@ -91,21 +91,24 @@ def get_config(self):
         }
 
         # Add quantization support by utilizing `DTypePolicyMap`
+        dtype = None
         try:
             if isinstance(
                 self.dtype_policy, keras.dtype_policies.DTypePolicyMap
             ):
-                config.update({"dtype": self.dtype_policy})
+                dtype=  self.dtype_policy
             else:
                 policy_map = keras.dtype_policies.DTypePolicyMap()
                 for layer in self._flatten_layers():
                     if layer.quantization_mode is not None:
                         policy_map[layer.path] = layer.dtype_policy
                 if len(policy_map) > 0:
-                    config.update({"dtype": policy_map})
+                    dtype = policy_map
         # Before Keras 3.2, there is no `keras.dtype_policies.get`.
         except AttributeError:
-            config.update({"dtype": None})
+            pass
+
+        config.update({"dtype": dtype})
         return config
 
     @classmethod