From 29431defe7282f70f66b035c3ecca26585bc2de5 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Fri, 10 Jan 2025 15:31:10 +0800 Subject: [PATCH 1/2] fix Signed-off-by: youkaichao --- vllm/model_executor/model_loader/weight_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 11d5fd7135d9..7ccf6b934d83 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -29,7 +29,7 @@ try: from runai_model_streamer import SafetensorsStreamer -except ImportError: +except (ImportError, OSError): runai_model_streamer = PlaceholderModule( "runai_model_streamer") # type: ignore[assignment] SafetensorsStreamer = runai_model_streamer.placeholder_attr( From 7a0f735aa14f40afcb4bb9d4546eb770987df5f6 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Fri, 10 Jan 2025 16:07:56 +0800 Subject: [PATCH 2/2] add comments Signed-off-by: youkaichao --- vllm/model_executor/model_loader/weight_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 7ccf6b934d83..9cfcdbf620d2 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -30,6 +30,8 @@ try: from runai_model_streamer import SafetensorsStreamer except (ImportError, OSError): + # see https://github.com/run-ai/runai-model-streamer/issues/26 + # OSError will be raised on arm64 platform runai_model_streamer = PlaceholderModule( "runai_model_streamer") # type: ignore[assignment] SafetensorsStreamer = runai_model_streamer.placeholder_attr(