chore: address various coverity issues (#301)

**Description:**

This PR addresses various coverity findings.

**Changes Made:**

* don't send `x-powered-by` header on media server
* re-enable certificate verification in downloader
* switch from md5 to sha256 for file caching

**Testing Done:**

Tested locally on B580

**Checklist:**

- [x] I have tested the changes locally.
- [x] I have self-reviewed the code changes.

Author: mschuettlerTNG

WebUI/electron/subprocesses/mediaServer.ts

@@ -6,6 +6,7 @@ const port: number = Number(process.env.PORT_NUMBER)
 const mediaPath: string = process.env.MEDIA_DIRECTORY!
 
 app.use(express.static(mediaPath))
+app.disable('x-powered-by')
 
 app.listen(port, () => {
   console.log(`Media server started on port ${port}`, 'electron-backend')

backend-shared/utils.py

@@ -135,35 +135,28 @@ def check_defaultbackend_mmodel_exist(type: int, repo_id: str) -> bool:
         )
 
 # File operations
-def calculate_md5(file_path: str):
-    """Calculate the MD5 hash of a file"""
+def calculate_sha256(file_path: str):
+    """Calculate the SHA256 hash of a file"""
     with open(file_path, "rb") as f:
         file_hash = hashlib.md5()
         while chunk := f.read(8192):
             file_hash.update(chunk)
     return file_hash.hexdigest()
 
-def calculate_md5_from_stream(file_stream: IO[bytes]):
-    """Calculate the MD5 hash of a file stream"""
-    file_hash = hashlib.md5()
-    for chunk in iter(lambda: file_stream.read(8192), b""):
-        file_hash.update(chunk)
-    return file_hash.hexdigest()
-
-def create_cache_path(md5: str, file_size: int):
-    """Create a cache path for a file based on its MD5 hash and size"""
+def create_cache_path(sha256: str, file_size: int):
+    """Create a cache path for a file based on its SHA256 hash and size"""
     cache_dir = "./cache"
-    sub_dirs = [md5[i : i + 4] for i in range(0, len(md5), 4)]
+    sub_dirs = [sha256[i : i + 4] for i in range(0, len(sha256), 4)]
     cache_path = os.path.abspath(
-        os.path.join(cache_dir, *sub_dirs, f"{md5}_{file_size}")
+        os.path.join(cache_dir, *sub_dirs, f"{sha256}_{file_size}")
     )
     return cache_path
 
 def cache_file(file_path: IO[bytes] | str, file_size: int):
-    """Cache a file using its MD5 hash and size"""
-    md5 = calculate_md5(file_path)
+    """Cache a file using its SHA256 hash and size"""
+    sha256 = calculate_sha256(file_path)
 
-    cache_path = create_cache_path(md5, file_size)
+    cache_path = create_cache_path(sha256, file_size)
 
     if not os.path.exists(cache_path):
         os.makedirs(os.path.dirname(cache_path), exist_ok=True)

service/aipg_utils.py

@@ -24,9 +24,7 @@
 repo_local_root_dir_name = utils.repo_local_root_dir_name
 flat_repo_local_dir_name = utils.flat_repo_local_dir_name
 get_model_path = utils.get_model_path
-calculate_md5 = utils.calculate_md5
 create_cache_path = utils.create_cache_path
-calculate_md5_from_stream = utils.calculate_md5_from_stream
 cache_file = utils.cache_file
 is_single_file = utils.is_single_file
 remove_existing_filesystem_resource = utils.remove_existing_filesystem_resource

service/file_downloader.py

@@ -1,127 +1,126 @@
-from io import BufferedWriter
-import os
-import time
-import traceback
-from typing import Callable
-import requests
-from threading import Thread
-from exceptions import DownloadException
-
-
-class FileDownloader:
-    on_download_progress: Callable[[str, int, int, int], None] = None
-    on_download_completed: Callable[[str, Exception], None] = None
-    url: str
-    filename: str
-    basename: str
-    total_size: int
-    download_size: int
-    download_stop: bool
-    prev_sec_download_size: int
-
-    def __init__(self):
-        self.download_stop = False
-        self.download_size = 0
-        self.completed = False
-        self.total_size = 0
-        self.prev_sec_download_size = 0
-        self.report_thread = None
-
-    def download_file(self, url: str, file_path: str):
-        self.url = url
-        self.basename = os.path.basename(file_path)
-        self.download_stop = False
-        self.filename = file_path
-        self.prev_sec_download_size = 0
-        self.download_size = 0
-        self.completed = False
-        self.report_thread = None
-        error = None
-        report_thread = None
-        try:
-            response, fw = self.__init_download(self.url, self.filename)
-            self.total_size = int(response.headers.get("Content-Length"))
-            if self.on_download_progress is not None:
-                report_thread = self.__start_report_download_progress()
-            self.__start_download(response, fw)
-        except Exception as e:
-            error = e
-        finally:
-            self.completed = True
-            if report_thread is not None:
-                report_thread.join()
-
-        if self.on_download_completed is not None:
-            self.on_download_completed(self.basename, error)
-
-    def __init_download(
-        self, url: str, file_path: str
-    ) -> tuple[requests.Response, BufferedWriter]:
-        if os.path.exists(file_path):
-            start_pos = os.path.getsize(file_path)
-        else:
-            os.makedirs(os.path.dirname(file_path), exist_ok=True)
-            start_pos = 0
-
-        if start_pos > 0:
-            # download skip exists part
-            response = requests.get(
-                url,
-                stream=True,
-                verify=False,
-                headers={"Range": f"bytes={start_pos}-"},
-            )
-            fw = open(file_path, "ab")
-        else:
-            response = requests.get(url, stream=True, verify=False)
-            fw = open(file_path, "wb")
-
-        return response, fw
-
-    def __start_download(self, response: requests.Response, fw: BufferedWriter):
-        retry = 0
-        while True:
-            try:
-                with response:
-                    with fw:
-                        for bytes in response.iter_content(chunk_size=4096):
-                            self.download_size += bytes.__len__()
-                            fw.write(bytes)
-
-                            if self.download_stop:
-                                print(
-                                    f"FileDownloader thread {Thread.native_id} exit by stop"
-                                )
-                                break
-                break
-            except Exception:
-                traceback.print_exc()
-                retry += 1
-                if retry > 3:
-                    raise DownloadException(self.url)
-                else:
-                    print(
-                        f"FileDownloader thread {Thread.native_id} retry {retry} times"
-                    )
-                    time.sleep(1)
-                    response, fw = self.__init_download(self.url, self.filename)
-
-    def __start_report_download_progress(self):
-        report_thread = Thread(target=self.__report_download_progress)
-        report_thread.start()
-        return report_thread
-
-    def __report_download_progress(self):
-        while not self.download_stop and not self.completed:
-            self.on_download_progress(
-                self.basename,
-                self.download_size,
-                self.total_size,
-                self.download_size - self.prev_sec_download_size,
-            )
-
-            self.prev_sec_download_size = self.download_size
-            time.sleep(1)
-
-    def stop_download(self):
-        self.download_stop = True
+from io import BufferedWriter
+import os
+import time
+import traceback
+from typing import Callable
+import requests
+from threading import Thread
+from exceptions import DownloadException
+
+
+class FileDownloader:
+    on_download_progress: Callable[[str, int, int, int], None] = None
+    on_download_completed: Callable[[str, Exception], None] = None
+    url: str
+    filename: str
+    basename: str
+    total_size: int
+    download_size: int
+    download_stop: bool
+    prev_sec_download_size: int
+
+    def __init__(self):
+        self.download_stop = False
+        self.download_size = 0
+        self.completed = False
+        self.total_size = 0
+        self.prev_sec_download_size = 0
+        self.report_thread = None
+
+    def download_file(self, url: str, file_path: str):
+        self.url = url
+        self.basename = os.path.basename(file_path)
+        self.download_stop = False
+        self.filename = file_path
+        self.prev_sec_download_size = 0
+        self.download_size = 0
+        self.completed = False
+        self.report_thread = None
+        error = None
+        report_thread = None
+        try:
+            response, fw = self.__init_download(self.url, self.filename)
+            self.total_size = int(response.headers.get("Content-Length"))
+            if self.on_download_progress is not None:
+                report_thread = self.__start_report_download_progress()
+            self.__start_download(response, fw)
+        except Exception as e:
+            error = e
+        finally:
+            self.completed = True
+            if report_thread is not None:
+                report_thread.join()
+
+        if self.on_download_completed is not None:
+            self.on_download_completed(self.basename, error)
+
+    def __init_download(
+        self, url: str, file_path: str
+    ) -> tuple[requests.Response, BufferedWriter]:
+        if os.path.exists(file_path):
+            start_pos = os.path.getsize(file_path)
+        else:
+            os.makedirs(os.path.dirname(file_path), exist_ok=True)
+            start_pos = 0
+
+        if start_pos > 0:
+            # download skip exists part
+            response = requests.get(
+                url,
+                stream=True,
+                headers={"Range": f"bytes={start_pos}-"},
+            )
+            fw = open(file_path, "ab")
+        else:
+            response = requests.get(url, stream=True)
+            fw = open(file_path, "wb")
+
+        return response, fw
+
+    def __start_download(self, response: requests.Response, fw: BufferedWriter):
+        retry = 0
+        while True:
+            try:
+                with response:
+                    with fw:
+                        for bytes in response.iter_content(chunk_size=4096):
+                            self.download_size += bytes.__len__()
+                            fw.write(bytes)
+
+                            if self.download_stop:
+                                print(
+                                    f"FileDownloader thread {Thread.native_id} exit by stop"
+                                )
+                                break
+                break
+            except Exception:
+                traceback.print_exc()
+                retry += 1
+                if retry > 3:
+                    raise DownloadException(self.url)
+                else:
+                    print(
+                        f"FileDownloader thread {Thread.native_id} retry {retry} times"
+                    )
+                    time.sleep(1)
+                    response, fw = self.__init_download(self.url, self.filename)
+
+    def __start_report_download_progress(self):
+        report_thread = Thread(target=self.__report_download_progress)
+        report_thread.start()
+        return report_thread
+
+    def __report_download_progress(self):
+        while not self.download_stop and not self.completed:
+            self.on_download_progress(
+                self.basename,
+                self.download_size,
+                self.total_size,
+                self.download_size - self.prev_sec_download_size,
+            )
+
+            self.prev_sec_download_size = self.download_size
+            time.sleep(1)
+
+    def stop_download(self):
+        self.download_stop = True

service/model_downloader.py

@@ -330,13 +330,12 @@ def init_download(self, file: HFDonloadItem):
             response = requests.get(
                 file.url,
                 stream=True,
-                verify=False,
                 headers=headers,
             )
             fw = open(file.save_filename, "ab")
         else:
             response = requests.get(
-                file.url, stream=True, verify=False, headers=headers
+                file.url, stream=True, headers=headers
             )
             fw = open(file.save_filename, "wb")
 
@@ -361,7 +360,7 @@ def is_access_granted(self, repo_id: str, model_type, backend : str):
         self.build_queue(file_list)
         file = self.file_queue.get_nowait()
 
-        response = requests.head(file.url, verify=False, headers=headers, allow_redirects=True)
+        response = requests.head(file.url, headers=headers, allow_redirects=True)
 
         return response.status_code == 200