Merge pull request #1250 from sechkova/fetcher

Abstract out the network IO

Author: Jussi Kukkonen

tests/test_arbitrary_package_attack.py

@@ -124,7 +124,7 @@ def setUp(self):
     # Set the url prefix required by the 'tuf/client/updater.py' updater.
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
-    url_prefix = 'http://localhost:' \
+    url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + repository_basepath
 
     # Setting 'tuf.settings.repository_directory' with the temporary client

tests/test_download.py

@@ -43,6 +43,7 @@
 
 import tuf
 import tuf.download as download
+import tuf.requests_fetcher
 import tuf.log
 import tuf.unittest_toolbox as unittest_toolbox
 import tuf.exceptions
@@ -76,7 +77,7 @@ def setUp(self):
     self.server_process_handler = utils.TestServerProcess(log=logger)
 
     rel_target_filepath = os.path.basename(target_filepath)
-    self.url = 'http://localhost:' \
+    self.url = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + '/' + rel_target_filepath
 
     # Computing hash of target file data.
@@ -85,6 +86,10 @@ def setUp(self):
     digest = m.hexdigest()
     self.target_hash = {'md5':digest}
 
+    # Initialize the default fetcher for the download
+    self.fetcher = tuf.requests_fetcher.RequestsFetcher()
+
+
 
   # Stop server process and perform clean up.
   def tearDown(self):
@@ -100,13 +105,36 @@ def tearDown(self):
   def test_download_url_to_tempfileobj(self):
 
     download_file = download.safe_download
-    with download_file(self.url, self.target_data_length) as temp_fileobj:
+    with download_file(self.url, self.target_data_length, self.fetcher) as temp_fileobj:
       temp_fileobj.seek(0)
       temp_file_data = temp_fileobj.read().decode('utf-8')
       self.assertEqual(self.target_data, temp_file_data)
       self.assertEqual(self.target_data_length, len(temp_file_data))
 
 
+  # Test: Download url in more than one chunk.
+  def test_download_url_in_chunks(self):
+
+    # Set smaller chunk size to ensure that the file will be downloaded
+    # in more than one chunk
+    default_chunk_size = tuf.settings.CHUNK_SIZE
+    tuf.settings.CHUNK_SIZE = 4
+    # We don't have access to chunks from  download_file()
+    # so we just confirm that the expectation of more than one chunk is
+    # correct and verify that no errors are raised during download
+    chunks_count = self.target_data_length/tuf.settings.CHUNK_SIZE
+    self.assertGreater(chunks_count, 1)
+
+    download_file = download.safe_download
+    with download_file(self.url, self.target_data_length, self.fetcher) as temp_fileobj:
+      temp_fileobj.seek(0)
+      temp_file_data = temp_fileobj.read().decode('utf-8')
+      self.assertEqual(self.target_data, temp_file_data)
+      self.assertEqual(self.target_data_length, len(temp_file_data))
+
+    # Restore default settings
+    tuf.settings.CHUNK_SIZE = default_chunk_size
+
 
   # Test: Incorrect lengths.
   def test_download_url_to_tempfileobj_and_lengths(self):
@@ -118,18 +146,18 @@ def test_download_url_to_tempfileobj_and_lengths(self):
     # the server-reported length of the file does not match the
     # required_length.  'updater.py' *does* verify the hashes of downloaded
     # content.
-    download.safe_download(self.url, self.target_data_length - 4).close()
-    download.unsafe_download(self.url, self.target_data_length - 4).close()
+    download.safe_download(self.url, self.target_data_length - 4, self.fetcher).close()
+    download.unsafe_download(self.url, self.target_data_length - 4, self.fetcher).close()
 
     # We catch 'tuf.exceptions.DownloadLengthMismatchError' for safe_download()
     # because it will not download more bytes than requested (in this case, a
     # length greater than the size of the target file).
     self.assertRaises(tuf.exceptions.DownloadLengthMismatchError,
-        download.safe_download, self.url, self.target_data_length + 1)
+        download.safe_download, self.url, self.target_data_length + 1, self.fetcher)
 
     # Calling unsafe_download() with a mismatched length should not raise an
     # exception.
-    download.unsafe_download(self.url, self.target_data_length + 1).close()
+    download.unsafe_download(self.url, self.target_data_length + 1, self.fetcher).close()
 
 
 
@@ -164,32 +192,26 @@ def test_download_url_to_tempfileobj_and_urls(self):
     download_file = download.safe_download
     unsafe_download_file = download.unsafe_download
 
-    self.assertRaises(securesystemslib.exceptions.FormatError,
-                      download_file, None, self.target_data_length)
-
-    self.assertRaises(tuf.exceptions.URLParsingError,
-                      download_file,
-                      self.random_string(), self.target_data_length)
+    with self.assertRaises(securesystemslib.exceptions.FormatError):
+      download_file(None, self.target_data_length, self.fetcher)
 
-    url = 'http://localhost:' \
+    url = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + '/' + self.random_string()
-    self.assertRaises(requests.exceptions.HTTPError,
-                      download_file,
-                      url,
-                      self.target_data_length)
-    url1 = 'http://localhost:' \
+    with self.assertRaises(tuf.exceptions.FetcherHTTPError) as cm:
+      download_file(url, self.target_data_length, self.fetcher)
+    self.assertEqual(cm.exception.status_code, 404)
+
+    url1 = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
       + str(self.server_process_handler.port + 1) + '/' + self.random_string()
-    self.assertRaises(requests.exceptions.ConnectionError,
-                      download_file,
-                      url1,
-                      self.target_data_length)
+    with self.assertRaises(requests.exceptions.ConnectionError):
+      download_file(url1, self.target_data_length, self.fetcher)
 
     # Specify an unsupported URI scheme.
     url_with_unsupported_uri = self.url.replace('http', 'file')
     self.assertRaises(requests.exceptions.InvalidSchema, download_file, url_with_unsupported_uri,
-                      self.target_data_length)
+                      self.target_data_length, self.fetcher)
     self.assertRaises(requests.exceptions.InvalidSchema, unsafe_download_file,
-                      url_with_unsupported_uri, self.target_data_length)
+                      url_with_unsupported_uri, self.target_data_length, self.fetcher)
 
 
 
@@ -290,7 +312,7 @@ def test_https_connection(self):
       os.environ['REQUESTS_CA_BUNDLE'] = bad_cert_fname
       # Clear sessions to ensure that the certificate we just specified is used.
       # TODO: Confirm necessity of this session clearing and lay out mechanics.
-      tuf.download._sessions = {}
+      self.fetcher._sessions = {}
 
       # Try connecting to the server process with the bad cert while trusting
       # the bad cert. Expect failure because even though we trust it, the
@@ -302,41 +324,41 @@ def test_https_connection(self):
             category=urllib3.exceptions.SubjectAltNameWarning)
 
         with self.assertRaises(requests.exceptions.SSLError):
-          download.safe_download(bad_https_url, target_data_length)
+          download.safe_download(bad_https_url, target_data_length, self.fetcher)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.unsafe_download(bad_https_url, target_data_length)
+          download.unsafe_download(bad_https_url, target_data_length, self.fetcher)
 
         # Try connecting to the server processes with the good certs while not
         # trusting the good certs (trusting the bad cert instead). Expect failure
         # because even though the server's cert file is otherwise OK, we don't
         # trust it.
         logger.info('Trying HTTPS download of target file: ' + good_https_url)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.safe_download(good_https_url, target_data_length)
+          download.safe_download(good_https_url, target_data_length, self.fetcher)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.unsafe_download(good_https_url, target_data_length)
+          download.unsafe_download(good_https_url, target_data_length, self.fetcher)
 
         logger.info('Trying HTTPS download of target file: ' + good2_https_url)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.safe_download(good2_https_url, target_data_length)
+          download.safe_download(good2_https_url, target_data_length, self.fetcher)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.unsafe_download(good2_https_url, target_data_length)
+          download.unsafe_download(good2_https_url, target_data_length, self.fetcher)
 
 
         # Configure environment to now trust the certfile that is expired.
         os.environ['REQUESTS_CA_BUNDLE'] = expired_cert_fname
         # Clear sessions to ensure that the certificate we just specified is used.
         # TODO: Confirm necessity of this session clearing and lay out mechanics.
-        tuf.download._sessions = {}
+        self.fetcher._sessions = {}
 
         # Try connecting to the server process with the expired cert while
         # trusting the expired cert. Expect failure because even though we trust
         # it, it is expired.
         logger.info('Trying HTTPS download of target file: ' + expired_https_url)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.safe_download(expired_https_url, target_data_length)
+          download.safe_download(expired_https_url, target_data_length, self.fetcher)
         with self.assertRaises(requests.exceptions.SSLError):
-          download.unsafe_download(expired_https_url, target_data_length)
+          download.unsafe_download(expired_https_url, target_data_length, self.fetcher)
 
 
         # Try connecting to the server processes with the good certs while
@@ -346,18 +368,18 @@ def test_https_connection(self):
         os.environ['REQUESTS_CA_BUNDLE'] = good_cert_fname
         # Clear sessions to ensure that the certificate we just specified is used.
         # TODO: Confirm necessity of this session clearing and lay out mechanics.
-        tuf.download._sessions = {}
+        self.fetcher._sessions = {}
         logger.info('Trying HTTPS download of target file: ' + good_https_url)
-        download.safe_download(good_https_url, target_data_length).close()
-        download.unsafe_download(good_https_url, target_data_length).close()
+        download.safe_download(good_https_url, target_data_length, self.fetcher).close()
+        download.unsafe_download(good_https_url, target_data_length,self.fetcher).close()
 
         os.environ['REQUESTS_CA_BUNDLE'] = good2_cert_fname
         # Clear sessions to ensure that the certificate we just specified is used.
         # TODO: Confirm necessity of this session clearing and lay out mechanics.
-        tuf.download._sessions = {}
+        self.fetcher._sessions = {}
         logger.info('Trying HTTPS download of target file: ' + good2_https_url)
-        download.safe_download(good2_https_url, target_data_length).close()
-        download.unsafe_download(good2_https_url, target_data_length).close()
+        download.safe_download(good2_https_url, target_data_length, self.fetcher).close()
+        download.unsafe_download(good2_https_url, target_data_length, self.fetcher).close()
 
     finally:
       for proc_handler in [

tests/test_endless_data_attack.py

@@ -126,7 +126,7 @@ def setUp(self):
     # Set the url prefix required by the 'tuf/client/updater.py' updater.
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
-    url_prefix = 'http://localhost:' \
+    url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + repository_basepath
 
     # Setting 'tuf.settings.repository_directory' with the temporary client

tests/test_extraneous_dependencies_attack.py

@@ -133,7 +133,7 @@ def setUp(self):
     # Set the url prefix required by the 'tuf/client/updater.py' updater.
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
-    url_prefix = 'http://localhost:' \
+    url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + repository_basepath
 
     # Setting 'tuf.settings.repository_directory' with the temporary client

tests/test_fetcher.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+# Copyright 2021, New York University and the TUF contributors
+# SPDX-License-Identifier: MIT OR Apache-2.0
+
+"""Unit test for RequestsFetcher.
+"""
+# Help with Python 2 compatibility, where the '/' operator performs
+# integer division.
+from __future__ import division
+
+import logging
+import os
+import io
+import sys
+import unittest
+import tempfile
+import math
+
+import tuf
+import tuf.exceptions
+import tuf.requests_fetcher
+import tuf.unittest_toolbox as unittest_toolbox
+
+from tests import utils
+
+logger = logging.getLogger(__name__)
+
+
+class TestFetcher(unittest_toolbox.Modified_TestCase):
+  def setUp(self):
+    """
+    Create a temporary file and launch a simple server in the
+    current working directory.
+    """
+
+    unittest_toolbox.Modified_TestCase.setUp(self)
+
+    # Making a temporary file.
+    current_dir = os.getcwd()
+    target_filepath = self.make_temp_data_file(directory=current_dir)
+    self.target_fileobj = open(target_filepath, 'r')
+    self.file_contents = self.target_fileobj.read()
+    self.file_length = len(self.file_contents)
+
+    # Launch a SimpleHTTPServer (serves files in the current dir).
+    self.server_process_handler = utils.TestServerProcess(log=logger)
+
+    rel_target_filepath = os.path.basename(target_filepath)
+    self.url = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
+        + str(self.server_process_handler.port) + '/' + rel_target_filepath
+
+    # Create a temporary file where the target file chunks are written
+    # during fetching
+    self.temp_file = tempfile.TemporaryFile()
+    self.fetcher = tuf.requests_fetcher.RequestsFetcher()
+
+
+  # Stop server process and perform clean up.
+  def tearDown(self):
+    unittest_toolbox.Modified_TestCase.tearDown(self)
+
+    # Cleans the resources and flush the logged lines (if any).
+    self.server_process_handler.clean()
+
+    self.target_fileobj.close()
+    self.temp_file.close()
+
+
+  # Test: Normal case.
+  def test_fetch(self):
+    for chunk in self.fetcher.fetch(self.url, self.file_length):
+        self.temp_file.write(chunk)
+
+    self.temp_file.seek(0)
+    temp_file_data = self.temp_file.read().decode('utf-8')
+    self.assertEqual(self.file_contents, temp_file_data)
+
+  # Test if fetcher downloads file up to a required length
+  def test_fetch_restricted_length(self):
+    for chunk in self.fetcher.fetch(self.url, self.file_length-4):
+        self.temp_file.write(chunk)
+
+    self.temp_file.seek(0, io.SEEK_END)
+    self.assertEqual(self.temp_file.tell(), self.file_length-4)
+
+
+  # Test that fetcher does not download more than actual file length
+  def test_fetch_upper_length(self):
+    for chunk in self.fetcher.fetch(self.url, self.file_length+4):
+        self.temp_file.write(chunk)
+
+    self.temp_file.seek(0, io.SEEK_END)
+    self.assertEqual(self.temp_file.tell(), self.file_length)
+
+
+  # Test incorrect URL parsing
+  def test_url_parsing(self):
+    with self.assertRaises(tuf.exceptions.URLParsingError):
+      self.fetcher.fetch(self.random_string(), self.file_length)
+
+
+  # Test: Normal case with url data downloaded in more than one chunk
+  def test_fetch_in_chunks(self):
+    # Set smaller chunk size to ensure that the file will be downloaded
+    # in more than one chunk
+    default_chunk_size = tuf.settings.CHUNK_SIZE
+    tuf.settings.CHUNK_SIZE = 4
+
+    # expected_chunks_count: 3
+    expected_chunks_count = math.ceil(self.file_length/tuf.settings.CHUNK_SIZE)
+    self.assertEqual(expected_chunks_count, 3)
+
+    chunks_count = 0
+    for chunk in self.fetcher.fetch(self.url, self.file_length):
+      self.temp_file.write(chunk)
+      chunks_count+=1
+
+    self.temp_file.seek(0)
+    temp_file_data = self.temp_file.read().decode('utf-8')
+    self.assertEqual(self.file_contents, temp_file_data)
+    # Check that we calculate chunks as expected
+    self.assertEqual(chunks_count, expected_chunks_count)
+
+    # Restore default settings
+    tuf.settings.CHUNK_SIZE = default_chunk_size
+
+
+
+# Run unit test.
+if __name__ == '__main__':
+  utils.configure_test_logging(sys.argv)
+  unittest.main()

tests/test_indefinite_freeze_attack.py

@@ -146,7 +146,7 @@ def setUp(self):
     # Set the url prefix required by the 'tuf/client/updater.py' updater.
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
-    url_prefix = 'http://localhost:' \
+    url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
         + str(self.server_process_handler.port) + repository_basepath
 
     # Setting 'tuf.settings.repository_directory' with the temporary client

tests/test_key_revocation_integration.py

@@ -132,7 +132,7 @@ def setUp(self):
 
     # 'path/to/tmp/repository' -> 'localhost:8001/tmp/repository'.
     repository_basepath = self.repository_directory[len(os.getcwd()):]
-    url_prefix = 'http://localhost:' \
+    url_prefix = 'http://' + utils.TEST_HOST_ADDRESS + ':' \
       + str(self.server_process_handler.port) + repository_basepath
 
     # Setting 'tuf.settings.repository_directory' with the temporary client