|
| 1 | +from collections import defaultdict |
| 2 | +from ctypes import cdll |
| 3 | +from ctypes.util import find_library |
| 4 | +from multiprocessing.managers import BaseManager |
| 5 | +from pathlib import Path |
| 6 | +import platform |
| 7 | +import shutil |
| 8 | +import subprocess |
| 9 | +import tempfile |
| 10 | +import uuid |
| 11 | + |
| 12 | + |
| 13 | +def load_library(filename): |
| 14 | + """Loads a shared library.""" |
| 15 | + lib = None |
| 16 | + if Path(filename).exists(): |
| 17 | + lib = cdll.LoadLibrary(filename) |
| 18 | + return lib |
| 19 | + |
| 20 | + |
| 21 | +class SharedLibraryManager(object): |
| 22 | + """LibraryManager creates (and deletes) copies of a shared library, which |
| 23 | + enables multiple copies of the same strategy to be run without the end user |
| 24 | + having to maintain many copies of the shared library. |
| 25 | +
|
| 26 | + This works by making a copy of the shared library file and loading it into |
| 27 | + memory again. Loading the same file again will return a reference to the |
| 28 | + same memory addresses. To be thread-safe, this class just passes filenames |
| 29 | + back to the Player class (which actually loads a reference to the library), |
| 30 | + ensuring that multiple copies of a given player type do not use the same |
| 31 | + copy of the shared library. |
| 32 | + """ |
| 33 | + |
| 34 | + def __init__(self, shared_library_name, verbose=False): |
| 35 | + self.shared_library_name = shared_library_name |
| 36 | + self.verbose = verbose |
| 37 | + self.filenames = [] |
| 38 | + self.player_indices = defaultdict(set) |
| 39 | + self.player_next = defaultdict(set) |
| 40 | + # Generate a random prefix for tempfile generation |
| 41 | + self.prefix = str(uuid.uuid4()) |
| 42 | + self.library_path = self.find_shared_library(shared_library_name) |
| 43 | + |
| 44 | + def find_shared_library(self, shared_library_name): |
| 45 | + # Hack for Linux since find_library doesn't return the full path. |
| 46 | + if 'Linux' in platform.system(): |
| 47 | + output = subprocess.check_output(["ldconfig", "-p"]) |
| 48 | + for line in str(output).split(r"\n"): |
| 49 | + rhs = line.split(" => ")[-1] |
| 50 | + if shared_library_name in rhs: |
| 51 | + return rhs |
| 52 | + raise ValueError("{} not found".format(shared_library_name)) |
| 53 | + else: |
| 54 | + return find_library( |
| 55 | + shared_library_name.replace("lib", "").replace(".so", "")) |
| 56 | + |
| 57 | + def create_library_copy(self): |
| 58 | + """Create a new copy of the shared library.""" |
| 59 | + # Copy the library file to a new (temp) location. |
| 60 | + temp_directory = tempfile.gettempdir() |
| 61 | + copy_number = len(self.filenames) |
| 62 | + filename = "{}-{}-{}".format( |
| 63 | + self.prefix, |
| 64 | + str(copy_number), |
| 65 | + self.shared_library_name) |
| 66 | + new_filename = str(Path(temp_directory, filename)) |
| 67 | + if self.verbose: |
| 68 | + print("Loading {}".format(new_filename)) |
| 69 | + shutil.copy2(self.library_path, new_filename) |
| 70 | + self.filenames.append(new_filename) |
| 71 | + |
| 72 | + def next_player_index(self, name): |
| 73 | + """Determine the index of the next free shared library copy to |
| 74 | + allocate for the player. If none is available then make another copy.""" |
| 75 | + # Is there a free index? |
| 76 | + if len(self.player_next[name]) > 0: |
| 77 | + return self.player_next[name].pop() |
| 78 | + # Do we need to load a new copy? |
| 79 | + player_count = len(self.player_indices[name]) |
| 80 | + if player_count == len(self.filenames): |
| 81 | + self.create_library_copy() |
| 82 | + return player_count |
| 83 | + # Find the first unused index |
| 84 | + for i in range(len(self.filenames)): |
| 85 | + if i not in self.player_indices[name]: |
| 86 | + return i |
| 87 | + raise ValueError("We shouldn't be here.") |
| 88 | + |
| 89 | + def get_filename_for_player(self, name): |
| 90 | + """For a given player return a filename for a copy of the shared library |
| 91 | + for use in a Player class, along with an index for later releasing.""" |
| 92 | + index = self.next_player_index(name) |
| 93 | + self.player_indices[name].add(index) |
| 94 | + if self.verbose: |
| 95 | + print("allocating {}".format(index)) |
| 96 | + return index, self.filenames[index] |
| 97 | + |
| 98 | + def release(self, name, index): |
| 99 | + """Release the copy of the library so that it can be re-allocated.""" |
| 100 | + self.player_indices[name].remove(index) |
| 101 | + if self.verbose: |
| 102 | + print("releasing {}".format(index)) |
| 103 | + self.player_next[name].add(index) |
| 104 | + |
| 105 | + def __del__(self): |
| 106 | + """Cleanup temp files on object deletion.""" |
| 107 | + for filename in self.filenames: |
| 108 | + path = Path(filename) |
| 109 | + if path.exists(): |
| 110 | + if self.verbose: |
| 111 | + print("deleting", str(path)) |
| 112 | + path.unlink() |
| 113 | + |
| 114 | + |
| 115 | +# Setup up thread safe library manager. |
| 116 | +class MultiprocessManager(BaseManager): |
| 117 | + pass |
| 118 | + |
| 119 | + |
| 120 | +MultiprocessManager.register('SharedLibraryManager', SharedLibraryManager) |
0 commit comments