Using $HOME directory instead of $TMPDIR to store matlab-proxy launch artifacts.

Author: diningPhilosopher64

matlab_proxy/app_state.py

@@ -1,7 +1,6 @@
 # Copyright 2020-2021 The MathWorks, Inc.
 
 import asyncio
-from matlab_proxy import mwi_embedded_connector as mwi_connector
 from matlab_proxy import mwi_environment_variables as mwi_env
 from matlab_proxy import util
 import os
@@ -40,8 +39,19 @@ def __init__(self, settings):
         """
         self.settings = settings
         self.processes = {"matlab": None, "xvfb": None}
+
+        # The port on which MATLAB(launched by this matlab-proxy process) starts on.
         self.matlab_port = None
+
+        # The file created and written by MATLAB's Embedded connector to signal readiness.
         self.matlab_ready_file = None
+
+        # The directory in which the instance of MATLAB (launched by this matlab-proxy process) will write logs to.
+        self.matlab_ready_file_dir = None
+
+        # The file created by this instance of matlab-proxy to signal to other matlab-proxy processes
+        # that this self.matlab_port will be used by this instance.
+        self.mwi_proxy_lock_file = None
         self.licensing = None
         self.tasks = {}
         self.logs = {
@@ -354,8 +364,15 @@ def persist_licensing(self):
             with open(cached_licensing_file, "w") as f:
                 f.write(json.dumps(self.licensing))
 
-    def get_free_matlab_port(self):
-        """Returns a free port for MATLAB Embedded Connector in the allowed range."""
+    def prepare_lock_files_for_MATLAB_launch(self):
+        """Finds and reserves a free port for MATLAB Embedded Connector in the allowed range.
+        Creates the lock file to prevent any other matlab-proxy process to use the reserved port of this
+        process.
+
+        Raises:
+            e: socket.error if the exception raised is other than port already occupied.
+        """
+
         # NOTE It is not guranteed that the port will remain free!
         # FIXME Because of https://github.com/http-party/node-http-proxy/issues/1342 the
         # node application in development mode always uses port 31515 to bypass the
@@ -372,16 +389,53 @@ def get_free_matlab_port(self):
             # try-except.
             # s.bind(("", 0))
             # self.matlab_port = s.getsockname()[1]
+
             for port in mw.range_matlab_connector_ports():
                 try:
                     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                     s.bind(("", port))
-                    s.close()
-                    break
+
+                    matlab_ready_file_dir = self.settings["mwi_logs_root_dir"] / str(
+                        port
+                    )
+
+                    mwi_proxy_lock_file = (
+                        matlab_ready_file_dir
+                        / self.settings["mwi_proxy_lock_file_name"]
+                    )
+
+                    matlab_ready_file = matlab_ready_file_dir / "connector.securePort"
+
+                    # Check if the mwi_proxy_lock_file exists.
+                    # Implies there was a competing matlab-proxy process which found the same port before this process
+                    if mwi_proxy_lock_file.exists():
+                        logger.debug(
+                            f"Skipping port number {port} for MATLAB as lock file already exists at {mwi_proxy_lock_file}"
+                        )
+                        s.close()
+
+                    else:
+                        # Create a folder to hold the matlab_ready_file that will be created by MATLAB to signal readiness.
+                        # This is the same folder to which MATLAB will write logs to.
+                        matlab_ready_file_dir.mkdir(parents=True, exist_ok=True)
+
+                        # Creating the mwi_proxy.lock file to indicate to any other matlab-proxy processes
+                        # that this self.matlab_port number is taken up by this process.
+                        mwi_proxy_lock_file.touch()
+
+                        # Update member variables of AppState class
+
+                        # Store the port number on which MATLAB will be launched for this matlab-proxy process.
+                        self.matlab_port = port
+                        self.mwi_proxy_lock_file = mwi_proxy_lock_file
+                        self.matlab_ready_file_dir = matlab_ready_file_dir
+                        self.matlab_ready_file = matlab_ready_file
+                        s.close()
+                        return
+
                 except socket.error as e:
                     if e.errno != errno.EADDRINUSE:
                         raise e
-        return port
 
     async def start_matlab(self, restart_matlab=False):
         """Start MATLAB.
@@ -423,15 +477,14 @@ async def start_matlab(self, restart_matlab=False):
         self.error = None
         self.logs["matlab"].clear()
 
-        # Reserve a port for MATLAB Embedded Connector
-        self.matlab_port = self.get_free_matlab_port()
+        # Finds and reserves a free port, then prepare lock files for the MATLAB process.
+        self.prepare_lock_files_for_MATLAB_launch()
 
-        # Create a folder to hold the matlab_ready_file that will be created by MATLAB to signal readiness
-        self.matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(
-            self.matlab_port
-        )
-        logger.debug(f"MATLAB_LOG_DIR:{str(matlab_log_dir)}")
+        logger.debug(f"MATLAB_LOG_DIR:{str( self.matlab_ready_file_dir )}")
         logger.debug(f"MATLAB_READY_FILE:{str(self.matlab_ready_file)}")
+        logger.debug(
+            f"Created MWI proxy lock file for this matlab-proxy process at {self.mwi_proxy_lock_file}"
+        )
 
         # Configure the environment MATLAB needs to start
         matlab_env = os.environ.copy()
@@ -445,7 +498,8 @@ async def start_matlab(self, restart_matlab=False):
         )
         matlab_env["MWAPIKEY"] = self.settings["mwapikey"]
         # The matlab ready file is written into this location by MATLAB
-        matlab_env["MATLAB_LOG_DIR"] = str(matlab_log_dir)
+        # The matlab_ready_file_dir is where MATLAB will write any subsequent logs
+        matlab_env["MATLAB_LOG_DIR"] = str(self.matlab_ready_file_dir)
         matlab_env["MW_CD_ANYWHERE_ENABLED"] = "true"
         if self.licensing["type"] == "mhlm":
             matlab_env["MLM_WEB_LICENSE"] = "true"
@@ -538,15 +592,16 @@ async def stop_matlab(self):
             xvfb.terminate()
             waiters.append(xvfb.wait())
 
-        # Clean up matlab_ready_file
         try:
+            # Clean up both connector.securePort file and mwi_proxy_lock_file
             if self.matlab_ready_file is not None:
-                logger.info(
-                    f"Cleaning up matlab_ready_file...{str(self.matlab_ready_file)}"
-                )
                 self.matlab_ready_file.unlink()
+
+            if self.mwi_proxy_lock_file is not None:
+                self.mwi_proxy_lock_file.unlink()
+
         except FileNotFoundError:
-            # Some other process deleted this file
+            # Files won't exist when stop_matlab is called for the first time.
             pass
 
         # Wait for termination

matlab_proxy/devel.py

@@ -4,8 +4,7 @@
 import asyncio
 from aiohttp import web
 import socket, time, os, sys
-from matlab_proxy import mwi_environment_variables as mwi_env
-from matlab_proxy import mwi_embedded_connector as mwi_connector
+from matlab_proxy import mwi_environment_variables as mwi_env, settings
 
 desktop_html = b"""
 <h1>Fake MATLAB Web Desktop</h1>
@@ -134,15 +133,20 @@ async def fake_matlab_started(app):
 
     # Real MATLAB always uses  $MATLAB_LOG_DIR/connection.securePort as the ready file
     # We mock reading from the environment variable by calling the helper functions
-    matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(app["port"])
+    matlab_ready_file_dir = settings.get(dev=True)["mwi_logs_root_dir"] / str(
+        app["port"]
+    )
+    matlab_ready_file_dir.mkdir(parents=True, exist_ok=True)
+
+    app["matlab_ready_file"] = matlab_ready_file_dir / "connector.securePort"
 
     ready_delay = app["ready_delay"]
     try:
         await asyncio.sleep(ready_delay)
         print(
-            f"Creating fake MATLAB Embedded Connector ready file at {matlab_ready_file}"
+            f"Creating fake MATLAB Embedded Connector ready file at {app['matlab_ready_file']}"
         )
-        matlab_ready_file.touch()
+        app["matlab_ready_file"].touch()
     except asyncio.CancelledError:
         pass
 
@@ -164,8 +168,8 @@ async def cleanup_background_tasks(app):
     """
     # Delete ready file on tear down
     # NOTE MATLAB does not delete this file on shutdown.
-    matlab_ready_file, matlab_log_dir = mwi_connector.get_matlab_ready_file(app["port"])
-    matlab_ready_file.unlink()
+
+    app["matlab_ready_file"].unlink()
 
 
 def matlab(args):

matlab_proxy/mwi_embedded_connector.py

@@ -45,7 +45,7 @@ def get_ws_env_settings():
 
 def get_dev_settings(config):
     devel_file = Path(__file__).resolve().parent / "./devel.py"
-    test_temp_dir = get_test_temp_dir()
+    mwi_config_folder = get_test_temp_dir()
     ws_env, ws_env_suffix = get_ws_env_settings()
     return {
         "matlab_path": Path(),
@@ -64,14 +64,16 @@ def get_dev_settings(config):
         "matlab_protocol": "http",
         "matlab_display": ":1",
         "nlm_conn_str": os.environ.get(mwi_env.get_env_name_network_license_manager()),
-        "matlab_config_file": test_temp_dir / "proxy_app_config.json",
+        "matlab_config_file": mwi_config_folder / "proxy_app_config.json",
         "ws_env": ws_env,
         "mwa_api_endpoint": f"https://login{ws_env_suffix}.mathworks.com/authenticationws/service/v4",
         "mhlm_api_endpoint": f"https://licensing{ws_env_suffix}.mathworks.com/mls/service/v1/entitlement/list",
         "mwa_login": f"https://login{ws_env_suffix}.mathworks.com",
         "mwi_custom_http_headers": mwi_custom_http_headers.get(),
         "env_config": mwi_validators.validate_env_config(config),
         "ssl_context": None,
+        "mwi_logs_root_dir": mwi_config_folder / "ports",
+        "mwi_proxy_lock_file_name": "mwi_proxy.lock",
     }
 
 
@@ -122,6 +124,11 @@ def get(config=matlab_proxy.get_default_config_name(), dev=False):
             os.getenv(mwi_env.get_env_name_ssl_key_file(), None),
             os.getenv(mwi_env.get_env_name_ssl_cert_file(), None),
         )
+
+        # All config related to matlab-proxy will be saved to user's home folder.
+        # This will allow for other user's to launch the integration from the same system
+        # and not have their config's overwritten.
+        mwi_config_folder = Path.home() / ".matlab" / "MWI"
         return {
             "matlab_path": matlab_path,
             "matlab_version": get_matlab_version(matlab_path),
@@ -146,7 +153,7 @@ def get(config=matlab_proxy.get_default_config_name(), dev=False):
             "nlm_conn_str": mwi_validators.validate_mlm_license_file(
                 os.environ.get(mwi_env.get_env_name_network_license_manager())
             ),
-            "matlab_config_file": Path.home() / ".matlab" / "proxy_app_config.json",
+            "matlab_config_file": mwi_config_folder / "proxy_app_config.json",
             "ws_env": ws_env,
             "mwa_api_endpoint": f"https://login{ws_env_suffix}.mathworks.com/authenticationws/service/v4",
             "mhlm_api_endpoint": f"https://licensing{ws_env_suffix}.mathworks.com/mls/service/v1/entitlement/list",
@@ -156,6 +163,11 @@ def get(config=matlab_proxy.get_default_config_name(), dev=False):
             "ssl_context": get_ssl_context(
                 ssl_cert_file=ssl_cert_file, ssl_key_file=ssl_key_file
             ),
+            # This directory will be used to store connector.securePort(matlab_ready_file) and its corresponding files. This will be
+            # a central place to store logs of all the running instances of MATLAB launched by matlab-proxy
+            "mwi_logs_root_dir": mwi_config_folder / "ports",
+            # Name of the lock file which will be created by this instance of matlab-proxy process.
+            "mwi_proxy_lock_file_name": "mwi_proxy.lock",
         }
 
 
@@ -196,21 +208,6 @@ def create_xvfb_cmd():
     return xvfb_cmd, dpipe
 
 
-def get_matlab_tempdir():
-    """The temp directory used by MATLAB based on tempdir.m"""
-
-    for env_name in mwi_env.get_env_name_matlab_tempdir():
-        matlab_tempdir = os.environ.get(env_name)
-        if matlab_tempdir is not None:
-            break
-
-    # MATLAB defaults to '/tmp'
-    if matlab_tempdir is None:
-        matlab_tempdir = "/tmp"
-
-    return matlab_tempdir
-
-
 def get_test_temp_dir():
     """The temp directory to be used by tests"""
     test_temp_dir = Path(tempfile.gettempdir()) / "MWI" / "tests"