qiita-cron-job initialize-resource-allocations-redis

Author: Gossty

qiita_db/meta_util.py

@@ -37,11 +37,17 @@
 from re import sub
 from json import loads, dump, dumps
 
-from qiita_db.util import create_nested_path
+from qiita_db.util import create_nested_path, _retrieve_resource_data
+from qiita_db.util import resource_allocation_plot, get_software_commands
 from qiita_core.qiita_settings import qiita_config, r_client
 from qiita_core.configuration_manager import ConfigurationManager
 import qiita_db as qdb
 
+# global constant list used in resource_allocation_page
+columns = [
+    "sName", "sVersion", "cID", "cName", "processing_job_id",
+    "parameters", "samples", "columns", "input_size", "extra_info",
+    "MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"]
 
 def _get_data_fpids(constructor, object_id):
     """Small function for getting filepath IDS associated with data object
@@ -546,3 +552,42 @@ def generate_plugin_releases():
         # important to "flush" variables to avoid errors
         r_client.delete(redis_key)
         f(redis_key, v)
+
+
+def initialize_resource_allocations_redis():
+    time = datetime.now().strftime('%m-%d-%y %H:%M:%S')
+    scommands = get_software_commands()
+    for software, versions in scommands.items():
+        for version, commands in versions.items():
+            for command in commands:
+                print("Generating plot for:", software, version, command)
+                update_resource_allocation_redis(command, software,
+                                                 version, time)
+    redis_key = 'resources:commands'
+    r_client.set(redis_key, str(scommands))
+
+
+def update_resource_allocation_redis(cname, sname, version, time):
+    col_name = "samples * columns"
+
+    df = _retrieve_resource_data(cname, sname, version, columns)
+    fig, axs = resource_allocation_plot(df, cname, sname, col_name)
+
+    fig.tight_layout()
+    plot = BytesIO()
+    fig.savefig(plot, format='png')
+    plot.seek(0)
+
+    img = 'data:image/png;base64,' + quote(b64encode(plot.getbuffer()).decode('ascii'))
+    plt.close(fig)
+
+    # SID, CID, col_name
+    values = [
+        ("img", img, r_client.set),
+        ('time', time, r_client.set)
+    ]
+
+    for k, v, f in values:
+        redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k)
+        r_client.delete(redis_key)
+        f(redis_key, v)

qiita_db/test/test_meta_util.py

@@ -519,6 +519,23 @@ def test_generate_plugin_releases(self):
             '-', '').replace(':', '').replace(' ', '-')
         self.assertEqual(tgz_obs, [time])
 
+    def test_update_resource_allocation_redis(self):
+        cname = "Split libraries FASTQ"
+        sname = "QIIMEq2"
+        col_name = "samples * columns"
+        qdb.meta_util.update_resource_allocation_redis(cname, sname)
+
+        vals = [
+            ("img", b's', r_client.get),
+            ('time', b'2024-11-11', r_client.get)
+        ]
+
+        for k, exp, f in vals:
+            redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k)
+            # checking redis values
+            print(f(redis_key))
+            # self.assertEqual(f(redis_key), exp)
+
 
 if __name__ == '__main__':
     main()

qiita_db/test/test_util.py

@@ -1311,8 +1311,9 @@ def test_quick_mounts_purge(self):
 
 class ResourceAllocationPlotTests(TestCase):
     def setUp(self):
-        self.CNAME = "Split libraries FASTQ"
-        self.SNAME = "QIIMEq2"
+        self.cname = "Split libraries FASTQ"
+        self.sname = "QIIMEq2"
+        self.version = "1.9.1"
         self.col_name = 'samples * columns'
         self.columns = [
                 "sName", "sVersion", "cID", "cName", "processing_job_id",
@@ -1322,12 +1323,12 @@ def setUp(self):
         # df is a dataframe that represents a table with columns specified in
         # self.columns
         self.df = qdb.util._retrieve_resource_data(
-                self.CNAME, self.SNAME, self.columns)
+                self.cname, self.sname, self.version, self.columns)
 
     def test_plot_return(self):
         # check the plot returns correct objects
         fig1, axs1 = qdb.util.resource_allocation_plot(
-            self.df, self.CNAME, self.SNAME, self.col_name)
+            self.df, self.cname, self.sname, self.col_name)
         self.assertIsInstance(
             fig1, Figure,
             "Returned object fig1 is not a Matplotlib Figure")
@@ -1338,13 +1339,13 @@ def test_plot_return(self):
 
     def test_minimize_const(self):
         self.df = self.df[
-            (self.df.cName == self.CNAME) & (self.df.sName == self.SNAME)]
+            (self.df.cName == self.cname) & (self.df.sName == self.sname)]
         self.df.dropna(subset=['samples', 'columns'], inplace=True)
         self.df[self.col_name] = self.df.samples * self.df['columns']
         fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False)
 
         bm, options = qdb.util._resource_allocation_plot_helper(
-            self.df, axs[0], self.CNAME, self.SNAME, 'MaxRSSRaw',
+            self.df, axs[0], self.cname, self.sname, 'MaxRSSRaw',
             qdb.util.MODELS_MEM, self.col_name)
         # check that the algorithm chooses correct model for MaxRSSRaw and
         # has 0 failures
@@ -1366,7 +1367,7 @@ def test_minimize_const(self):
         # check that the algorithm chooses correct model for ElapsedRaw and
         # has 1 failure
         bm, options = qdb.util._resource_allocation_plot_helper(
-            self.df, axs[1], self.CNAME, self.SNAME, 'ElapsedRaw',
+            self.df, axs[1], self.cname, self.sname, 'ElapsedRaw',
             qdb.util.MODELS_TIME, self.col_name)
         k, a, b = options.x
         failures_df = qdb.util._resource_allocation_failures(
@@ -1423,7 +1424,7 @@ def test_db_update(self):
 
         for curr_cname, ids in types.items():
             updated_df = qdb.util._retrieve_resource_data(
-                    curr_cname, self.SNAME, self.columns)
+                    curr_cname, self.sname, self.version, self.columns)
             updated_ids_set = set(updated_df['processing_job_id'])
             previous_ids_set = set(self.df['processing_job_id'])
             for id in ids:

qiita_db/util.py

@@ -2381,7 +2381,7 @@ def resource_allocation_plot(df, cname, sname, col_name):
     return fig, axs
 
 
-def _retrieve_resource_data(cname, sname, columns):
+def _retrieve_resource_data(cname, sname, version, columns):
     with qdb.sql_connection.TRN:
         sql = """
             SELECT
@@ -2411,9 +2411,10 @@ def _retrieve_resource_data(cname, sname, columns):
                 ON pr.processing_job_id = sra.processing_job_id
             WHERE
                 sc.name = %s
-                AND s.name = %s;
+                AND s.name = %s
+                AND s.version = %s
             """
-        qdb.sql_connection.TRN.add(sql, sql_args=[cname, sname])
+        qdb.sql_connection.TRN.add(sql, sql_args=[cname, sname, version])
         res = qdb.sql_connection.TRN.execute_fetchindex()
         df = pd.DataFrame(res, columns=columns)
         return df
@@ -2482,15 +2483,19 @@ def _resource_allocation_plot_helper(
     y_plot = best_model(x_plot, k, a, b)
     ax.plot(x_plot, y_plot, linewidth=1, color='orange')
 
+    cmin_value = min(y_plot)
+    cmax_value = max(y_plot)
+
     maxi = naturalsize(df[curr].max(), gnu=True) if curr == "MaxRSSRaw" else \
         timedelta(seconds=float(df[curr].max()))
-    cmax = naturalsize(max(y_plot), gnu=True) if curr == "MaxRSSRaw" else \
-        timedelta(seconds=float(max(y_plot)))
+    cmax = naturalsize(cmax_value, gnu=True) if curr == "MaxRSSRaw" else \
+        str(timedelta(seconds=round(cmax_value, 2))).rstrip('0').rstrip('.')
 
     mini = naturalsize(df[curr].min(), gnu=True) if curr == "MaxRSSRaw" else \
         timedelta(seconds=float(df[curr].min()))
-    cmin = naturalsize(min(y_plot), gnu=True) if curr == "MaxRSSRaw" else \
-        timedelta(seconds=float(min(y_plot)))
+    cmin = naturalsize(cmin_value, gnu=True) if curr == "MaxRSSRaw" else \
+        str(timedelta(seconds=round(cmin_value, 2))).rstrip('0').rstrip('.')
+
 
     x_plot = np.array(df[col_name])
     failures_df = _resource_allocation_failures(
@@ -2919,3 +2924,35 @@ def merge_rows(rows):
                 row['node_model']]
             qdb.sql_connection.TRN.add(sql, sql_args=to_insert)
             qdb.sql_connection.TRN.execute()
+
+
+def get_software_commands():
+    res = []
+    with qdb.sql_connection.TRN:
+        sql_command = """
+            SELECT DISTINCT
+                s.name AS sName,
+                s.version AS sVersion,
+                sc.name AS cName
+            FROM
+                qiita.slurm_resource_allocations sra
+            JOIN
+                qiita.processing_job pr
+                ON sra.processing_job_id = pr.processing_job_id
+            JOIN
+                qiita.software_command sc on pr.command_id = sc.command_id
+            JOIN
+                qiita.software s ON sc.software_id = s.software_id
+            """
+        qdb.sql_connection.TRN.add(sql_command)
+        res = qdb.sql_connection.TRN.execute_fetchindex()
+
+    software_commands = dict()
+    for s_name, s_version, c_name in res:
+        if s_name not in software_commands:
+            software_commands[s_name] = {}
+        if s_version not in software_commands[s_name]:
+            software_commands[s_name][s_version] = set()
+        software_commands[s_name][s_version].add(c_name)
+
+    return software_commands

qiita_pet/handlers/resources.py

@@ -0,0 +1,67 @@
+
+from tornado.gen import coroutine, Task
+from tornado.web import authenticated, HTTPError
+
+
+from .base_handlers import BaseHandler
+from qiita_core.qiita_settings import qiita_config, r_client
+from qiita_core.util import execute_as_transaction
+
+default_cname = "Split libraries FASTQ"
+default_sname = "QIIMEq2"
+default_col_name = "samples * columns"
+commands = 'resources:commands'
+
+class ResourcesHandler(BaseHandler):
+    def check_admin(self):
+        if self.current_user.level != "admin":
+            raise HTTPError(403, reason="%s does not have access to portal "
+                            "editing!" % self.current_user.id)
+
+
+    @execute_as_transaction
+    def _get_resources(self, callback):
+        resources = {}
+        vals = [
+            ('img', r_client.get),
+            ('time', r_client.get)
+        ]
+        for k, f in vals:
+            redis_key = 'resources$#%s$#%s$#%s:%s' % (default_cname,
+                                            default_sname, default_col_name, k)
+            resources[k] = f(redis_key)
+        callback(resources)
+
+
+    def _get_commands(self, callback):
+        r_client.get(commands)
+        callback(r_client)
+
+    
+    @authenticated
+    @coroutine
+    @execute_as_transaction
+    def get(self):
+        self.check_admin()
+        resources = yield Task(self._get_resources)
+        commands = yield Task(self._get_commands) # TODO: it would make more sense to have this rendered once instead of everytime we make a get request, but i'm not sure how to do that right now
+        self.render('resources.html',
+            img=resources['img'], time=resources['time'],
+            commands=commands,
+            software=default_sname, command=default_cname,
+            col_name=default_col_name
+        )
+
+
+    @authenticated
+    @execute_as_transaction
+    def post(self):
+        self.check_admin()
+        software = self.get_argument("software", "")
+        command = self.get_argument("command", "")
+        column_type = self.get_argument("column_type", "")
+        
+
+        self.get()
+        
+

qiita_pet/templates/resources.html

@@ -0,0 +1,52 @@
+{% extends sitebase.html %}
+{% block head %}
+    <link rel="stylesheet" href="{% raw qiita_config.portal_dir %}/static/vendor/css/ol.css" type="text/css">
+    <style type="text/css">
+        /* Add any custom styles here */
+    </style>
+    <script src="{% raw qiita_config.portal_dir %}/static/vendor/js/ol.js"></script>
+{% end %}
+
+{% block content %}
+    <div style="width: 80%; margin: 0 auto; max-width: 1600px;">
+        <div style="display: flex; flex-direction: column; gap: 10px; background-color: #f8f9fa; padding: 15px; border-radius: 5px;">
+            <p3>Generated on: {{time}}</p3>
+            <p3>Software: {{software}}</p3>
+            <p3>Command: {{command}}</p3>
+            <p3>Column name: {{col_name}}</p3>
+        </div>
+    
+      <form method="POST">
+        <div class="form-group">
+          <label for="software">Software:</label>
+          <select id="software" name="software" class="form-control">
+            <option value="">Select Software</option>
+            <option value="QIIMEq2" selected>QIIMEq2</option>
+          </select>
+        </div>
+      
+        <div class="form-group">
+          <label for="command">Command:</label>
+          <select id="command" name="command" class="form-control">
+            <option value="">Select Command</option>
+            <option value="Split libraries FASTQ" selected>Split libraries FASTQ</option>
+          </select>
+        </div>
+      
+        <div class="form-group">
+          <label for="column_type">Column Type:</label>
+          <select id="column_type" name="column_type" class="form-control">
+            <option value="">Select Column Type</option>
+            <option value="samples * columns" selected>samples * columns</option>
+          </select>
+        </div>
+      
+        <button type="submit" class="btn btn-primary">Submit</button>
+      </form>
+      {% if img and img is not None %}
+        <div>
+          <img height="100%" width="100%" src="{% raw img %}"/>
+        </div>
+      {% end %}
+    </div>
+{% end %}

qiita_pet/templates/sitebase.html

@@ -384,6 +384,7 @@
                       <li><a href="{% raw qiita_config.portal_dir %}/admin/approval/">View Studies awaiting approval</a></li>
                       <li><a href="{% raw qiita_config.portal_dir %}/admin/portals/studies/">Edit study portal connections</a></li>
                       <li><a href="{% raw qiita_config.portal_dir %}/admin/purge_users/">Purge non-validated users</a></li>
+                      <li><a href="{% raw qiita_config.portal_dir %}/admin/resources/">View Resource Allocation Plots</a></li>
                     {% end %}
                     <li><a href="{% raw qiita_config.portal_dir %}/admin/sample_validation/">Sample Validation</a></li>
                     <li><a href="{% raw qiita_config.portal_dir %}/admin/processing_jobs/">Processing Jobs</a></li>

qiita_pet/webserver.py

@@ -50,6 +50,8 @@
 from qiita_pet.handlers.upload import (
     UploadFileHandler, StudyUploadFileHandler, StudyUploadViaRemote)
 from qiita_pet.handlers.stats import StatsHandler
+from qiita_pet.handlers.resources import ResourcesHandler
+
 from qiita_pet.handlers.download import (
     DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease,
     DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions,
@@ -136,6 +138,7 @@ def __init__(self):
             (r"/admin/sample_validation/", SampleValidation),
             (r"/admin/purge_users/", PurgeUsersHandler),
             (r"/admin/purge_usersAjax/", PurgeUsersAJAXHandler),
+            (r"/admin/resources/", ResourcesHandler),
             (r"/ebi_submission/(.*)", EBISubmitHandler),
             # Study handlers
             (r"/study/create/", StudyEditHandler),

scripts/all-qiita-cron-job

@@ -4,5 +4,6 @@ qiita-cron-job empty-trash-upload-folder
 qiita-cron-job generate-biom-and-metadata-release
 qiita-cron-job purge-filepaths
 qiita-cron-job update-redis-stats
+qiita-cron-job initialize-resource-allocations-redis
 qiita-cron-job generate-plugin-releases
 qiita-cron-job purge-json-web-tokens