From a47c13bef6edb08fb9a84bbbdf446a2311f4948b Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 10:18:49 -0700 Subject: [PATCH 1/8] Move qiita_db/private.py -> qiita_ware/private_plugin.py --- qiita_db/private.py => qiita_ware/private_plugin.py | 0 scripts/qiita-private-plugin | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename qiita_db/private.py => qiita_ware/private_plugin.py (100%) diff --git a/qiita_db/private.py b/qiita_ware/private_plugin.py similarity index 100% rename from qiita_db/private.py rename to qiita_ware/private_plugin.py diff --git a/scripts/qiita-private-plugin b/scripts/qiita-private-plugin index a4a13377f..ce5f8ef6e 100755 --- a/scripts/qiita-private-plugin +++ b/scripts/qiita-private-plugin @@ -10,7 +10,7 @@ import click -import qiita_db as qdb +from qiita_ware.private_plugin import private_task @click.command() @@ -23,7 +23,7 @@ def execute(url, job_id, output_dir): The parameters url and output_dir are ignored, but they are added for compatibility with the plugin system. """ - qdb.private.private_task(job_id) + private_task(job_id) if __name__ == '__main__': From c16b177d4daf11fa0c450895d0fbbc9da3776e6e Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 11:18:05 -0700 Subject: [PATCH 2/8] Transferring VAMPS submission to internal job --- qiita_db/support_files/patches/58.sql | 4 ++ .../patches/python_patches/58.py | 19 +++++++++ .../handlers/study_handlers/vamps_handlers.py | 40 +++++++++---------- qiita_ware/dispatchable.py | 7 +--- qiita_ware/private_plugin.py | 16 +++++++- 5 files changed, 59 insertions(+), 27 deletions(-) create mode 100644 qiita_db/support_files/patches/58.sql create mode 100644 qiita_db/support_files/patches/python_patches/58.py diff --git a/qiita_db/support_files/patches/58.sql b/qiita_db/support_files/patches/58.sql new file mode 100644 index 000000000..461b1f869 --- /dev/null +++ b/qiita_db/support_files/patches/58.sql @@ -0,0 +1,4 @@ +-- Aug 31, 2017 +-- Remove MOI and transfer all jobs to internal QIITA plugin + +SELECT 42; diff --git a/qiita_db/support_files/patches/python_patches/58.py b/qiita_db/support_files/patches/python_patches/58.py new file mode 100644 index 000000000..86015bd35 --- /dev/null +++ b/qiita_db/support_files/patches/python_patches/58.py @@ -0,0 +1,19 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from qiita_db.sql_connection import TRN +from qiita_db.software import Software, Command + +with TRN: + # Retrieve the Qiita plugin + qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') + + # Create the submit to VAMPS command + parameters = {'artifact': ['artifact:["Demultiplexed"]', None]} + Command.create(qiita_plugin, "submit_to_VAMPS", + "submits an artifact to VAMPS", parameters) diff --git a/qiita_pet/handlers/study_handlers/vamps_handlers.py b/qiita_pet/handlers/study_handlers/vamps_handlers.py index 0996837f8..075492e48 100644 --- a/qiita_pet/handlers/study_handlers/vamps_handlers.py +++ b/qiita_pet/handlers/study_handlers/vamps_handlers.py @@ -10,10 +10,10 @@ from tornado.web import authenticated, HTTPError from qiita_files.demux import stats as demux_stats -from qiita_ware.context import submit -from qiita_ware.dispatchable import submit_to_VAMPS from qiita_db.exceptions import QiitaDBUnknownIDError from qiita_db.artifact import Artifact +from qiita_db.software import Software, Parameters +from qiita_db.processing_job import ProcessingJob from qiita_pet.handlers.base_handlers import BaseHandler from qiita_core.util import execute_as_transaction @@ -99,23 +99,23 @@ def post(self, preprocessed_data_id): user.id) msg = '' msg_level = 'success' - study = Artifact(preprocessed_data_id).study - study_id = study.id - state = study.ebi_submission_status - if state == 'submitting': - msg = "Cannot resubmit! Current state is: %s" % state - msg_level = 'danger' - else: - channel = user.id - job_id = submit(channel, submit_to_VAMPS, - int(preprocessed_data_id)) - self.render('compute_wait.html', - job_id=job_id, title='VAMPS Submission', - completion_redirect=('/study/description/%s?top_tab=' - 'preprocessed_data_tab&sub_tab=%s' - % (study_id, - preprocessed_data_id))) - return + plugin = Software.from_name_and_version('Qiita', 'alpha') + cmd = plugin.get_command('submit_to_VAMPS') + artifact = Artifact(preprocessed_data_id) + + # Check if the artifact is already being submitted to VAMPS + is_being_submitted = any( + [j.status in ('queued', 'running') + for j in artifact.jobs(cmd=cmd)]) - self.display_template(preprocessed_data_id, msg, msg_level) + if is_being_submitted == 'submitting': + msg = "Cannot resubmit! Data is already being submitted" + msg_level = 'danger' + self.display_template(preprocessed_data_id, msg, msg_level) + else: + params = Parameters.load( + cmd, values_dict={'artifact': preprocessed_data_id}) + job = ProcessingJob.create(user, params) + job.submit() + self.redirect('/study/description/%s' % artifact.study.study_id) diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index c111fa6b9..c958df127 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -5,7 +5,7 @@ # # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- -from qiita_ware.commands import submit_EBI, submit_VAMPS +from qiita_ware.commands import submit_EBI def submit_to_ebi(preprocessed_data_id, submission_type): @@ -13,11 +13,6 @@ def submit_to_ebi(preprocessed_data_id, submission_type): submit_EBI(preprocessed_data_id, submission_type, True) -def submit_to_VAMPS(preprocessed_data_id): - """Submit a study to VAMPS""" - return submit_VAMPS(preprocessed_data_id) - - def create_raw_data(artifact_type, prep_template, filepaths, name=None): """Creates a new raw data diff --git a/qiita_ware/private_plugin.py b/qiita_ware/private_plugin.py index 0f10187ca..8b3abbf56 100644 --- a/qiita_ware/private_plugin.py +++ b/qiita_ware/private_plugin.py @@ -12,6 +12,7 @@ import traceback import qiita_db as qdb +from qiita_ware.commands import submit_VAMPS def build_analysis_files(job): @@ -66,8 +67,21 @@ def release_validators(job): job._set_status('success') +def submit_to_VAMPS(job): + """Submits an artifact to VAMPS + + Parameters + ---------- + job : qiita_db.processing_job.ProcessingJob + The processing job performing the task + """ + with qdb.sql_connection.TRN: + submit_VAMPS(job.parameters.values['artifact']) + + TASK_DICT = {'build_analysis_files': build_analysis_files, - 'release_validators': release_validators} + 'release_validators': release_validators, + 'submit_to_VAMPS': submit_to_VAMPS} def private_task(job_id): From 4f43236ecfd270fbc55e4b127f1217c38e073eab Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 11:50:21 -0700 Subject: [PATCH 3/8] Fixing merge conflicts --- qiita_ware/dispatchable.py | 33 --------------------------------- qiita_ware/private_plugin.py | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/qiita_ware/dispatchable.py b/qiita_ware/dispatchable.py index 52e94f1bf..e06592a86 100644 --- a/qiita_ware/dispatchable.py +++ b/qiita_ware/dispatchable.py @@ -13,39 +13,6 @@ def submit_to_ebi(preprocessed_data_id, submission_type): submit_EBI(preprocessed_data_id, submission_type, True) -def copy_raw_data(prep_template, artifact_id): - """Creates a new raw data by copying from artifact_id - - Parameters - ---------- - prep_template : qiita_db.metadata_template.prep_template.PrepTemplate - The template to attach the artifact - artifact_id : int - The id of the artifact to duplicate - - Returns - ------- - dict of {str: str} - A dict of the form {'status': str, 'message': str} - """ - from qiita_db.artifact import Artifact - - status = 'success' - msg = '' - - try: - Artifact.copy(Artifact(artifact_id), prep_template) - except Exception as e: - # We should hit this exception rarely (that's why it is an - # exception) since at this point we have done multiple checks. - # However, it can occur in weird cases, so better let the GUI know - # that this failed - return {'status': 'danger', - 'message': "Error creating artifact: %s" % str(e)} - - return {'status': status, 'message': msg} - - def delete_artifact(artifact_id): """Deletes an artifact from the system diff --git a/qiita_ware/private_plugin.py b/qiita_ware/private_plugin.py index 8b3abbf56..f550a147f 100644 --- a/qiita_ware/private_plugin.py +++ b/qiita_ware/private_plugin.py @@ -77,11 +77,29 @@ def submit_to_VAMPS(job): """ with qdb.sql_connection.TRN: submit_VAMPS(job.parameters.values['artifact']) + job._set_status('success') + + +def copy_artifact(job): + """Creates a copy of an artifact + + Parameters + ---------- + job : qiita_db.processing_job.ProcessingJob + The processing job performing the task + """ + with qdb.sql_connection.TRN: + param_vals = job.parameters.values + qdb.artifact.Artifact.copy( + qdb.artifact.Artifact(param_vals['artifact']), + param_vals['prep_template']) + job._set_status('success') TASK_DICT = {'build_analysis_files': build_analysis_files, 'release_validators': release_validators, - 'submit_to_VAMPS': submit_to_VAMPS} + 'submit_to_VAMPS': submit_to_VAMPS, + 'copy_artifact': copy_artifact} def private_task(job_id): From a528e04dc7f2b73b99bef0f7ab2ff7fde557cd0d Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 12:10:17 -0700 Subject: [PATCH 4/8] Adding tests to private plugin --- .../patches/python_patches/58.py | 6 +++ qiita_ware/private_plugin.py | 5 ++- qiita_ware/test/test_dispatchable.py | 7 ---- qiita_ware/test/test_private_plugin.py | 40 +++++++++++++++++++ 4 files changed, 49 insertions(+), 9 deletions(-) create mode 100644 qiita_ware/test/test_private_plugin.py diff --git a/qiita_db/support_files/patches/python_patches/58.py b/qiita_db/support_files/patches/python_patches/58.py index 86015bd35..5e1a21f5d 100644 --- a/qiita_db/support_files/patches/python_patches/58.py +++ b/qiita_db/support_files/patches/python_patches/58.py @@ -17,3 +17,9 @@ parameters = {'artifact': ['artifact:["Demultiplexed"]', None]} Command.create(qiita_plugin, "submit_to_VAMPS", "submits an artifact to VAMPS", parameters) + + # Create the copy artifact command + parameters = {'artifact': ['artifact:["Demultiplexed"]', None], + 'prep_template': ['prep_template', None]} + Command.create(qiita_plugin, "copy_artifact", + "Creates a copy of an artifact", parameters) diff --git a/qiita_ware/private_plugin.py b/qiita_ware/private_plugin.py index f550a147f..8aaff0fb2 100644 --- a/qiita_ware/private_plugin.py +++ b/qiita_ware/private_plugin.py @@ -90,9 +90,10 @@ def copy_artifact(job): """ with qdb.sql_connection.TRN: param_vals = job.parameters.values - qdb.artifact.Artifact.copy( - qdb.artifact.Artifact(param_vals['artifact']), + orig_artifact = qdb.artifact.Artifact(param_vals['artifact']) + prep_template = qdb.metadata_template.prep_template.PrepTemplate( param_vals['prep_template']) + qdb.artifact.Artifact.copy(orig_artifact, prep_template) job._set_status('success') diff --git a/qiita_ware/test/test_dispatchable.py b/qiita_ware/test/test_dispatchable.py index 67557010e..a981a1bec 100644 --- a/qiita_ware/test/test_dispatchable.py +++ b/qiita_ware/test/test_dispatchable.py @@ -42,13 +42,6 @@ def tearDown(self): if exists(fp): remove(fp) - def test_copy_raw_data(self): - obs = copy_raw_data(PrepTemplate(1), 1) - exp = {'status': 'danger', - 'message': "Error creating artifact: Prep template 1 already " - "has an artifact associated"} - self.assertEqual(obs, exp) - def test_delete_artifact(self): obs = delete_artifact(1) exp = {'status': 'danger', diff --git a/qiita_ware/test/test_private_plugin.py b/qiita_ware/test/test_private_plugin.py new file mode 100644 index 000000000..2eb601a62 --- /dev/null +++ b/qiita_ware/test/test_private_plugin.py @@ -0,0 +1,40 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +from unittest import TestCase, main + +from qiita_core.util import qiita_test_checker +from qiita_db.software import Software, Parameters +from qiita_db.processing_job import ProcessingJob +from qiita_db.user import User +from qiita_ware.private_plugin import private_task + + +@qiita_test_checker() +class TestPrivatePlugin(TestCase): + def _create_job(self, cmd, values_dict): + user = User('test@foo.bar') + qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') + cmd = qiita_plugin.get_command('copy_artifact') + params = Parameters.load(cmd, values_dict=values_dict) + job = ProcessingJob.create(user, params) + job._set_status('queued') + return job + + def test_copy_artifact(self): + job = self._create_job('copy_artifact', + {'artifact': 1, 'prep_template': 1}) + + private_task(job.id) + self.assertEqual(job.status, 'error') + self.assertIn("Prep template 1 already has an artifact associated", + job.log.msg) + + +if __name__ == '__main__': + main() From 0bf730757259497e7f606818c1ec51b121bd0834 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 12:12:21 -0700 Subject: [PATCH 5/8] Removing import --- qiita_ware/test/test_dispatchable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_ware/test/test_dispatchable.py b/qiita_ware/test/test_dispatchable.py index a981a1bec..4ff7318bc 100644 --- a/qiita_ware/test/test_dispatchable.py +++ b/qiita_ware/test/test_dispatchable.py @@ -17,8 +17,7 @@ from qiita_core.util import qiita_test_checker from qiita_ware.dispatchable import ( create_sample_template, update_sample_template, delete_sample_template, - update_prep_template, delete_artifact, copy_raw_data, - delete_sample_or_column) + update_prep_template, delete_artifact, delete_sample_or_column) from qiita_db.study import Study from qiita_db.artifact import Artifact from qiita_db.exceptions import QiitaDBUnknownIDError, QiitaDBWarning From 6ee90227f5524c47dd20011506092611df5b4b52 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 12:14:42 -0700 Subject: [PATCH 6/8] Fixing import --- qiita_db/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiita_db/__init__.py b/qiita_db/__init__.py index e9e71d654..ceab0f2bf 100644 --- a/qiita_db/__init__.py +++ b/qiita_db/__init__.py @@ -26,7 +26,6 @@ import study import user import processing_job -import private __version__ = "0.2.0-dev" @@ -34,4 +33,4 @@ "exceptions", "investigation", "logger", "meta_util", "ontology", "portal", "reference", "search", "software", "sql_connection", "study", "user", "util", - "metadata_template", "processing_job", "private"] + "metadata_template", "processing_job"] From c3bc77c8462f7435c2279151151f611e2cb8fa6f Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 12:27:22 -0700 Subject: [PATCH 7/8] Modifying GUI to use the plugin --- qiita_pet/handlers/api_proxy/artifact.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/qiita_pet/handlers/api_proxy/artifact.py b/qiita_pet/handlers/api_proxy/artifact.py index 9c0be2945..16e31b5e6 100644 --- a/qiita_pet/handlers/api_proxy/artifact.py +++ b/qiita_pet/handlers/api_proxy/artifact.py @@ -16,13 +16,12 @@ from qiita_core.qiita_settings import qiita_config, r_client from qiita_pet.handlers.api_proxy.util import check_access, check_fp from qiita_ware.context import safe_submit -from qiita_ware.dispatchable import copy_raw_data from qiita_db.artifact import Artifact from qiita_db.user import User from qiita_db.metadata_template.prep_template import PrepTemplate from qiita_db.util import ( get_mountpoint, get_visibilities, get_artifacts_information) -from qiita_db.software import Command, Parameters +from qiita_db.software import Command, Parameters, Software from qiita_db.processing_job import ProcessingJob PREP_TEMPLATE_KEY_FORMAT = 'prep_template_%s' @@ -174,10 +173,15 @@ def artifact_post_req(user_id, filepaths, artifact_type, name, if access_error: return access_error + user = User(user_id) + if artifact_id: # if the artifact id has been provided, import the artifact - job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id) - is_qiita_job = False + qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') + cmd = qiita_plugin.get_command('copy_artifact') + params = Parameters.load(cmd, values_dict={'artifact': artifact_id, + 'prep_template': prep.id}) + job = ProcessingJob.create(user, params) else: uploads_path = get_mountpoint('uploads')[0][1] path_builder = partial(join, uploads_path, str(study_id)) @@ -210,18 +214,18 @@ def artifact_post_req(user_id, filepaths, artifact_type, name, command = Command.get_validator(artifact_type) job = ProcessingJob.create( - User(user_id), + user, Parameters.load(command, values_dict={ 'template': prep_template_id, 'files': dumps(cleaned_filepaths), 'artifact_type': artifact_type })) - job.submit() - job_id = job.id - is_qiita_job = True + + # Submit the job + job.submit() r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, - dumps({'job_id': job_id, 'is_qiita_job': is_qiita_job})) + dumps({'job_id': job.id, 'is_qiita_job': True})) return {'status': 'success', 'message': ''} From f2a12c6a9a5d6b5ca65965de261db1e968264859 Mon Sep 17 00:00:00 2001 From: Jose Navas Date: Thu, 31 Aug 2017 14:00:25 -0700 Subject: [PATCH 8/8] Adding success test --- qiita_ware/test/test_private_plugin.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/qiita_ware/test/test_private_plugin.py b/qiita_ware/test/test_private_plugin.py index 2eb601a62..fc26c5983 100644 --- a/qiita_ware/test/test_private_plugin.py +++ b/qiita_ware/test/test_private_plugin.py @@ -8,10 +8,14 @@ from unittest import TestCase, main +import pandas as pd + from qiita_core.util import qiita_test_checker from qiita_db.software import Software, Parameters from qiita_db.processing_job import ProcessingJob from qiita_db.user import User +from qiita_db.study import Study +from qiita_db.metadata_template.prep_template import PrepTemplate from qiita_ware.private_plugin import private_task @@ -27,6 +31,7 @@ def _create_job(self, cmd, values_dict): return job def test_copy_artifact(self): + # Failure test job = self._create_job('copy_artifact', {'artifact': 1, 'prep_template': 1}) @@ -35,6 +40,24 @@ def test_copy_artifact(self): self.assertIn("Prep template 1 already has an artifact associated", job.log.msg) + # Success test + metadata_dict = { + 'SKB8.640193': {'center_name': 'ANL', + 'primer': 'GTGCCAGCMGCCGCGGTAA', + 'barcode': 'GTCCGCAAGTTA', + 'run_prefix': "s_G1_L001_sequences", + 'platform': 'ILLUMINA', + 'instrument_model': 'Illumina MiSeq', + 'library_construction_protocol': 'AAAA', + 'experiment_design_description': 'BBBB'}} + metadata = pd.DataFrame.from_dict(metadata_dict, orient='index', + dtype=str) + prep = PrepTemplate.create(metadata, Study(1), "16S") + job = self._create_job('copy_artifact', {'artifact': 1, + 'prep_template': prep.id}) + private_task(job.id) + self.assertEqual(job.status, 'success') + if __name__ == '__main__': main()