From 9f5ba452ebd896823b26bce6fb5a083f963ecf1c Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 2 Sep 2020 18:21:22 -0600 Subject: [PATCH 1/9] fix #2868 --- qiita_db/analysis.py | 44 +++++++++++++-------------------- qiita_db/sql_connection.py | 18 ++++++++++++++ qiita_db/test/test_meta_util.py | 24 ++++++------------ 3 files changed, 43 insertions(+), 43 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index c41a6ac0e..453887f77 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -355,11 +355,9 @@ def description(self, description): QiitaDBStatusError Analysis is public """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET description = %s - WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [description, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET description = %s + WHERE analysis_id = %s""".format(self._table) + qdb.sql_connection.single_query(sql, [description, self._id]) @property def samples(self): @@ -513,11 +511,9 @@ def pmid(self, pmid): ----- An analysis should only ever have one PMID attached to it. """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET pmid = %s - WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [pmid, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET pmid = %s + WHERE analysis_id = %s""".format(self._table) + qdb.sql_connection.single_query(sql, [pmid, self._id]) @property def can_be_publicized(self): @@ -618,13 +614,11 @@ def set_error(self, error_msg): error_msg : str The error message """ - with qdb.sql_connection.TRN: - le = qdb.logger.LogEntry.create('Runtime', error_msg) - sql = """UPDATE qiita.analysis - SET logging_id = %s - WHERE analysis_id = %s""" - qdb.sql_connection.TRN.add(sql, [le.id, self.id]) - qdb.sql_connection.TRN.execute() + le = qdb.logger.LogEntry.create('Runtime', error_msg) + sql = """UPDATE qiita.analysis + SET logging_id = %s + WHERE analysis_id = %s""" + qdb.sql_connection.single_query(sql, [le.id, self.id]) def has_access(self, user): """Returns whether the given user has access to the analysis @@ -696,11 +690,9 @@ def share(self, user): if user.id == self.owner or user.id in self.shared_with: return - with qdb.sql_connection.TRN: - sql = """INSERT INTO qiita.analysis_users (analysis_id, email) - VALUES (%s, %s)""" - qdb.sql_connection.TRN.add(sql, [self._id, user.id]) - qdb.sql_connection.TRN.execute() + sql = """INSERT INTO qiita.analysis_users (analysis_id, email) + VALUES (%s, %s)""" + qdb.sql_connection.single_query(sql, [self._id, user.id]) def unshare(self, user): """Unshare the analysis with another user @@ -710,11 +702,9 @@ def unshare(self, user): user: User object The user to unshare the analysis with """ - with qdb.sql_connection.TRN: - sql = """DELETE FROM qiita.analysis_users - WHERE analysis_id = %s AND email = %s""" - qdb.sql_connection.TRN.add(sql, [self._id, user.id]) - qdb.sql_connection.TRN.execute() + sql = """DELETE FROM qiita.analysis_users + WHERE analysis_id = %s AND email = %s""" + qdb.sql_connection.single_query(sql, [self._id, user.id]) def _lock_samples(self): """Only dflt analyses can have samples added/removed diff --git a/qiita_db/sql_connection.py b/qiita_db/sql_connection.py index 5955d094e..3725ce94f 100644 --- a/qiita_db/sql_connection.py +++ b/qiita_db/sql_connection.py @@ -493,6 +493,24 @@ def add_post_rollback_func(self, func, *args, **kwargs): TRNADMIN = Transaction(admin=True) +def single_query(sql, parameters=None): + """Opens, adds and executes "SQL" as a single transaction + + Parameters + ---------- + sql : str + The SQL to execute + parameters: object, optional + The object of parameters to pass to the TRN.add command + """ + with TRN: + if parameters: + TRN.add(sql, parameters) + else: + TRN.add(sql) + TRN.execute() + + def create_new_transaction(): """Creates a new global transaction diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index 740a41ceb..cc07d7f2d 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -34,17 +34,13 @@ def tearDown(self): def _set_artifact_private(self): id_status = qdb.util.convert_to_id('private', 'visibility') - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE qiita.artifact SET visibility_id = %d" % id_status) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.single_query( + "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def _set_artifact_public(self): id_status = qdb.util.convert_to_id('public', 'visibility') - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE qiita.artifact SET visibility_id = %d" % id_status) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.single_query( + "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def test_validate_filepath_access_by_user(self): self._set_artifact_private() @@ -108,10 +104,8 @@ def test_validate_filepath_access_by_user(self): self.assertTrue(obs) # test in case there is a prep template that failed - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "INSERT INTO qiita.prep_template (data_type_id) VALUES (2)") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.single_query( + "INSERT INTO qiita.prep_template (data_type_id) VALUES (2)") for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: obs = qdb.meta_util.validate_filepath_access_by_user(user, i) if i < 3: @@ -467,10 +461,8 @@ def test_generate_biom_and_metadata_release(self): self.assertEqual(txt_obs, txt_exp) # returning configuration - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE settings SET base_data_dir = '%s'" % obdr) - bdr = qdb.sql_connection.TRN.execute() + qdb.sql_connection.single_query( + "UPDATE settings SET base_data_dir = '%s'" % obdr) # testing public/default release qdb.meta_util.generate_biom_and_metadata_release() From 6831e5c674c62473373b3de70a21c2395fe21dde Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Wed, 2 Sep 2020 18:59:26 -0600 Subject: [PATCH 2/9] 2nd round --- qiita_db/analysis.py | 10 +++--- qiita_db/artifact.py | 32 ++++++++----------- qiita_db/download_link.py | 12 +++---- qiita_db/logger.py | 23 ++++++------- qiita_db/meta_util.py | 8 ++--- .../base_metadata_template.py | 21 ++++++------ qiita_db/metadata_template/prep_template.py | 31 ++++++++---------- qiita_db/sql_connection.py | 2 +- qiita_db/test/test_meta_util.py | 8 ++--- 9 files changed, 62 insertions(+), 85 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index 453887f77..f31ee04a7 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -357,7 +357,7 @@ def description(self, description): """ sql = """UPDATE qiita.{0} SET description = %s WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.single_query(sql, [description, self._id]) + qdb.sql_connection.encapsulated_query(sql, [description, self._id]) @property def samples(self): @@ -513,7 +513,7 @@ def pmid(self, pmid): """ sql = """UPDATE qiita.{0} SET pmid = %s WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.single_query(sql, [pmid, self._id]) + qdb.sql_connection.encapsulated_query(sql, [pmid, self._id]) @property def can_be_publicized(self): @@ -618,7 +618,7 @@ def set_error(self, error_msg): sql = """UPDATE qiita.analysis SET logging_id = %s WHERE analysis_id = %s""" - qdb.sql_connection.single_query(sql, [le.id, self.id]) + qdb.sql_connection.encapsulated_query(sql, [le.id, self.id]) def has_access(self, user): """Returns whether the given user has access to the analysis @@ -692,7 +692,7 @@ def share(self, user): sql = """INSERT INTO qiita.analysis_users (analysis_id, email) VALUES (%s, %s)""" - qdb.sql_connection.single_query(sql, [self._id, user.id]) + qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) def unshare(self, user): """Unshare the analysis with another user @@ -704,7 +704,7 @@ def unshare(self, user): """ sql = """DELETE FROM qiita.analysis_users WHERE analysis_id = %s AND email = %s""" - qdb.sql_connection.single_query(sql, [self._id, user.id]) + qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) def _lock_samples(self): """Only dflt analyses can have samples added/removed diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 900c80dd0..4d00ef509 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -388,8 +388,7 @@ def _associate_with_analysis(instance, analysis_id): (analysis_id, artifact_id) VALUES (%s, %s)""" sql_args = [analysis_id, instance.id] - qdb.sql_connection.TRN.add(sql, sql_args) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query(sql, sql_args) with qdb.sql_connection.TRN: if parents: @@ -673,12 +672,10 @@ def name(self, value): ValueError If `value` contains more than 35 chars """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.artifact - SET name = %s - WHERE artifact_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.artifact + SET name = %s + WHERE artifact_id = %s""" + qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) @property def timestamp(self): @@ -751,8 +748,7 @@ def _set_visibility(self, value): sql = """UPDATE qiita.artifact SET visibility_id = %s WHERE artifact_id IN %s""" - qdb.sql_connection.TRN.add(sql, [vis_id, tuple(ids)]) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.TRN.encapsulated_query(sql, [vis_id, tuple(ids)]) @visibility.setter def visibility(self, value): @@ -989,15 +985,13 @@ def is_submitted_to_vamps(self, value): QiitaDBOperationNotPermittedError If the artifact cannot be submitted to VAMPS """ - with qdb.sql_connection.TRN: - if not self.can_be_submitted_to_vamps: - raise qdb.exceptions.QiitaDBOperationNotPermittedError( - "Artifact %s cannot be submitted to VAMPS" % self.id) - sql = """UPDATE qiita.artifact - SET submitted_to_vamps = %s - WHERE artifact_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + if not self.can_be_submitted_to_vamps: + raise qdb.exceptions.QiitaDBOperationNotPermittedError( + "Artifact %s cannot be submitted to VAMPS" % self.id) + sql = """UPDATE qiita.artifact + SET submitted_to_vamps = %s + WHERE artifact_id = %s""" + qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) @property def filepaths(self): diff --git a/qiita_db/download_link.py b/qiita_db/download_link.py index 92f9efebd..f191444c7 100644 --- a/qiita_db/download_link.py +++ b/qiita_db/download_link.py @@ -72,10 +72,8 @@ def delete(cls, jti): jti : object The jwt token identifier """ - with qdb.sql_connection.TRN: - sql = """DELETE FROM qiita.{0} WHERE jti=%s""".format(cls._table) - qdb.sql_connection.TRN.add(sql, [jti]) - qdb.sql_connection.TRN.execute() + sql = """DELETE FROM qiita.{0} WHERE jti=%s""".format(cls._table) + qdb.sql_connection.encapsulated_query(sql, [jti]) @classmethod def exists(cls, jti): @@ -98,10 +96,8 @@ def delete_expired(cls): r"""Deletes all expired download links""" now = datetime.now(timezone.utc) - with qdb.sql_connection.TRN: - sql = """DELETE FROM qiita.{0} WHERE exp<%s""".format(cls._table) - qdb.sql_connection.TRN.add(sql, [now]) - qdb.sql_connection.TRN.execute() + sql = """DELETE FROM qiita.{0} WHERE exp<%s""".format(cls._table) + qdb.sql_connection.TRN.encapsulated_query(sql, [now]) @classmethod def get(cls, jti): diff --git a/qiita_db/logger.py b/qiita_db/logger.py index 5e83c49df..484145ae4 100644 --- a/qiita_db/logger.py +++ b/qiita_db/logger.py @@ -181,12 +181,9 @@ def msg(self): def clear_info(self): """Resets the list of info dicts to be an empty list """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{} SET information = %s - WHERE logging_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [dumps([]), self.id]) - - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{} SET information = %s + WHERE logging_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [dumps([]), self.id]) def add_info(self, info): """Adds new information to the info associated with this LogEntry @@ -201,12 +198,10 @@ def add_info(self, info): - When `info` is added, keys can be of any type, but upon retrieval, they will be of type str """ - with qdb.sql_connection.TRN: - current_info = self.info - current_info.append(info) - new_info = dumps(current_info) + current_info = self.info + current_info.append(info) + new_info = dumps(current_info) - sql = """UPDATE qiita.{} SET information = %s - WHERE logging_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [new_info, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{} SET information = %s + WHERE logging_id = %s""".format(self._table) + qdb.sql_connection.TRN.encapsulated_query(sql, [new_info, self.id]) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 721c732e3..2e70d8aa7 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -335,11 +335,9 @@ def sizeof_fmt(value, position): # preparing vals to insert into DB vals = dumps(dict([x[:-1] for x in vals])) - with qdb.sql_connection.TRN: - sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp) - VALUES (%s, NOW())""" - qdb.sql_connection.TRN.add(sql, [vals]) - qdb.sql_connection.TRN.execute() + sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp) + VALUES (%s, NOW())""" + qdb.sql_connection.TRN.encapsulated_query(sql, [vals]) return missing_files diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index 51ea5f0a1..d24e48682 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -279,19 +279,18 @@ def setitem(self, column, value): QiitaDBColumnError If the column does not exist in the table """ - with qdb.sql_connection.TRN: - # Check if the column exist in the table - if column not in self._get_categories(): - raise qdb.exceptions.QiitaDBColumnError( - "Column %s does not exist in %s" % - (column, self._dynamic_table)) + # Check if the column exist in the table + if column not in self._get_categories(): + raise qdb.exceptions.QiitaDBColumnError( + "Column %s does not exist in %s" % + (column, self._dynamic_table)) - sql = """UPDATE qiita.{0} - SET sample_values = sample_values || %s - WHERE sample_id = %s""".format(self._dynamic_table) + sql = """UPDATE qiita.{0} + SET sample_values = sample_values || %s + WHERE sample_id = %s""".format(self._dynamic_table) - qdb.sql_connection.TRN.add(sql, [dumps({column: value}), self.id]) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.TRN.encapsulated_query( + sql, [dumps({column: value}), self.id]) def __setitem__(self, column, value): r"""Sets the metadata value for the category `column` diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index ecea21ac1..43f8a3319 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -453,14 +453,13 @@ def investigation_type(self, investigation_type): QiitaDBColumnError If the investigation type is not a valid ENA ontology """ - with qdb.sql_connection.TRN: - if investigation_type is not None: - self.validate_investigation_type(investigation_type) + if investigation_type is not None: + self.validate_investigation_type(investigation_type) - sql = """UPDATE qiita.prep_template SET investigation_type = %s - WHERE {0} = %s""".format(self._id_column) - qdb.sql_connection.TRN.add(sql, [investigation_type, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.prep_template SET investigation_type = %s + WHERE {0} = %s""".format(self._id_column) + qdb.sql_connection.encapsulated_query( + sql, [investigation_type, self.id]) @property def study_id(self): @@ -494,11 +493,9 @@ def deprecated(self, deprecated): deprecated : bool If the prep info file is deprecated """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.prep_template SET deprecated = %s - WHERE {0} = %s""".format(self._id_column) - qdb.sql_connection.TRN.add(sql, [deprecated, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.prep_template SET deprecated = %s + WHERE {0} = %s""".format(self._id_column) + qdb.sql_connection.TRN.encapsulated_query(sql, [deprecated, self.id]) def generate_files(self, samples=None, columns=None): r"""Generates all the files that contain data from this template @@ -761,9 +758,7 @@ def name(self): @name.setter def name(self, value): """Changes the name of the prep template""" - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.prep_template - SET name = %s - WHERE prep_template_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.prep_template + SET name = %s + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) diff --git a/qiita_db/sql_connection.py b/qiita_db/sql_connection.py index 3725ce94f..ad1f0435a 100644 --- a/qiita_db/sql_connection.py +++ b/qiita_db/sql_connection.py @@ -493,7 +493,7 @@ def add_post_rollback_func(self, func, *args, **kwargs): TRNADMIN = Transaction(admin=True) -def single_query(sql, parameters=None): +def encapsulated_query(sql, parameters=None): """Opens, adds and executes "SQL" as a single transaction Parameters diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index cc07d7f2d..b24117c43 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -34,12 +34,12 @@ def tearDown(self): def _set_artifact_private(self): id_status = qdb.util.convert_to_id('private', 'visibility') - qdb.sql_connection.single_query( + qdb.sql_connection.encapsulated_query( "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def _set_artifact_public(self): id_status = qdb.util.convert_to_id('public', 'visibility') - qdb.sql_connection.single_query( + qdb.sql_connection.encapsulated_query( "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def test_validate_filepath_access_by_user(self): @@ -104,7 +104,7 @@ def test_validate_filepath_access_by_user(self): self.assertTrue(obs) # test in case there is a prep template that failed - qdb.sql_connection.single_query( + qdb.sql_connection.encapsulated_query( "INSERT INTO qiita.prep_template (data_type_id) VALUES (2)") for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: obs = qdb.meta_util.validate_filepath_access_by_user(user, i) @@ -461,7 +461,7 @@ def test_generate_biom_and_metadata_release(self): self.assertEqual(txt_obs, txt_exp) # returning configuration - qdb.sql_connection.single_query( + qdb.sql_connection.encapsulated_query( "UPDATE settings SET base_data_dir = '%s'" % obdr) # testing public/default release From eb7b52e1809b381d2a3c4cd35ad504c9c9b0f13b Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 06:33:02 -0600 Subject: [PATCH 3/9] fix errors --- qiita_db/artifact.py | 6 +++--- qiita_db/download_link.py | 2 +- qiita_db/logger.py | 2 +- qiita_db/meta_util.py | 2 +- .../base_metadata_template.py | 2 +- qiita_db/metadata_template/prep_template.py | 4 ++-- qiita_db/ontology.py | 18 ++++++++---------- 7 files changed, 17 insertions(+), 19 deletions(-) diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index 4d00ef509..c60e4295b 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -675,7 +675,7 @@ def name(self, value): sql = """UPDATE qiita.artifact SET name = %s WHERE artifact_id = %s""" - qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) @property def timestamp(self): @@ -748,7 +748,7 @@ def _set_visibility(self, value): sql = """UPDATE qiita.artifact SET visibility_id = %s WHERE artifact_id IN %s""" - qdb.sql_connection.TRN.encapsulated_query(sql, [vis_id, tuple(ids)]) + qdb.sql_connection.encapsulated_query(sql, [vis_id, tuple(ids)]) @visibility.setter def visibility(self, value): @@ -991,7 +991,7 @@ def is_submitted_to_vamps(self, value): sql = """UPDATE qiita.artifact SET submitted_to_vamps = %s WHERE artifact_id = %s""" - qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) @property def filepaths(self): diff --git a/qiita_db/download_link.py b/qiita_db/download_link.py index f191444c7..245aba893 100644 --- a/qiita_db/download_link.py +++ b/qiita_db/download_link.py @@ -97,7 +97,7 @@ def delete_expired(cls): now = datetime.now(timezone.utc) sql = """DELETE FROM qiita.{0} WHERE exp<%s""".format(cls._table) - qdb.sql_connection.TRN.encapsulated_query(sql, [now]) + qdb.sql_connection.encapsulated_query(sql, [now]) @classmethod def get(cls, jti): diff --git a/qiita_db/logger.py b/qiita_db/logger.py index 484145ae4..bda60b336 100644 --- a/qiita_db/logger.py +++ b/qiita_db/logger.py @@ -204,4 +204,4 @@ def add_info(self, info): sql = """UPDATE qiita.{} SET information = %s WHERE logging_id = %s""".format(self._table) - qdb.sql_connection.TRN.encapsulated_query(sql, [new_info, self.id]) + qdb.sql_connection.encapsulated_query(sql, [new_info, self.id]) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 2e70d8aa7..6e031c801 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -337,7 +337,7 @@ def sizeof_fmt(value, position): vals = dumps(dict([x[:-1] for x in vals])) sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp) VALUES (%s, NOW())""" - qdb.sql_connection.TRN.encapsulated_query(sql, [vals]) + qdb.sql_connection.encapsulated_query(sql, [vals]) return missing_files diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index d24e48682..f871a9c3a 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -289,7 +289,7 @@ def setitem(self, column, value): SET sample_values = sample_values || %s WHERE sample_id = %s""".format(self._dynamic_table) - qdb.sql_connection.TRN.encapsulated_query( + qdb.sql_connection.encapsulated_query( sql, [dumps({column: value}), self.id]) def __setitem__(self, column, value): diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 43f8a3319..42534c78a 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -495,7 +495,7 @@ def deprecated(self, deprecated): """ sql = """UPDATE qiita.prep_template SET deprecated = %s WHERE {0} = %s""".format(self._id_column) - qdb.sql_connection.TRN.encapsulated_query(sql, [deprecated, self.id]) + qdb.sql_connection.encapsulated_query(sql, [deprecated, self.id]) def generate_files(self, samples=None, columns=None): r"""Generates all the files that contain data from this template @@ -761,4 +761,4 @@ def name(self, value): sql = """UPDATE qiita.prep_template SET name = %s WHERE prep_template_id = %s""" - qdb.sql_connection.TRN.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) diff --git a/qiita_db/ontology.py b/qiita_db/ontology.py index 63f37c8d6..0ba31d7f6 100644 --- a/qiita_db/ontology.py +++ b/qiita_db/ontology.py @@ -77,16 +77,14 @@ def add_user_defined_term(self, term): term : str New user defined term to add into a given ontology """ - with qdb.sql_connection.TRN: - # we don't need to add an existing term - terms = self.user_defined_terms + self.terms - - if term not in terms: - sql = """INSERT INTO qiita.term - (ontology_id, term, user_defined) - VALUES (%s, %s, true);""" - qdb.sql_connection.TRN.add(sql, [self.id, term]) - qdb.sql_connection.TRN.execute() + # we don't need to add an existing term + terms = self.user_defined_terms + self.terms + + if term not in terms: + sql = """INSERT INTO qiita.term + (ontology_id, term, user_defined) + VALUES (%s, %s, true);""" + qdb.sql_connection.encapsulated_query(sql, [self.id, term]) def term_type(self, term): """Get the type of a given ontology term From 99a7fd18a8d3cfcb343a6fa7ac26c7a6eec9ba6a Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 07:23:28 -0600 Subject: [PATCH 4/9] more changes --- qiita_db/portal.py | 60 +++++++------- qiita_db/processing_job.py | 40 ++++------ qiita_db/software.py | 28 +++---- qiita_db/sql_connection.py | 2 +- qiita_db/study.py | 114 +++++++++++---------------- qiita_db/test/test_analysis.py | 20 ++--- qiita_db/test/test_archive.py | 7 +- qiita_db/test/test_commands.py | 28 +++---- qiita_db/test/test_ontology.py | 6 +- qiita_db/test/test_processing_job.py | 25 +++--- qiita_db/test/test_software.py | 26 +++--- qiita_db/test/test_study.py | 10 +-- qiita_db/test/test_user.py | 4 +- qiita_db/test/test_util.py | 44 ++++------- qiita_db/user.py | 30 +++---- 15 files changed, 182 insertions(+), 262 deletions(-) diff --git a/qiita_db/portal.py b/qiita_db/portal.py index d9cb5ae55..435d437d3 100644 --- a/qiita_db/portal.py +++ b/qiita_db/portal.py @@ -72,38 +72,36 @@ def create(cls, portal, desc): QiitaDBDuplicateError Portal name already exists """ - with qdb.sql_connection.TRN: - if cls.exists(portal): - raise qdb.exceptions.QiitaDBDuplicateError("Portal", portal) - - # Add portal and default analyses for all users - sql = """DO $do$ - DECLARE - pid bigint; - eml varchar; - aid bigint; - BEGIN - INSERT INTO qiita.portal_type (portal, portal_description) - VALUES (%s, %s) - RETURNING portal_type_id INTO pid; - - FOR eml IN - SELECT email FROM qiita.qiita_user - LOOP - INSERT INTO qiita.analysis - (email, name, description, dflt) - VALUES (eml, eml || '-dflt', 'dflt', true) - RETURNING analysis_id INTO aid; - - INSERT INTO qiita.analysis_portal - (analysis_id, portal_type_id) - VALUES (aid, pid); - END LOOP; - END $do$;""" - qdb.sql_connection.TRN.add(sql, [portal, desc]) - qdb.sql_connection.TRN.execute() + if cls.exists(portal): + raise qdb.exceptions.QiitaDBDuplicateError("Portal", portal) + + # Add portal and default analyses for all users + sql = """DO $do$ + DECLARE + pid bigint; + eml varchar; + aid bigint; + BEGIN + INSERT INTO qiita.portal_type (portal, portal_description) + VALUES (%s, %s) + RETURNING portal_type_id INTO pid; + + FOR eml IN + SELECT email FROM qiita.qiita_user + LOOP + INSERT INTO qiita.analysis + (email, name, description, dflt) + VALUES (eml, eml || '-dflt', 'dflt', true) + RETURNING analysis_id INTO aid; + + INSERT INTO qiita.analysis_portal + (analysis_id, portal_type_id) + VALUES (aid, pid); + END LOOP; + END $do$;""" + qdb.sql_connection.encapsulated_query(sql, [portal, desc]) - return cls(portal) + return cls(portal) @staticmethod def delete(portal): diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 58de9a8ef..448d8f4c0 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -814,12 +814,10 @@ def external_id(self, value): - If the current status of the job is 'running' and `value` is 'queued' """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.processing_job - SET external_job_id = %s - WHERE processing_job_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.processing_job + SET external_job_id = %s + WHERE processing_job_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) @property def release_validator_job(self): @@ -1305,13 +1303,11 @@ def _set_validator_jobs(self, validator_jobs): validator_jobs : list of ProcessingJob The validator_jobs for the current job """ - with qdb.sql_connection.TRN: - sql = """INSERT INTO qiita.processing_job_validator - (processing_job_id, validator_id) - VALUES (%s, %s)""" - sql_args = [[self.id, j.id] for j in validator_jobs] - qdb.sql_connection.TRN.add(sql, sql_args, many=True) - qdb.sql_connection.TRN.execute() + sql = """INSERT INTO qiita.processing_job_validator + (processing_job_id, validator_id) + VALUES (%s, %s)""" + sql_args = [[self.id, j.id] for j in validator_jobs] + qdb.sql_connection.encapsulated_query(sql, sql_args, many=True) def complete(self, success, artifacts_data=None, error=None): """Completes the job, either with a success or error status @@ -1475,16 +1471,14 @@ def step(self, value): qiita_db.exceptions.QiitaDBOperationNotPermittedError If the status of the job is not 'running' """ - with qdb.sql_connection.TRN: - if self.status != 'running': - raise qdb.exceptions.QiitaDBOperationNotPermittedError( - "Cannot change the step of a job whose status is not " - "'running'") - sql = """UPDATE qiita.processing_job - SET step = %s - WHERE processing_job_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + if self.status != 'running': + raise qdb.exceptions.QiitaDBOperationNotPermittedError( + "Cannot change the step of a job whose status is not " + "'running'") + sql = """UPDATE qiita.processing_job + SET step = %s + WHERE processing_job_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) @property def children(self): diff --git a/qiita_db/software.py b/qiita_db/software.py index 171490597..5231e5f9d 100644 --- a/qiita_db/software.py +++ b/qiita_db/software.py @@ -632,12 +632,10 @@ def active(self): def activate(self): """Activates the command""" - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software_command - SET active = %s - WHERE command_id = %s""" - qdb.sql_connection.TRN.add(sql, [True, self.id]) - return qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software_command + SET active = %s + WHERE command_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [True, self.id]) @property def analysis_only(self): @@ -1277,11 +1275,9 @@ def deprecated(self, deprecate): deprecate : bool New software deprecate value """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software SET deprecated = %s - WHERE software_id = %s""" - qdb.sql_connection.TRN.add(sql, [deprecate, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software SET deprecated = %s + WHERE software_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [deprecate, self._id]) @property def active(self): @@ -1299,12 +1295,10 @@ def active(self): def activate(self): """Activates the plugin""" - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software - SET active = %s - WHERE software_id = %s""" - qdb.sql_connection.TRN.add(sql, [True, self.id]) - return qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software + SET active = %s + WHERE software_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [True, self.id]) @property def client_id(self): diff --git a/qiita_db/sql_connection.py b/qiita_db/sql_connection.py index ad1f0435a..5e4de64d7 100644 --- a/qiita_db/sql_connection.py +++ b/qiita_db/sql_connection.py @@ -494,7 +494,7 @@ def add_post_rollback_func(self, func, *args, **kwargs): def encapsulated_query(sql, parameters=None): - """Opens, adds and executes "SQL" as a single transaction + """Opens, adds and executes sql as a single transaction Parameters ---------- diff --git a/qiita_db/study.py b/qiita_db/study.py index 77543854a..5e95f95fa 100644 --- a/qiita_db/study.py +++ b/qiita_db/study.py @@ -471,15 +471,13 @@ def insert_tags(cls, user, tags): tags : list of str The list of tags to add """ - with qdb.sql_connection.TRN: - email = user.email - sql = """INSERT INTO qiita.study_tags (email, study_tag) - SELECT %s, %s WHERE NOT EXISTS ( - SELECT 1 FROM qiita.study_tags WHERE study_tag = %s)""" - sql_args = [[email, tag, tag] for tag in tags] + email = user.email + sql = """INSERT INTO qiita.study_tags (email, study_tag) + SELECT %s, %s WHERE NOT EXISTS ( + SELECT 1 FROM qiita.study_tags WHERE study_tag = %s)""" + sql_args = [[email, tag, tag] for tag in tags] - qdb.sql_connection.TRN.add(sql, sql_args, many=True) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query(sql, sql_args, many=True) # --- Attributes --- @property @@ -506,11 +504,9 @@ def title(self, title): title : str The study title """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET study_title = %s - WHERE study_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [title, self._id]) - return qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET study_title = %s + WHERE study_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [title, self._id]) @property def notes(self): @@ -536,11 +532,9 @@ def notes(self, notes): notes : str The study notes """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET notes = %s - WHERE study_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [notes, self._id]) - return qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET notes = %s + WHERE study_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [notes, self._id]) @property def public_raw_download(self): @@ -566,11 +560,10 @@ def public_raw_download(self, public_raw_download): public_raw_download : bool The study public_raw_download """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET public_raw_download = %s - WHERE study_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [public_raw_download, self._id]) - return qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET public_raw_download = %s + WHERE study_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query( + sql, [public_raw_download, self._id]) @property def info(self): @@ -772,13 +765,10 @@ def specimen_id_column(self, value): " contain unique " "values.") - with qdb.sql_connection.TRN: - # Set the new ones - sql = """UPDATE qiita.study SET - specimen_id_column = %s - WHERE study_id = %s""" - qdb.sql_connection.TRN.add(sql, [value, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.study SET + specimen_id_column = %s + WHERE study_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [value, self._id]) @property def investigation(self): @@ -965,16 +955,14 @@ def ebi_study_accession(self, value): QiitDBError If the study already has an EBI study accession """ - with qdb.sql_connection.TRN: - if self.ebi_study_accession is not None: - raise qdb.exceptions.QiitaDBError( - "Study %s already has an EBI study accession" - % self.id) - sql = """UPDATE qiita.{} - SET ebi_study_accession = %s - WHERE study_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [value, self.id]) - qdb.sql_connection.TRN.execute() + if self.ebi_study_accession is not None: + raise qdb.exceptions.QiitaDBError( + "Study %s already has an EBI study accession" + % self.id) + sql = """UPDATE qiita.{} + SET ebi_study_accession = %s + WHERE study_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [value, self.id]) def _ebi_submission_jobs(self): """Helper code to avoid duplication""" @@ -1215,18 +1203,16 @@ def share(self, user): user: User object The user to share the study with """ - with qdb.sql_connection.TRN: - # Make sure the study is not already shared with the given user - if user in self.shared_with: - return - # Do not allow the study to be shared with the owner - if user == self.owner: - return - - sql = """INSERT INTO qiita.study_users (study_id, email) - VALUES (%s, %s)""" - qdb.sql_connection.TRN.add(sql, [self._id, user.id]) - qdb.sql_connection.TRN.execute() + # Make sure the study is not already shared with the given user + if user in self.shared_with: + return + # Do not allow the study to be shared with the owner + if user == self.owner: + return + + sql = """INSERT INTO qiita.study_users (study_id, email) + VALUES (%s, %s)""" + qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) def unshare(self, user): """Unshare the study with another user @@ -1236,11 +1222,9 @@ def unshare(self, user): user: User object The user to unshare the study with """ - with qdb.sql_connection.TRN: - sql = """DELETE FROM qiita.study_users - WHERE study_id = %s AND email = %s""" - qdb.sql_connection.TRN.add(sql, [self._id, user.id]) - qdb.sql_connection.TRN.execute() + sql = """DELETE FROM qiita.study_users + WHERE study_id = %s AND email = %s""" + qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) def update_tags(self, user, tags): """Sets the tags of the study @@ -1550,11 +1534,9 @@ def address(self, value): value : str New address for person """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET address = %s - WHERE study_person_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [value, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET address = %s + WHERE study_person_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [value, self._id]) @property def phone(self): @@ -1580,8 +1562,6 @@ def phone(self, value): value : str New phone number for person """ - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.{0} SET phone = %s - WHERE study_person_id = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [value, self._id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.{0} SET phone = %s + WHERE study_person_id = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [value, self._id]) diff --git a/qiita_db/test/test_analysis.py b/qiita_db/test/test_analysis.py index 8e5472a0a..cf715e142 100644 --- a/qiita_db/test/test_analysis.py +++ b/qiita_db/test/test_analysis.py @@ -555,12 +555,10 @@ def test_build_files_post_processing_cmd(self): # convert to json representation and store in PostgreSQL results = dumps(results) - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software_command - SET post_processing_cmd = %s - WHERE command_id = %s""" - qdb.sql_connection.TRN.add(sql, [results, cmd_id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software_command + SET post_processing_cmd = %s + WHERE command_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [results, cmd_id]) # create a sample analysis and run build_files on it. analysis = self._create_analyses_with_samples() @@ -581,12 +579,10 @@ def test_build_files_post_processing_cmd(self): self.assertEqual(obs, exp) # cleanup (assume command was NULL previously) - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software_command - SET post_processing_cmd = NULL - WHERE command_id = %s""" - qdb.sql_connection.TRN.add(sql, [cmd_id]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software_command + SET post_processing_cmd = NULL + WHERE command_id = %s""" + qdb.sql_connection.encapsulated_query(sql, [cmd_id]) def test_build_files_merge_duplicated_sample_ids(self): user = qdb.user.User("demo@microbio.me") diff --git a/qiita_db/test/test_archive.py b/qiita_db/test/test_archive.py index 2c1f6e473..54c4b1a07 100644 --- a/qiita_db/test/test_archive.py +++ b/qiita_db/test/test_archive.py @@ -28,10 +28,9 @@ def test_insert_from_biom_and_retrieve_feature_values(self): # 7 - to test error due to not filepath biom aid = 7 - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add("DELETE FROM qiita.artifact_filepath " - "WHERE artifact_id = %d" % aid) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "DELETE FROM qiita.artifact_filepath " + "WHERE artifact_id = %d" % aid) with self.assertRaises(ValueError) as err: qdb.archive.Archive.insert_from_artifact( qdb.artifact.Artifact(aid), {}) diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 4960de0ec..4f08b8ea3 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -276,10 +276,8 @@ def _assert_current_patch(self, patch_to_check): def test_unpatched(self): """Test patching from unpatched state""" # Reset the settings table to the unpatched state - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE settings SET current_patch = 'unpatched'") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE settings SET current_patch = 'unpatched'") self._assert_current_patch('unpatched') qdb.environment_manager.patch(self.patches_dir) @@ -289,10 +287,8 @@ def test_unpatched(self): def test_skip_patch(self): """Test patching from a patched state""" - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE settings SET current_patch = '2.sql'") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE settings SET current_patch = '2.sql'") self._assert_current_patch('2.sql') # If it tried to apply patch 2.sql again, this will error @@ -306,10 +302,8 @@ def test_skip_patch(self): def test_nonexistent_patch(self): """Test case where current patch does not exist""" - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE settings SET current_patch = 'nope.sql'") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE settings SET current_patch = 'nope.sql'") self._assert_current_patch('nope.sql') with self.assertRaises(RuntimeError): @@ -322,10 +316,8 @@ def test_python_patch(self): f.write(PY_PATCH) # Reset the settings table to the unpatched state - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE settings SET current_patch = 'unpatched'") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE settings SET current_patch = 'unpatched'") self._assert_current_patch('unpatched') @@ -491,12 +483,12 @@ def test_update_artifact_from_cmd(self): PY_PATCH = """ from qiita_db.study import Study -from qiita_db.sql_connection import TRN +from qiita_db.sql_connection import encapsulated_query study = Study(1) with TRN: sql = "INSERT INTO qiita.patchtest10 (testing) VALUES (%s)" - TRN.add(sql, [[study.id], [study.id*100]], many=True) + encapsulated_query(sql, [[study.id], [study.id*100]], many=True) TRN.execute() """ diff --git a/qiita_db/test/test_ontology.py b/qiita_db/test/test_ontology.py index e0e42951a..9286d752f 100644 --- a/qiita_db/test/test_ontology.py +++ b/qiita_db/test/test_ontology.py @@ -18,10 +18,8 @@ def setUp(self): self.ontology = qdb.ontology.Ontology(999999999) def _remove_term(self, term): - with qdb.sql_connection.TRN: - sql = "DELETE FROM qiita.term WHERE ontology_id = %s AND term = %s" - qdb.sql_connection.TRN.add(sql, [self.ontology.id, term]) - qdb.sql_connection.TRN.execute() + sql = "DELETE FROM qiita.term WHERE ontology_id = %s AND term = %s" + qdb.sql_connection.encapsulated_query(sql, [self.ontology.id, term]) def testConvertToID(self): self.assertEqual(qdb.util.convert_to_id('ENA', 'ontology'), 999999999) diff --git a/qiita_db/test/test_processing_job.py b/qiita_db/test/test_processing_job.py index 28e89b150..c140baf5c 100644 --- a/qiita_db/test/test_processing_job.py +++ b/qiita_db/test/test_processing_job.py @@ -488,13 +488,12 @@ def test_complete_success(self): # parameter information of the command - since the ones existing # in the database currently do not require using any input parameter # to name the output artifact - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.command_parameter - SET name_order = %s - WHERE command_parameter_id = %s""" - # Hard-coded values; 19 -> barcode_type, 20 -> max_barcode_errors - qdb.sql_connection.TRN.add(sql, [[1, 19], [2, 20]], many=True) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.command_parameter + SET name_order = %s + WHERE command_parameter_id = %s""" + # Hard-coded values; 19 -> barcode_type, 20 -> max_barcode_errors + qdb.sql_connection.encapsulated_query( + sql, [[1, 19], [2, 20]], many=True) fd, fp = mkstemp(suffix='_table.biom') self._clean_up_files.append(fp) @@ -792,13 +791,11 @@ def test_get_resource_allocation_info(self): # helper to set memory allocations easier def _set_allocation(memory): - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.processing_job_resource_allocation - SET allocation = '{0}' - WHERE name = 'Split libraries FASTQ'""".format( - '-q qiita -l mem=%s' % memory) - qdb.sql_connection.TRN.add(sql) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.processing_job_resource_allocation + SET allocation = '{0}' + WHERE name = 'Split libraries FASTQ'""".format( + '-q qiita -l mem=%s' % memory) + qdb.sql_connection.encapsulated_query(sql) # let's start with something simple, samples*1000 # 27*1000 ~ 27000 diff --git a/qiita_db/test/test_software.py b/qiita_db/test/test_software.py index 022fd0889..bef007308 100644 --- a/qiita_db/test/test_software.py +++ b/qiita_db/test/test_software.py @@ -142,12 +142,10 @@ def test_post_processing_cmd(self): results = dumps(results) # modify table directly, in order to test method - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software_command - SET post_processing_cmd = %s - WHERE command_id = 1""" - qdb.sql_connection.TRN.add(sql, [results]) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software_command + SET post_processing_cmd = %s + WHERE command_id = 1""" + qdb.sql_connection.encapsulated_query(sql, [results]) results = qdb.software.Command(1).post_processing_cmd @@ -159,12 +157,10 @@ def test_post_processing_cmd(self): self.assertEqual(results['script_params'], {'a': 'A', 'b': 'B'}) # clean up table - with qdb.sql_connection.TRN: - sql = """UPDATE qiita.software_command - SET post_processing_cmd = NULL - WHERE command_id = 1""" - qdb.sql_connection.TRN.add(sql) - qdb.sql_connection.TRN.execute() + sql = """UPDATE qiita.software_command + SET post_processing_cmd = NULL + WHERE command_id = 1""" + qdb.sql_connection.encapsulated_query(sql) def test_description(self): self.assertEqual( @@ -478,10 +474,8 @@ def test_iter(self): '-q qiita -l nodes=1:ppn=5 -l pmem=8gb -l walltime=168:00:00') # delete allocations to test errors - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "DELETE FROM qiita.processing_job_resource_allocation") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "DELETE FROM qiita.processing_job_resource_allocation") with self.assertRaisesRegex(ValueError, "Could not match 'Split " "libraries' to a resource allocation!"): diff --git a/qiita_db/test/test_study.py b/qiita_db/test/test_study.py index 5a3cfe54b..befca5792 100644 --- a/qiita_db/test/test_study.py +++ b/qiita_db/test/test_study.py @@ -204,10 +204,8 @@ def _change_processed_data_status(self, new_status): # Change the status of the studies by changing the status of their # artifacts id_status = qdb.util.convert_to_id(new_status, 'visibility') - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE qiita.artifact SET visibility_id = %s", (id_status,)) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE qiita.artifact SET visibility_id = %s", (id_status,)) def test_get_info(self): # Test get all info for single study @@ -333,9 +331,7 @@ def test_public_raw_download(self): def test_share(self): # Clear all sharing associations self._change_processed_data_status('sandbox') - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add("delete from qiita.study_users") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query("delete from qiita.study_users") self.assertEqual(self.study.shared_with, []) # Try to share with the owner, which should not work diff --git a/qiita_db/test/test_user.py b/qiita_db/test/test_user.py index dc40a3d9b..c72e7df64 100644 --- a/qiita_db/test/test_user.py +++ b/qiita_db/test/test_user.py @@ -295,9 +295,7 @@ def test_verify_code(self): user_verify_code='verifycode', pass_reset_code='resetcode' WHERE email=%s""" - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add(sql, [email]) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query(sql, [email]) self.assertFalse( qdb.user.User.verify_code(email, 'wrongcode', 'create')) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index b86c9557e..8d5b12e43 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -458,19 +458,15 @@ def test_get_mountpoint(self): # inserting new ones so we can test that it retrieves these and # doesn't alter other ones - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE qiita.data_directory SET active=false WHERE " - "data_directory_id=1") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE qiita.data_directory SET active=false WHERE " + "data_directory_id=1") count = qdb.util.get_count('qiita.data_directory') sql = """INSERT INTO qiita.data_directory (data_type, mountpoint, subdirectory, active) VALUES ('analysis', 'analysis_tmp', true, true), ('raw_data', 'raw_data_tmp', true, false)""" - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add(sql) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query(sql) # this should have been updated exp = [(count + 1, join(qdb.util.get_db_files_base_dir(), @@ -518,19 +514,15 @@ def test_get_mountpoint_path_by_id(self): # inserting new ones so we can test that it retrieves these and # doesn't alter other ones - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add( - "UPDATE qiita.data_directory SET active=false WHERE " - "data_directory_id=1") - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query( + "UPDATE qiita.data_directory SET active=false WHERE " + "data_directory_id=1") count = qdb.util.get_count('qiita.data_directory') sql = """INSERT INTO qiita.data_directory (data_type, mountpoint, subdirectory, active) VALUES ('analysis', 'analysis_tmp', true, true), ('raw_data', 'raw_data_tmp', true, false)""" - with qdb.sql_connection.TRN: - qdb.sql_connection.TRN.add(sql) - qdb.sql_connection.TRN.execute() + qdb.sql_connection.encapsulated_query(sql) # this should have been updated exp = join(qdb.util.get_db_files_base_dir(), 'analysis_tmp') @@ -655,12 +647,10 @@ def test_filepath_id_to_rel_path(self): self.files_to_remove.append(fp) test = qdb.util.insert_filepaths( [(fp, "raw_forward_seqs")], 2, "FASTQ")[0] - with qdb.sql_connection.TRN: - sql = """INSERT INTO qiita.artifact_filepath - (artifact_id, filepath_id) - VALUES (%s, %s)""" - qdb.sql_connection.TRN.add(sql, [2, test]) - qdb.sql_connection.TRN.execute() + sql = """INSERT INTO qiita.artifact_filepath + (artifact_id, filepath_id) + VALUES (%s, %s)""" + qdb.sql_connection.encapsulated_query(sql, [2, test]) obs = qdb.util.filepath_id_to_rel_path(test) exp = 'FASTQ/2/%s' % basename(fp) @@ -674,12 +664,10 @@ def test_filepath_ids_to_rel_paths(self): self.files_to_remove.append(fp) test = qdb.util.insert_filepaths( [(fp, "raw_forward_seqs")], 2, "FASTQ")[0] - with qdb.sql_connection.TRN: - sql = """INSERT INTO qiita.artifact_filepath - (artifact_id, filepath_id) - VALUES (%s, %s)""" - qdb.sql_connection.TRN.add(sql, [2, test]) - qdb.sql_connection.TRN.execute() + sql = """INSERT INTO qiita.artifact_filepath + (artifact_id, filepath_id) + VALUES (%s, %s)""" + qdb.sql_connection.encapsulated_query(sql, [2, test]) obs = qdb.util.filepath_ids_to_rel_paths([1, 3, test]) exp = {1: 'raw_data/1_s_G1_L001_sequences.fastq.gz', diff --git a/qiita_db/user.py b/qiita_db/user.py index 557941240..beba6b1c7 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -551,13 +551,11 @@ def change_password(self, oldpass, newpass): def generate_reset_code(self): """Generates a password reset code for user""" - with qdb.sql_connection.TRN: - reset_code = qdb.util.create_rand_string(20, punct=False) - sql = """UPDATE qiita.{0} - SET pass_reset_code = %s, pass_reset_timestamp = NOW() - WHERE email = %s""".format(self._table) - qdb.sql_connection.TRN.add(sql, [reset_code, self._id]) - qdb.sql_connection.TRN.execute() + reset_code = qdb.util.create_rand_string(20, punct=False) + sql = """UPDATE qiita.{0} + SET pass_reset_code = %s, pass_reset_timestamp = NOW() + WHERE email = %s""".format(self._table) + qdb.sql_connection.encapsulated_query(sql, [reset_code, self._id]) def change_forgot_password(self, code, newpass): """Changes the password if the code is valid @@ -581,16 +579,14 @@ def change_forgot_password(self, code, newpass): return False def _change_pass(self, newpass): - with qdb.sql_connection.TRN: - if not validate_password(newpass): - raise IncorrectPasswordError("Bad password given!") - - sql = """UPDATE qiita.{0} - SET password=%s, pass_reset_code = NULL - WHERE email = %s""".format(self._table) - qdb.sql_connection.TRN.add( - sql, [qdb.util.hash_password(newpass), self._id]) - qdb.sql_connection.TRN.execute() + if not validate_password(newpass): + raise IncorrectPasswordError("Bad password given!") + + sql = """UPDATE qiita.{0} + SET password=%s, pass_reset_code = NULL + WHERE email = %s""".format(self._table) + qdb.sql_connection.encapsulated_query( + sql, [qdb.util.hash_password(newpass), self._id]) def messages(self, count=None): """Return messages in user's queue From 1de5044fad2a9fefa9e18776d0920c65b61099ab Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 07:48:25 -0600 Subject: [PATCH 5/9] fix errors --- qiita_db/processing_job.py | 12 +++++++----- qiita_db/study.py | 15 ++++++++------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 448d8f4c0..321cd52cb 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -1303,11 +1303,13 @@ def _set_validator_jobs(self, validator_jobs): validator_jobs : list of ProcessingJob The validator_jobs for the current job """ - sql = """INSERT INTO qiita.processing_job_validator - (processing_job_id, validator_id) - VALUES (%s, %s)""" - sql_args = [[self.id, j.id] for j in validator_jobs] - qdb.sql_connection.encapsulated_query(sql, sql_args, many=True) + with qdb.sql_connection.TRN: + sql = """INSERT INTO qiita.processing_job_validator + (processing_job_id, validator_id) + VALUES (%s, %s)""" + sql_args = [[self.id, j.id] for j in validator_jobs] + qdb.sql_connection.TRN.add(sql, sql_args, many=True) + qdb.sql_connection.TRN.execute() def complete(self, success, artifacts_data=None, error=None): """Completes the job, either with a success or error status diff --git a/qiita_db/study.py b/qiita_db/study.py index 5e95f95fa..34e8e71de 100644 --- a/qiita_db/study.py +++ b/qiita_db/study.py @@ -471,13 +471,14 @@ def insert_tags(cls, user, tags): tags : list of str The list of tags to add """ - email = user.email - sql = """INSERT INTO qiita.study_tags (email, study_tag) - SELECT %s, %s WHERE NOT EXISTS ( - SELECT 1 FROM qiita.study_tags WHERE study_tag = %s)""" - sql_args = [[email, tag, tag] for tag in tags] - - qdb.sql_connection.encapsulated_query(sql, sql_args, many=True) + with qdb.sql_connection.TRN: + email = user.email + sql = """INSERT INTO qiita.study_tags (email, study_tag) + SELECT %s, %s WHERE NOT EXISTS ( + SELECT 1 FROM qiita.study_tags WHERE study_tag = %s)""" + sql_args = [[email, tag, tag] for tag in tags] + qdb.sql_connection.TRN.add(sql, sql_args, many=True) + qdb.sql_connection.TRN.execute() # --- Attributes --- @property From 82d87c179e2864e213dc9ea41e5e226b24cbfd39 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 08:16:01 -0600 Subject: [PATCH 6/9] fix ProcessingJobTest --- qiita_db/test/test_processing_job.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/qiita_db/test/test_processing_job.py b/qiita_db/test/test_processing_job.py index c140baf5c..444294ba4 100644 --- a/qiita_db/test/test_processing_job.py +++ b/qiita_db/test/test_processing_job.py @@ -488,12 +488,13 @@ def test_complete_success(self): # parameter information of the command - since the ones existing # in the database currently do not require using any input parameter # to name the output artifact - sql = """UPDATE qiita.command_parameter - SET name_order = %s - WHERE command_parameter_id = %s""" - # Hard-coded values; 19 -> barcode_type, 20 -> max_barcode_errors - qdb.sql_connection.encapsulated_query( - sql, [[1, 19], [2, 20]], many=True) + with qdb.sql_connection.TRN: + sql = """UPDATE qiita.command_parameter + SET name_order = %s + WHERE command_parameter_id = %s""" + # Hard-coded values; 19 -> barcode_type, 20 -> max_barcode_errors + qdb.sql_connection.TRN.add(sql, [[1, 19], [2, 20]], many=True) + qdb.sql_connection.TRN.execute() fd, fp = mkstemp(suffix='_table.biom') self._clean_up_files.append(fp) From 28eda5c63d8fa9b9b0db61c4fe658d89f9a3576a Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 09:25:24 -0600 Subject: [PATCH 7/9] fix PY_PATCH --- qiita_db/test/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 4f08b8ea3..e7c911693 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -483,12 +483,12 @@ def test_update_artifact_from_cmd(self): PY_PATCH = """ from qiita_db.study import Study -from qiita_db.sql_connection import encapsulated_query +from qiita_db.sql_connection import TRN study = Study(1) with TRN: sql = "INSERT INTO qiita.patchtest10 (testing) VALUES (%s)" - encapsulated_query(sql, [[study.id], [study.id*100]], many=True) + TRN(sql, [[study.id], [study.id*100]], many=True) TRN.execute() """ From 556dc9c82c51f52768ba54f15cd64f07f728ecf3 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 3 Sep 2020 09:55:30 -0600 Subject: [PATCH 8/9] add missing TRN.add --- qiita_db/test/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index e7c911693..46ed78ec2 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -488,7 +488,7 @@ def test_update_artifact_from_cmd(self): with TRN: sql = "INSERT INTO qiita.patchtest10 (testing) VALUES (%s)" - TRN(sql, [[study.id], [study.id*100]], many=True) + TRN.add(sql, [[study.id], [study.id*100]], many=True) TRN.execute() """ From 0d1ffb7ac82a4b37fabf8bdbedf8221fe24183d1 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 4 Sep 2020 07:48:44 -0600 Subject: [PATCH 9/9] encapsulated_query -> perform_as_transaction --- qiita_db/analysis.py | 10 +++++----- qiita_db/artifact.py | 8 ++++---- qiita_db/download_link.py | 4 ++-- qiita_db/logger.py | 4 ++-- qiita_db/meta_util.py | 2 +- .../base_metadata_template.py | 2 +- qiita_db/metadata_template/prep_template.py | 6 +++--- qiita_db/ontology.py | 2 +- qiita_db/portal.py | 2 +- qiita_db/processing_job.py | 4 ++-- qiita_db/software.py | 6 +++--- qiita_db/sql_connection.py | 2 +- qiita_db/study.py | 18 +++++++++--------- qiita_db/test/test_analysis.py | 4 ++-- qiita_db/test/test_archive.py | 2 +- qiita_db/test/test_commands.py | 8 ++++---- qiita_db/test/test_meta_util.py | 8 ++++---- qiita_db/test/test_ontology.py | 3 ++- qiita_db/test/test_processing_job.py | 2 +- qiita_db/test/test_software.py | 6 +++--- qiita_db/test/test_study.py | 5 +++-- qiita_db/test/test_user.py | 2 +- qiita_db/test/test_util.py | 12 ++++++------ qiita_db/user.py | 4 ++-- 24 files changed, 64 insertions(+), 62 deletions(-) diff --git a/qiita_db/analysis.py b/qiita_db/analysis.py index f31ee04a7..e0be83ae9 100644 --- a/qiita_db/analysis.py +++ b/qiita_db/analysis.py @@ -357,7 +357,7 @@ def description(self, description): """ sql = """UPDATE qiita.{0} SET description = %s WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [description, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [description, self._id]) @property def samples(self): @@ -513,7 +513,7 @@ def pmid(self, pmid): """ sql = """UPDATE qiita.{0} SET pmid = %s WHERE analysis_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [pmid, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [pmid, self._id]) @property def can_be_publicized(self): @@ -618,7 +618,7 @@ def set_error(self, error_msg): sql = """UPDATE qiita.analysis SET logging_id = %s WHERE analysis_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [le.id, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [le.id, self.id]) def has_access(self, user): """Returns whether the given user has access to the analysis @@ -692,7 +692,7 @@ def share(self, user): sql = """INSERT INTO qiita.analysis_users (analysis_id, email) VALUES (%s, %s)""" - qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) + qdb.sql_connection.perform_as_transaction(sql, [self._id, user.id]) def unshare(self, user): """Unshare the analysis with another user @@ -704,7 +704,7 @@ def unshare(self, user): """ sql = """DELETE FROM qiita.analysis_users WHERE analysis_id = %s AND email = %s""" - qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) + qdb.sql_connection.perform_as_transaction(sql, [self._id, user.id]) def _lock_samples(self): """Only dflt analyses can have samples added/removed diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index c60e4295b..7b70f35b1 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -388,7 +388,7 @@ def _associate_with_analysis(instance, analysis_id): (analysis_id, artifact_id) VALUES (%s, %s)""" sql_args = [analysis_id, instance.id] - qdb.sql_connection.encapsulated_query(sql, sql_args) + qdb.sql_connection.perform_as_transaction(sql, sql_args) with qdb.sql_connection.TRN: if parents: @@ -675,7 +675,7 @@ def name(self, value): sql = """UPDATE qiita.artifact SET name = %s WHERE artifact_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) @property def timestamp(self): @@ -748,7 +748,7 @@ def _set_visibility(self, value): sql = """UPDATE qiita.artifact SET visibility_id = %s WHERE artifact_id IN %s""" - qdb.sql_connection.encapsulated_query(sql, [vis_id, tuple(ids)]) + qdb.sql_connection.perform_as_transaction(sql, [vis_id, tuple(ids)]) @visibility.setter def visibility(self, value): @@ -991,7 +991,7 @@ def is_submitted_to_vamps(self, value): sql = """UPDATE qiita.artifact SET submitted_to_vamps = %s WHERE artifact_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) @property def filepaths(self): diff --git a/qiita_db/download_link.py b/qiita_db/download_link.py index 245aba893..2fd7c971f 100644 --- a/qiita_db/download_link.py +++ b/qiita_db/download_link.py @@ -73,7 +73,7 @@ def delete(cls, jti): The jwt token identifier """ sql = """DELETE FROM qiita.{0} WHERE jti=%s""".format(cls._table) - qdb.sql_connection.encapsulated_query(sql, [jti]) + qdb.sql_connection.perform_as_transaction(sql, [jti]) @classmethod def exists(cls, jti): @@ -97,7 +97,7 @@ def delete_expired(cls): now = datetime.now(timezone.utc) sql = """DELETE FROM qiita.{0} WHERE exp<%s""".format(cls._table) - qdb.sql_connection.encapsulated_query(sql, [now]) + qdb.sql_connection.perform_as_transaction(sql, [now]) @classmethod def get(cls, jti): diff --git a/qiita_db/logger.py b/qiita_db/logger.py index bda60b336..3934fedcd 100644 --- a/qiita_db/logger.py +++ b/qiita_db/logger.py @@ -183,7 +183,7 @@ def clear_info(self): """ sql = """UPDATE qiita.{} SET information = %s WHERE logging_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [dumps([]), self.id]) + qdb.sql_connection.perform_as_transaction(sql, [dumps([]), self.id]) def add_info(self, info): """Adds new information to the info associated with this LogEntry @@ -204,4 +204,4 @@ def add_info(self, info): sql = """UPDATE qiita.{} SET information = %s WHERE logging_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [new_info, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [new_info, self.id]) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 6e031c801..733713dc6 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -337,7 +337,7 @@ def sizeof_fmt(value, position): vals = dumps(dict([x[:-1] for x in vals])) sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp) VALUES (%s, NOW())""" - qdb.sql_connection.encapsulated_query(sql, [vals]) + qdb.sql_connection.perform_as_transaction(sql, [vals]) return missing_files diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py index f871a9c3a..ba54c201b 100644 --- a/qiita_db/metadata_template/base_metadata_template.py +++ b/qiita_db/metadata_template/base_metadata_template.py @@ -289,7 +289,7 @@ def setitem(self, column, value): SET sample_values = sample_values || %s WHERE sample_id = %s""".format(self._dynamic_table) - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( sql, [dumps({column: value}), self.id]) def __setitem__(self, column, value): diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 42534c78a..4dbc98935 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -458,7 +458,7 @@ def investigation_type(self, investigation_type): sql = """UPDATE qiita.prep_template SET investigation_type = %s WHERE {0} = %s""".format(self._id_column) - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( sql, [investigation_type, self.id]) @property @@ -495,7 +495,7 @@ def deprecated(self, deprecated): """ sql = """UPDATE qiita.prep_template SET deprecated = %s WHERE {0} = %s""".format(self._id_column) - qdb.sql_connection.encapsulated_query(sql, [deprecated, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [deprecated, self.id]) def generate_files(self, samples=None, columns=None): r"""Generates all the files that contain data from this template @@ -761,4 +761,4 @@ def name(self, value): sql = """UPDATE qiita.prep_template SET name = %s WHERE prep_template_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) diff --git a/qiita_db/ontology.py b/qiita_db/ontology.py index 0ba31d7f6..0f5010efe 100644 --- a/qiita_db/ontology.py +++ b/qiita_db/ontology.py @@ -84,7 +84,7 @@ def add_user_defined_term(self, term): sql = """INSERT INTO qiita.term (ontology_id, term, user_defined) VALUES (%s, %s, true);""" - qdb.sql_connection.encapsulated_query(sql, [self.id, term]) + qdb.sql_connection.perform_as_transaction(sql, [self.id, term]) def term_type(self, term): """Get the type of a given ontology term diff --git a/qiita_db/portal.py b/qiita_db/portal.py index 435d437d3..a5585aa69 100644 --- a/qiita_db/portal.py +++ b/qiita_db/portal.py @@ -99,7 +99,7 @@ def create(cls, portal, desc): VALUES (aid, pid); END LOOP; END $do$;""" - qdb.sql_connection.encapsulated_query(sql, [portal, desc]) + qdb.sql_connection.perform_as_transaction(sql, [portal, desc]) return cls(portal) diff --git a/qiita_db/processing_job.py b/qiita_db/processing_job.py index 321cd52cb..a63ed0839 100644 --- a/qiita_db/processing_job.py +++ b/qiita_db/processing_job.py @@ -817,7 +817,7 @@ def external_id(self, value): sql = """UPDATE qiita.processing_job SET external_job_id = %s WHERE processing_job_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) @property def release_validator_job(self): @@ -1480,7 +1480,7 @@ def step(self, value): sql = """UPDATE qiita.processing_job SET step = %s WHERE processing_job_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) @property def children(self): diff --git a/qiita_db/software.py b/qiita_db/software.py index 5231e5f9d..052ae7cbc 100644 --- a/qiita_db/software.py +++ b/qiita_db/software.py @@ -635,7 +635,7 @@ def activate(self): sql = """UPDATE qiita.software_command SET active = %s WHERE command_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [True, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [True, self.id]) @property def analysis_only(self): @@ -1277,7 +1277,7 @@ def deprecated(self, deprecate): """ sql = """UPDATE qiita.software SET deprecated = %s WHERE software_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [deprecate, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [deprecate, self._id]) @property def active(self): @@ -1298,7 +1298,7 @@ def activate(self): sql = """UPDATE qiita.software SET active = %s WHERE software_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [True, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [True, self.id]) @property def client_id(self): diff --git a/qiita_db/sql_connection.py b/qiita_db/sql_connection.py index 5e4de64d7..2f4855bf3 100644 --- a/qiita_db/sql_connection.py +++ b/qiita_db/sql_connection.py @@ -493,7 +493,7 @@ def add_post_rollback_func(self, func, *args, **kwargs): TRNADMIN = Transaction(admin=True) -def encapsulated_query(sql, parameters=None): +def perform_as_transaction(sql, parameters=None): """Opens, adds and executes sql as a single transaction Parameters diff --git a/qiita_db/study.py b/qiita_db/study.py index 34e8e71de..2289619ba 100644 --- a/qiita_db/study.py +++ b/qiita_db/study.py @@ -507,7 +507,7 @@ def title(self, title): """ sql = """UPDATE qiita.{0} SET study_title = %s WHERE study_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [title, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [title, self._id]) @property def notes(self): @@ -535,7 +535,7 @@ def notes(self, notes): """ sql = """UPDATE qiita.{0} SET notes = %s WHERE study_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [notes, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [notes, self._id]) @property def public_raw_download(self): @@ -563,7 +563,7 @@ def public_raw_download(self, public_raw_download): """ sql = """UPDATE qiita.{0} SET public_raw_download = %s WHERE study_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( sql, [public_raw_download, self._id]) @property @@ -769,7 +769,7 @@ def specimen_id_column(self, value): sql = """UPDATE qiita.study SET specimen_id_column = %s WHERE study_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [value, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self._id]) @property def investigation(self): @@ -963,7 +963,7 @@ def ebi_study_accession(self, value): sql = """UPDATE qiita.{} SET ebi_study_accession = %s WHERE study_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [value, self.id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self.id]) def _ebi_submission_jobs(self): """Helper code to avoid duplication""" @@ -1213,7 +1213,7 @@ def share(self, user): sql = """INSERT INTO qiita.study_users (study_id, email) VALUES (%s, %s)""" - qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) + qdb.sql_connection.perform_as_transaction(sql, [self._id, user.id]) def unshare(self, user): """Unshare the study with another user @@ -1225,7 +1225,7 @@ def unshare(self, user): """ sql = """DELETE FROM qiita.study_users WHERE study_id = %s AND email = %s""" - qdb.sql_connection.encapsulated_query(sql, [self._id, user.id]) + qdb.sql_connection.perform_as_transaction(sql, [self._id, user.id]) def update_tags(self, user, tags): """Sets the tags of the study @@ -1537,7 +1537,7 @@ def address(self, value): """ sql = """UPDATE qiita.{0} SET address = %s WHERE study_person_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [value, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self._id]) @property def phone(self): @@ -1565,4 +1565,4 @@ def phone(self, value): """ sql = """UPDATE qiita.{0} SET phone = %s WHERE study_person_id = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [value, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [value, self._id]) diff --git a/qiita_db/test/test_analysis.py b/qiita_db/test/test_analysis.py index cf715e142..952e0914b 100644 --- a/qiita_db/test/test_analysis.py +++ b/qiita_db/test/test_analysis.py @@ -558,7 +558,7 @@ def test_build_files_post_processing_cmd(self): sql = """UPDATE qiita.software_command SET post_processing_cmd = %s WHERE command_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [results, cmd_id]) + qdb.sql_connection.perform_as_transaction(sql, [results, cmd_id]) # create a sample analysis and run build_files on it. analysis = self._create_analyses_with_samples() @@ -582,7 +582,7 @@ def test_build_files_post_processing_cmd(self): sql = """UPDATE qiita.software_command SET post_processing_cmd = NULL WHERE command_id = %s""" - qdb.sql_connection.encapsulated_query(sql, [cmd_id]) + qdb.sql_connection.perform_as_transaction(sql, [cmd_id]) def test_build_files_merge_duplicated_sample_ids(self): user = qdb.user.User("demo@microbio.me") diff --git a/qiita_db/test/test_archive.py b/qiita_db/test/test_archive.py index 54c4b1a07..f1015897e 100644 --- a/qiita_db/test/test_archive.py +++ b/qiita_db/test/test_archive.py @@ -28,7 +28,7 @@ def test_insert_from_biom_and_retrieve_feature_values(self): # 7 - to test error due to not filepath biom aid = 7 - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "DELETE FROM qiita.artifact_filepath " "WHERE artifact_id = %d" % aid) with self.assertRaises(ValueError) as err: diff --git a/qiita_db/test/test_commands.py b/qiita_db/test/test_commands.py index 46ed78ec2..495367fd0 100644 --- a/qiita_db/test/test_commands.py +++ b/qiita_db/test/test_commands.py @@ -276,7 +276,7 @@ def _assert_current_patch(self, patch_to_check): def test_unpatched(self): """Test patching from unpatched state""" # Reset the settings table to the unpatched state - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE settings SET current_patch = 'unpatched'") self._assert_current_patch('unpatched') @@ -287,7 +287,7 @@ def test_unpatched(self): def test_skip_patch(self): """Test patching from a patched state""" - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE settings SET current_patch = '2.sql'") self._assert_current_patch('2.sql') @@ -302,7 +302,7 @@ def test_skip_patch(self): def test_nonexistent_patch(self): """Test case where current patch does not exist""" - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE settings SET current_patch = 'nope.sql'") self._assert_current_patch('nope.sql') @@ -316,7 +316,7 @@ def test_python_patch(self): f.write(PY_PATCH) # Reset the settings table to the unpatched state - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE settings SET current_patch = 'unpatched'") self._assert_current_patch('unpatched') diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index b24117c43..bae0f1251 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -34,12 +34,12 @@ def tearDown(self): def _set_artifact_private(self): id_status = qdb.util.convert_to_id('private', 'visibility') - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def _set_artifact_public(self): id_status = qdb.util.convert_to_id('public', 'visibility') - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE qiita.artifact SET visibility_id = %d" % id_status) def test_validate_filepath_access_by_user(self): @@ -104,7 +104,7 @@ def test_validate_filepath_access_by_user(self): self.assertTrue(obs) # test in case there is a prep template that failed - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "INSERT INTO qiita.prep_template (data_type_id) VALUES (2)") for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: obs = qdb.meta_util.validate_filepath_access_by_user(user, i) @@ -461,7 +461,7 @@ def test_generate_biom_and_metadata_release(self): self.assertEqual(txt_obs, txt_exp) # returning configuration - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE settings SET base_data_dir = '%s'" % obdr) # testing public/default release diff --git a/qiita_db/test/test_ontology.py b/qiita_db/test/test_ontology.py index 9286d752f..5144d0082 100644 --- a/qiita_db/test/test_ontology.py +++ b/qiita_db/test/test_ontology.py @@ -19,7 +19,8 @@ def setUp(self): def _remove_term(self, term): sql = "DELETE FROM qiita.term WHERE ontology_id = %s AND term = %s" - qdb.sql_connection.encapsulated_query(sql, [self.ontology.id, term]) + qdb.sql_connection.perform_as_transaction( + sql, [self.ontology.id, term]) def testConvertToID(self): self.assertEqual(qdb.util.convert_to_id('ENA', 'ontology'), 999999999) diff --git a/qiita_db/test/test_processing_job.py b/qiita_db/test/test_processing_job.py index 444294ba4..c4248c634 100644 --- a/qiita_db/test/test_processing_job.py +++ b/qiita_db/test/test_processing_job.py @@ -796,7 +796,7 @@ def _set_allocation(memory): SET allocation = '{0}' WHERE name = 'Split libraries FASTQ'""".format( '-q qiita -l mem=%s' % memory) - qdb.sql_connection.encapsulated_query(sql) + qdb.sql_connection.perform_as_transaction(sql) # let's start with something simple, samples*1000 # 27*1000 ~ 27000 diff --git a/qiita_db/test/test_software.py b/qiita_db/test/test_software.py index bef007308..344af7b64 100644 --- a/qiita_db/test/test_software.py +++ b/qiita_db/test/test_software.py @@ -145,7 +145,7 @@ def test_post_processing_cmd(self): sql = """UPDATE qiita.software_command SET post_processing_cmd = %s WHERE command_id = 1""" - qdb.sql_connection.encapsulated_query(sql, [results]) + qdb.sql_connection.perform_as_transaction(sql, [results]) results = qdb.software.Command(1).post_processing_cmd @@ -160,7 +160,7 @@ def test_post_processing_cmd(self): sql = """UPDATE qiita.software_command SET post_processing_cmd = NULL WHERE command_id = 1""" - qdb.sql_connection.encapsulated_query(sql) + qdb.sql_connection.perform_as_transaction(sql) def test_description(self): self.assertEqual( @@ -474,7 +474,7 @@ def test_iter(self): '-q qiita -l nodes=1:ppn=5 -l pmem=8gb -l walltime=168:00:00') # delete allocations to test errors - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "DELETE FROM qiita.processing_job_resource_allocation") with self.assertRaisesRegex(ValueError, "Could not match 'Split " diff --git a/qiita_db/test/test_study.py b/qiita_db/test/test_study.py index befca5792..6c9f3b206 100644 --- a/qiita_db/test/test_study.py +++ b/qiita_db/test/test_study.py @@ -204,7 +204,7 @@ def _change_processed_data_status(self, new_status): # Change the status of the studies by changing the status of their # artifacts id_status = qdb.util.convert_to_id(new_status, 'visibility') - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE qiita.artifact SET visibility_id = %s", (id_status,)) def test_get_info(self): @@ -331,7 +331,8 @@ def test_public_raw_download(self): def test_share(self): # Clear all sharing associations self._change_processed_data_status('sandbox') - qdb.sql_connection.encapsulated_query("delete from qiita.study_users") + qdb.sql_connection.perform_as_transaction( + "delete from qiita.study_users") self.assertEqual(self.study.shared_with, []) # Try to share with the owner, which should not work diff --git a/qiita_db/test/test_user.py b/qiita_db/test/test_user.py index c72e7df64..c9932698d 100644 --- a/qiita_db/test/test_user.py +++ b/qiita_db/test/test_user.py @@ -295,7 +295,7 @@ def test_verify_code(self): user_verify_code='verifycode', pass_reset_code='resetcode' WHERE email=%s""" - qdb.sql_connection.encapsulated_query(sql, [email]) + qdb.sql_connection.perform_as_transaction(sql, [email]) self.assertFalse( qdb.user.User.verify_code(email, 'wrongcode', 'create')) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 8d5b12e43..27a3c9b52 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -458,7 +458,7 @@ def test_get_mountpoint(self): # inserting new ones so we can test that it retrieves these and # doesn't alter other ones - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE qiita.data_directory SET active=false WHERE " "data_directory_id=1") count = qdb.util.get_count('qiita.data_directory') @@ -466,7 +466,7 @@ def test_get_mountpoint(self): subdirectory, active) VALUES ('analysis', 'analysis_tmp', true, true), ('raw_data', 'raw_data_tmp', true, false)""" - qdb.sql_connection.encapsulated_query(sql) + qdb.sql_connection.perform_as_transaction(sql) # this should have been updated exp = [(count + 1, join(qdb.util.get_db_files_base_dir(), @@ -514,7 +514,7 @@ def test_get_mountpoint_path_by_id(self): # inserting new ones so we can test that it retrieves these and # doesn't alter other ones - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( "UPDATE qiita.data_directory SET active=false WHERE " "data_directory_id=1") count = qdb.util.get_count('qiita.data_directory') @@ -522,7 +522,7 @@ def test_get_mountpoint_path_by_id(self): subdirectory, active) VALUES ('analysis', 'analysis_tmp', true, true), ('raw_data', 'raw_data_tmp', true, false)""" - qdb.sql_connection.encapsulated_query(sql) + qdb.sql_connection.perform_as_transaction(sql) # this should have been updated exp = join(qdb.util.get_db_files_base_dir(), 'analysis_tmp') @@ -650,7 +650,7 @@ def test_filepath_id_to_rel_path(self): sql = """INSERT INTO qiita.artifact_filepath (artifact_id, filepath_id) VALUES (%s, %s)""" - qdb.sql_connection.encapsulated_query(sql, [2, test]) + qdb.sql_connection.perform_as_transaction(sql, [2, test]) obs = qdb.util.filepath_id_to_rel_path(test) exp = 'FASTQ/2/%s' % basename(fp) @@ -667,7 +667,7 @@ def test_filepath_ids_to_rel_paths(self): sql = """INSERT INTO qiita.artifact_filepath (artifact_id, filepath_id) VALUES (%s, %s)""" - qdb.sql_connection.encapsulated_query(sql, [2, test]) + qdb.sql_connection.perform_as_transaction(sql, [2, test]) obs = qdb.util.filepath_ids_to_rel_paths([1, 3, test]) exp = {1: 'raw_data/1_s_G1_L001_sequences.fastq.gz', diff --git a/qiita_db/user.py b/qiita_db/user.py index beba6b1c7..60e7a5333 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -555,7 +555,7 @@ def generate_reset_code(self): sql = """UPDATE qiita.{0} SET pass_reset_code = %s, pass_reset_timestamp = NOW() WHERE email = %s""".format(self._table) - qdb.sql_connection.encapsulated_query(sql, [reset_code, self._id]) + qdb.sql_connection.perform_as_transaction(sql, [reset_code, self._id]) def change_forgot_password(self, code, newpass): """Changes the password if the code is valid @@ -585,7 +585,7 @@ def _change_pass(self, newpass): sql = """UPDATE qiita.{0} SET password=%s, pass_reset_code = NULL WHERE email = %s""".format(self._table) - qdb.sql_connection.encapsulated_query( + qdb.sql_connection.perform_as_transaction( sql, [qdb.util.hash_password(newpass), self._id]) def messages(self, count=None):