Fix 3436 (#3440)

* fix #3438

* fix #3436

* self->cls

* rm ()

* adding tests

* fix test

* fix flake8

CHANGELOG.md

@@ -14,7 +14,9 @@ Deployed on October 14th, 2024
 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support.
 * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates.
 * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks!
-* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439).
+* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440).
+* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225).
+
 
 
 Version 2024.07

qiita_db/artifact.py

@@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self):
             # words has more that one processing step behind it
             fine_to_send = []
             fine_to_send.extend([pt.artifact for pt in self.prep_templates])
-            fine_to_send.extend([c for a in fine_to_send for c in a.children])
+            fine_to_send.extend([c for a in fine_to_send if a is not None
+                                 for c in a.children])
             if self not in fine_to_send:
                 return False
 

qiita_db/metadata_template/prep_template.py

@@ -272,6 +272,18 @@ def delete(cls, id_):
                     "Cannot remove prep template %d because it has an artifact"
                     " associated with it" % id_)
 
+            # artifacts that are archived are not returned as part of the code
+            # above and we need to clean them before moving forward
+            sql = """SELECT artifact_id
+                     FROM qiita.preparation_artifact
+                     WHERE prep_template_id = %s"""
+            qdb.sql_connection.TRN.add(sql, args)
+            archived_artifacts = set(
+                qdb.sql_connection.TRN.execute_fetchflatten())
+            if archived_artifacts:
+                for aid in archived_artifacts:
+                    qdb.artifact.Artifact.delete(aid)
+
             # Delete the prep template filepaths
             sql = """DELETE FROM qiita.prep_template_filepath
                      WHERE prep_template_id = %s"""

qiita_db/test/test_artifact.py

@@ -10,7 +10,7 @@
 from tempfile import mkstemp, mkdtemp
 from datetime import datetime
 from os import close, remove
-from os.path import exists, join, basename
+from os.path import exists, join, basename, dirname, abspath
 from shutil import copyfile
 from functools import partial
 from json import dumps
@@ -23,6 +23,7 @@
 from qiita_core.util import qiita_test_checker
 from qiita_core.testing import wait_for_processing_job
 import qiita_db as qdb
+from qiita_ware.private_plugin import _delete_analysis_artifacts
 
 
 class ArtifactTestsReadOnly(TestCase):
@@ -1518,15 +1519,57 @@ def test_archive(self):
                                     'be archived'):
             A.archive(8)
 
-        for aid in range(4, 7):
+        for aid in range(5, 7):
             ms = A(aid).merging_scheme
             A.archive(aid)
             self.assertEqual(ms, A(aid).merging_scheme)
             exp_nodes.remove(A(aid))
             self.assertCountEqual(A(1).descendants.nodes(), exp_nodes)
 
         obs_artifacts = len(qdb.util.get_artifacts_information([4, 5, 6, 8]))
-        self.assertEqual(1, obs_artifacts)
+        self.assertEqual(2, obs_artifacts)
+
+        # in the tests above we generated and validated archived artifacts
+        # so this allows us to add tests to delete a prep-info with archived
+        # artifacts. The first bottleneck to do this is that this tests will
+        # actually remove files, which we will need for other tests so lets
+        # make a copy and then restore them
+        mfolder = dirname(dirname(abspath(__file__)))
+        mpath = join(mfolder, 'support_files', 'test_data')
+        mp = partial(join, mpath)
+        fps = [
+            mp('processed_data/1_study_1001_closed_reference_otu_table.biom'),
+            mp('processed_data/'
+               '1_study_1001_closed_reference_otu_table_Silva.biom'),
+            mp('raw_data/1_s_G1_L001_sequences.fastq.gz'),
+            mp('raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz')]
+        for fp in fps:
+            copyfile(fp, f'{fp}.bk')
+
+        PT = qdb.metadata_template.prep_template.PrepTemplate
+        QEE = qdb.exceptions.QiitaDBExecutionError
+        pt = A(1).prep_templates[0]
+        # it should fail as this prep is public and have been submitted to ENA
+        with self.assertRaisesRegex(QEE, 'Cannot remove prep template 1'):
+            PT.delete(pt.id)
+        # now, remove those restrictions + analysis + linked artifacts
+        sql = "DELETE FROM qiita.artifact_processing_job"
+        qdb.sql_connection.perform_as_transaction(sql)
+        sql = "DELETE FROM qiita.ebi_run_accession"
+        qdb.sql_connection.perform_as_transaction(sql)
+        sql = "UPDATE qiita.artifact SET visibility_id = 1"
+        qdb.sql_connection.perform_as_transaction(sql)
+        _delete_analysis_artifacts(qdb.analysis.Analysis(1))
+        _delete_analysis_artifacts(qdb.analysis.Analysis(2))
+        _delete_analysis_artifacts(qdb.analysis.Analysis(3))
+        for aid in [3, 2, 1]:
+            A.delete(aid)
+
+        PT.delete(pt.id)
+
+        # bringing back the filepaths
+        for fp in fps:
+            copyfile(f'{fp}.bk', fp)
 
 
 if __name__ == '__main__':