python3.6 master (#2818)

* fix #2809, fix #2806 * flake8 * fix errors * redbiom.util.ids_from exact True->False * print to debug * print to debug * print to debug * fix tests * cosmetic updates to * added deblur feature search example * update taxonomy search example * fixing documentation * addressing @charles-cowart commets * fix log in * letters -> ascii_letters * fix redbiom * 4 -> '4'
qiita-spots · Feb 24, 2019 · 67c2fc0 · 67c2fc0
1 parent 57fc48f
commit 67c2fc0
Show file tree

Hide file tree

Showing 13 changed files with 141 additions and 264 deletions.
diff --git a/qiita_db/metadata_template/base_metadata_template.py b/qiita_db/metadata_template/base_metadata_template.py
@@ -823,7 +823,7 @@ def _common_extend_steps(self, md_template):
             new_cols = set(headers).difference(self.categories())
 
             if not new_cols and not new_samples:
-                return
+                return None, None
 
             is_extendable, error_msg = self.can_be_extended(new_samples,
                                                             new_cols)
@@ -1723,7 +1723,7 @@ def _identify_column_names_with_invalid_characters(cls, column_names):
             set of words containing invalid (illegal) characters.
         """
         valid_initial_char = ascii_letters
-        valid_rest = set(ascii_letters+digits+'_')
+        valid_rest = set(ascii_letters+digits+'_:|')
         invalid = []
         for s in column_names:
             if s[0] not in valid_initial_char:

diff --git a/qiita_db/metadata_template/test/test_base_metadata_template.py b/qiita_db/metadata_template/test/test_base_metadata_template.py
@@ -84,14 +84,17 @@ def test_identify_invalid_characters(self):
             'sampleid',
             'sample_id',
             '{',
-            'this|is',
+            'bla:1',
+            'bla|2',
+            'bla1:2|3',
+            'this&is',
             '4column',
             'just_fine2'])
         self.assertCountEqual(set(results), {'tax on',
                                              'bla.',
                                              '.',
                                              '{',
-                                             'this|is',
+                                             'this&is',
                                              '4column'})
 
 

diff --git a/qiita_db/metadata_template/test/test_sample_template.py b/qiita_db/metadata_template/test/test_sample_template.py
@@ -838,7 +838,7 @@ def test_clean_validate_template_no_invalid_chars2(self):
 
     def test_clean_validate_template_no_invalid_chars3(self):
         ST = qdb.metadata_template.sample_template.SampleTemplate
-        self.metadata.rename(columns={'taxon_id': 'this|is'}, inplace=True)
+        self.metadata.rename(columns={'taxon_id': 'this&is'}, inplace=True)
         with self.assertRaises(qdb.exceptions.QiitaDBColumnError):
             ST._clean_validate_template(self.metadata, 2)
 
@@ -1781,6 +1781,10 @@ def test_extend_update(self):
         st = qdb.metadata_template.sample_template.SampleTemplate.create(
             self.metadata, self.new_study)
 
+        # test updating with same data, none of the rest of the code/tests
+        # should change
+        st.extend_and_update(self.metadata)
+
         self.metadata_dict['Sample4'] = {
             'physical_specimen_location': 'location1',
             'physical_specimen_remaining': 'true',

diff --git a/qiita_pet/handlers/base_handlers.py b/qiita_pet/handlers/base_handlers.py
@@ -16,10 +16,10 @@
 class BaseHandler(RequestHandler):
     def get_current_user(self):
         '''Overrides default method of returning user curently connected'''
-        username = self.get_secure_cookie(b"user")
+        username = self.get_secure_cookie("user")
         if username is not None:
             # strip off quotes added by get_secure_cookie
-            username = username.strip("\"' ")
+            username = username.decode('ascii').strip("\"' ")
             return User(username)
         else:
             self.clear_cookie("user")

diff --git a/qiita_pet/handlers/qiita_redbiom.py b/qiita_pet/handlers/qiita_redbiom.py
@@ -14,6 +14,7 @@
 import redbiom.util
 import redbiom.fetch
 from tornado.gen import coroutine, Task
+from tornado.web import HTTPError
 
 from qiita_core.util import execute_as_transaction
 from qiita_db.util import generate_study_list_without_artifacts
@@ -58,7 +59,7 @@ def _redbiom_feature_search(self, query, contexts):
         study_artifacts = defaultdict(lambda: defaultdict(list))
         query = [f for f in query.split(' ')]
         for ctx in contexts:
-            for idx in redbiom.util.ids_from(query, True, 'feature', ctx):
+            for idx in redbiom.util.ids_from(query, False, 'feature', ctx):
                 aid, sample_id = idx.split('_', 1)
                 sid = sample_id.split('.', 1)[0]
                 study_artifacts[sid][aid].append(sample_id)
@@ -71,7 +72,12 @@ def _redbiom_taxon_search(self, query, contexts):
             # find the features with those taxonomies and then search
             # those features in the samples
             features = redbiom.fetch.taxon_descendents(ctx, query)
-            for idx in redbiom.util.ids_from(features, True, 'feature', ctx):
+            # from empirical evidence we saw that when we return more than 600
+            # features we'll reach issue #2312 so avoiding saturating the
+            # workers and raise this error quickly
+            if len(features) > 600:
+                raise HTTPError(504)
+            for idx in redbiom.util.ids_from(features, False, 'feature', ctx):
                 aid, sample_id = idx.split('_', 1)
                 sid = sample_id.split('.', 1)[0]
                 study_artifacts[sid][aid].append(sample_id)

diff --git a/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst b/qiita_pet/support_files/doc/source/checklist-for-ebi-ena-submission.rst
@@ -1,8 +1,44 @@
-Checklist to send data to EBI-ENA
-=================================
+.. role:: red
+
+Send data to EBI-ENA
+====================
+
+Qiita allows users to deposit their study, sample, experiment and sequence data to the
+`European Nucleotide Archive (ENA) <https://www.ebi.ac.uk/ena>`__, which is the permanent data
+repository of the `European Bioinformatics Institute (EBI) <https://www.ebi.ac.uk/>`__. Submitting to
+this repository will provide you with a unique identifier for your study, which is generally a
+requirement for publications. Your study will be housed with all other Qiita submissions
+and so we require adherence to the `MiXs standard <http://gensc.org/mixs/>`__.
 
 `Here <https://knightlab.ucsd.edu/wordpress/wp-content/uploads/2016/04/QiitaTemplate_20181218.xlsx>`__ you will find a document outlining these requirements, with examples, when possible.
 
+Note that submissions are time consuming and need full collaboration from the user.
+:red:`Do not wait until the last minute to request help.` In general, the best
+time to request a submission is when you are writing your paper. Remember that the
+data can be submitted to EBI and can be kept private and simply make public when
+the paper is accepted. Note that EBI/ENA takes up to 15 days to change the status
+from private to public, so consider this when submitting data and your manuscript.
+
+.. note::
+   For convenience Qiita allows you to upload a QIIME mapping file to process your data. However,
+   the QIIME mapping file, in general, does not have all the EBI/ENA fields. Thus, you will need to
+   update your information files (sample or preparation) via the update option. To simplify this process,
+   you can download the system generated files and add/modify these fields for each file.
+
+
+EBI-ENA NULL values vocabulary
+------------------------------
+
+We support only the following values: *not applicable*, *not collected*, *not provided*, *restricted access*.
+
+For the latest definitions and explanation visit the `EBI/ENA Missing value reporting <http://www.ebi.ac.uk/ena/about/missing-values-reporting>`__.
+
+.. warning::
+   Column names in your information files cannot be named as a Postgres reserved word. For example, a column cannot be named `CONDITION`, but could instead be named `DISEASE_CONDITION`. For a full list of these reserved words, see this `link <https://www.postgresql.org/docs/9.3/static/sql-keywords-appendix.html>`__.
+
+Checklist
+---------
+
 For each preparation that needs to be uploaded to EBI-ENA we will check:
 
   1. Data processing
@@ -27,6 +63,40 @@ For each preparation that needs to be uploaded to EBI-ENA we will check:
       7. *elevation*, *latitude*, *longitude*
       8. *empo_1*, *empo_2*, *empo_3*
 
+         .. table::
+            :widths: auto
+
+            =============== ================= ======================= ================================================================================
+            empo_1          empo_2            empo_3	                Examples
+            Free-living	    Non-saline        Water (non-saline)      fresh water from lake, pond, or river (<5 psu)
+            Free-living	    Non-saline        Sediment (non-saline)	  sediment from lake, pond, or river (<5 psu)
+            Free-living	    Non-saline        Soil (non-saline)       soil from forest, grassland, tundra, desert, etc.
+            Free-living	    Non-saline        Surface (non-saline)    biofilm from wet (<5 psu) or dry surface, wood, dust, or microbial mat
+            Free-living	    Non-saline        Subsurface (non-saline) deep or subsurface environment
+            Free-living	    Non-saline        Aerosol (non-saline)    aerosolized dust or liquid
+            Free-living	    Saline            Water (saline)          salt water from ocean, sea, estuary, mangrove, or coral reef (>5 psu)
+            Free-living	    Saline            Sediment (saline)       sediment from ocean, sea, estuary, mangrove, or beach (>5 psu)
+            Free-living	    Saline            Hypersaline (saline)    water from hypersaline sample or brine (>50 psu)
+            Free-living	    Saline            Surface (saline)        biofilm from wet or underwater surface or microbial mat (>5 psu)
+            Free-living	    Saline            Aerosol (saline)        seaspray or other aerosolized saline material (>5 psu)
+            Host-associated Animal-associated Animal distal gut       feces, stool
+            Host-associated Animal-associated Animal proximal gut     digesta
+            Host-associated Animal-associated Animal secretion        gut intestine, gizzard, crop, lumen, or mucosa
+            Host-associated Animal-associated Animal surface          skin, sebum, mucus, slime
+            Host-associated Animal-associated Animal corpus           tissue of sponge, coral, gill, siphon, carcass, etc. or whole small animal
+            Host-associated Fungus-associated Fungus corpus           tissue of mushroom or other fungi
+            Host-associated Fungus-associated Fungus surface          biofilm of mushroom
+            Host-associated Plant-associated  Plant secretion         pollen or sap
+            Host-associated Plant-associated  Plant surface          	leaf or kelp surface biofilm
+            Host-associated Plant-associated  Plant rhizosphere       plant root system, may include some soil
+            Host-associated Plant-associated  Plant corpus            tissue of leaf, stem, fruit, or algae
+            Control         Negative          Sterile water blank     sterile water blank used as negative control for extraction, PCR, and sequencing
+            Control         Positive          Mock community          known mixed community used as positive control
+            Control         Positive          Single strain           known single strain control culture
+            Unknown         Contradictory     Unknown (contradictory) unknown sample type because other metadata is contradictory
+            Unknown         Missing           Unknown (missing)       unknown sample type because metadata is unavailable
+            =============== ================= ======================= ================================================================================
+
     c. Extra minimal information for host associated studies:
 
       1. *host_body_habitat*, *host_body_site*, *host_body_product*

diff --git a/qiita_pet/support_files/doc/source/europeanbioinformaticsinstitute.rst b/qiita_pet/support_files/doc/source/europeanbioinformaticsinstitute.rst
diff --git a/qiita_pet/support_files/doc/source/index.rst b/qiita_pet/support_files/doc/source/index.rst
@@ -38,7 +38,6 @@ Looking for information about submitting your files to EBI? Please see the docum
 
 .. toctree::
 
-   europeanbioinformaticsinstitute.rst
    checklist-for-ebi-ena-submission
 
 Looking for comparable studies? Please see the document here: