From a21dc8b0f7a9f37fdfd7b1e25a1a4d3b3dc03f45 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Tue, 24 Oct 2023 21:09:33 -0400 Subject: [PATCH 01/13] Update test_simpleaf.yml try to enable mamba --- .github/workflows/test_simpleaf.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_simpleaf.yml b/.github/workflows/test_simpleaf.yml index eda5248..edd41da 100644 --- a/.github/workflows/test_simpleaf.yml +++ b/.github/workflows/test_simpleaf.yml @@ -32,16 +32,17 @@ jobs: - name: Build run: cargo build --verbose --release - + - name: Install conda env uses: conda-incubator/setup-miniconda@v2.2.0 with: python-version: "3.10" - # mamba-version: "*" - channels: default,conda-forge,bioconda + mamba-version: "*" + channels: conda-forge,default,bioconda channel-priority: true activate-environment: anaconda-client-env environment-file: simpleaf_conda_env.yml + - name: Test simpleaf shell: bash -l {0} run: | From 90ca81b154c0c1cb6a9c67a951eb54b86d0e81c2 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Tue, 24 Oct 2023 21:53:00 -0400 Subject: [PATCH 02/13] Update conf.py bump version in docs. --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index b492075..23cdcfa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -22,7 +22,7 @@ author = 'Dongze He, Rob Patro' # The full version, including alpha/beta/rc tags -release = '0.10.0' +release = '0.15.0' master_doc = 'index' From b4f02b2af78e753428c5d1129db60668c7bd52a3 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 17:53:57 -0400 Subject: [PATCH 03/13] doc : update simpleaf workflow util function doc --- docs/source/workflow-utility-library.rst | 538 ++++++++++++++++++----- 1 file changed, 426 insertions(+), 112 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index a2ac870..3d69ae8 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -1,12 +1,12 @@ Simpleaf workflow utility library =================================== -To ease the development of *simpleaf*workflow templates, the *simpleaf* team provides not only some `built-in variables `_, but also a workflow utility library, which will be automatically passed to the `internal Jsonnet engine ` of *simpleaf* when parsing a workflow template as the ``__utils`` external variable. One can receive this variable in their templates by adding ``utils=std.extVar("__utils")``, and use the functions in the utility library by calling ``utils.function_name(args)``, where *function_name* should be replaced by an actual function name listed below. To be consistent with the `Jsonnet official documentation `_, here we will list the function signatures with a brief description. One can find the function definitions on `this page `_. +To ease the development of *simpleaf* workflow templates, the *simpleaf* team provides not only some `built-in variables `_, but also a workflow utility library, which will be automatically passed to the `internal Jsonnet engine `_ of *simpleaf* when parsing a workflow template as the ``__utils`` external variable. One can receive this variable in their templates by adding ``utils=std.extVar("__utils")``, and use the functions in the utility library by calling ``utils.function_name(args)``, where *function_name* should be replaced by an actual function name listed below. To be consistent with the `Jsonnet official documentation `_, here we will list the function signatures with a brief description. One can find the function definitions on `this page `_. Import the utility library '''''''''''''''''''''''''' -As the built-in variables are provided by *simpleaf* to its intenal Jsonnet engine, they will be unavailable if we want to parse the template directly using Jsonnet or Jrsonnet. Therefore, when debugging templates which utilize the utility library with an external Jsonnet engine, we must manually provide the ``__utils`` external variable to Jsonnet, and either copy and paste the library file to the same directory as the template file, which is the default library searching path when calling jsonnet or provide the directory containing the library as an additional library searching path. Althought in the following code chunk we show the code for both ways, we just need to select one in practice. Here we assume that *simpleaf* has been configured correctly, i.e., the ``ALEVIN_FRY_HOME`` env variable has been set and a local copy of the protocol-esturary exists. If not, one can directly obtain the library file from its GitHub repository. +As the built-in variables are provided by *simpleaf* to its internal Jsonnet engine, they will be unavailable if we want to parse the template directly using Jsonnet or Jrsonnet. Therefore, when debugging templates that utilize the utility library with an external Jsonnet engine, we must manually provide the ``__utils`` external variable to Jsonnet and either copy and paste the library file to the same directory as the template file, which is the default library searching path when calling jsonnet, or provide the directory containing the library as an additional library searching path. Although in the following code chunk, we show the code for both ways, we only need to select one in practice. Here we assume that *simpleaf* has been configured correctly, i.e., the ``ALEVIN_FRY_HOME`` environment variable has been set and a local copy of the protocol-estuary exists. If not, one can directly obtain the library file from its GitHub repository. .. code-block:: shell @@ -16,7 +16,7 @@ As the built-in variables are provided by *simpleaf* to its intenal Jsonnet engi # Otherwise, we either copy the library file to the same dir as the template copy $ALEVIN_FRY_HOME/protocol-estuary/protocol-estuary-main/utils/simpleaf_workflow_utils.libsonnet . - # or provide the directory as an addtional library searching path via --jpath + # or provide the directory as an additional library searching path via --jpath jsonnet a_template_using_utils_lib.jsonnet --ext-code '__utils=import "simpleaf_workflow_utils.libsonnet"' --jpath "$ALEVIN_FRY_HOME/protocol-estuary/protocol-estuary-main/utils" where ``--ext-code`` is the flag for passing an external variable, and ``--jpath`` specifies the library searching path. @@ -29,173 +29,487 @@ To do this, we recommend adding the following code at the beginning of your work local utils=std.extVar("__utils"); -Of course, instead of ``utils``, you can receive this library as a different name such as ``local the_best_utility_library_ever=std.extVar("__utils");`` or something similar. If you do this, you should use the name you assigned to the library instead of ``utils.`` when calling the functions. - -Terminology -'''''''''''''''''''''''''' -- `Simpleaf command record `_: a sub-object in an object that has required identify fields, *Program Name* and *Step*, and the *Program Name* represents one of the simpleaf commands. -- `Recommended main sections `_: the recommended sub-fields of a workflow object. - - *meta_info* - - *minimal_config* - - *advanced_config* - - *external_commands* -- `Identity fields `_: the fields used for identifying a command record. -- A valid field or a valid meta-variable is a field or a meta-variable that exists and is not *null*. -- A simpleaf flag field is a field in a simpleaf command record that represents one of the `Simpleaf Program Arguments`_. - -Frequently Used Functions -''''''''''''''''''''''''''''''''''''''''''' - -utils.combine_main_sections(o) +ref_type """""""""""""""""""""""""""""""""""""""""""""" -**Input**: o: an object +**Input**: `o`: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, `{type: "spliceu", spliceu: {gtf: null, fasta: null}}`. -**Output**: An object with the same information and layout as the original object but with the *Recommended Configuration* and *Optional Configuration* sections combined. +**Output**: An object with the ``simpleaf index`` arguments that are related to the specified reference type in the input object. -This function combines two `recommended main sections `_, *Recommended Configuration* and *Optional Configuration*, keep all other main sections as it is, and ignore all other sections in the root layer. Those two sections being combined are designed to have an identical layout. If your template contains these two main sections, we recommend applying this function before any other processing steps. However, as this function will ignore all fields that are not simpleaf flag fields in all `command records `_, one should save important non-simpleaf flag fields to other variables before applying this function. +This function has four modes (reference types), triggered by the ``type`` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The four modes are: +- *spliceu* (*spliced+unspliced* reference): The required fields are: + - ``gtf``: A string representing the path to a gene annotation GTF file. + - ``fasta``: A string representing the path to a reference genome FASTA file. +- *splici* (*spliced+intronic* reference): The required fields are: + - ``gtf``: A string representing the path to a gene annotation GTF file. + - ``fasta``: A string representing the path to a reference genome FASTA file. + - ``rlen``: An *optional* field representing the read length in the dataset. If not provided, the default value, 91, will be used. +- *direct_ref*: The required fields are: + - ``ref_seq``: A string representing the path to a *transcriptome* FASTA file. + - ``t2g_map``: A string representing the path to a transcript-to-gene mapping file. +- *existing_index*: The required fields are: + - ``index``: A string representing the path to an existing index directory. + - ``t2g_map``: A string representing the path to a transcript-to-gene mapping file. -When merging the two main sections, the function will -1. bring all arguments in the nested layers of any simpleaf command record to the same layer as the identity fields of that record live. -2. merge the two sections and moving the subfields of the merged section out to the root layer (same layer as these two main sections), and remove these two sections because they are empty after moving. +**Wrapper functions**: We also provide separate functions for each of the four modes, ``splici``, ``spliceu``, ``direct_ref``, and ``existing_index``, which are thin wrappers of ``ref_type``. These four functions take an object containing their required fields introduced above. -utils.add_meta_args(o) -"""""""""""""""""""""""""""""""""""""""""""""" -**Input**: an object +I've corrected the formatting and typos in the text. +**Example Usage** -**Output**: The same object but with additional ``--threads``, ``--output``, and ``--use-piscem`` fields added to its simpleaf command records if applicable. +.. code-block:: jsonnet + + # import the utility library + local utils=std.extVar("__utils"); -This function finds the meta-variables, if any, defined in the *meta_info* main section and the build-in variables passed by *simpleaf* and assigns additional arguments to all qualified simpleaf command records if applicable. One example can be found in our `tutorial `_. Currently, this function can process three meta-variables: + local splici_args = { + gtf : "genes.gtf", + fasta : "genome.fa", + rlen : 91, + }; -- *threads*: if the *threads* meta-variable is valid (it exists and is not *null*), all simpleaf command records will be assigned a ``--threads`` field with this value if, for this command, ``--threads`` is a valid argument but it does not exist in the command record. -- *use-piscem*: if the *use-piscem* meta-variable is valid, all simpleaf command records will be assigned a ``--use-piscem`` field with this value if, for this command, ``--use-piscem`` is a valid flag but missing. -- For *output*, it will first decide the actual output directory: if the *output* meta-variable is valid, this value will be used. Otherwise, the `__output` `built-in variable `_ will be used. All simpleaf command records will be assigned a ``--output`` field with the actual output directory if, for this command, ``--use-piscem`` is a valid flag but is missing. + local ref_type = utils.ref_type({ + type : "splici", + splici : splici_args, + }) -utils.add_index_dir_for_simpleaf_index_quant_combo(o) -""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + local splici = utils.splici(splici_args); -**Input**: o: an object +In the above example, the objects `ref_type` and `splici` are identical and look like the following: -**Output**: The same object but with an additional ``--index`` field for each qualified *simpleaf quant* command record. +.. code-block:: jsonnet -This function automatically adds the ``--index`` flag field to qualified *simpleaf quant* command records in a workflow object. A qualified *simpleaf quant* command record must have the name *simpleaf_quant* and a corresponding *simpleaf index* command record in the same layer with the name *simpleaf_index*. + { + # hidden, system fields + type :: "splici", # hidden field + arguments :: {gtf : "genes.gtf", fasta : "genome.fa", rlen : 91}, # hidden field + + # fields shown in the manifest + "--ref-type" : "splici", + "--fasta" : "genome.fa", + "--gtf" : "genes.gtf", + "--rlen" : 91, + } -This function does the following steps: -1. It traverses the given workflow object to find all fields with a *simpleaf_index* and a *simpleaf_quant* sub-field. -2. For each field with the desired sub-fields found in step 1, it checks if its *simpeaf_index* has a ``--output`` valid field and if its *simpleaf_quant* misses the ``--index`` and ``--map-dir`` field. -3. for each *simpeaf_index* and *simpeaf_quant* field pair satisfied the criteria in step 2, it adds a ``--index`` field to that *simpleaf_quant*, by appending a */index* to the value of the ``--output`` field in the corresponding *simpleaf_index*. +simpleaf_index +"""""""""""""""""""""""""""""""""""""""""""""" -For example, if we run the following Jsonnet program, +**Input**: +- *step*: An integer indicating the step number (execution order) of this simpleaf command record in the workflow. +- *ref_type*: A ``ref_type`` object returned by calling `utils.ref_type` or any object with the same format. +- *arguments*: An object in which each field represents a ``simpleaf index`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. +- *output*: A string that will be passed to `simpleaf index --output`. -.. code-block:: console +**Output**: A well-defined ``simpleaf index`` command record. - local o = { - "simpleaf_index": { - "--output": "/path/to/output" - }, - "simpleaf_quant": {}, - "anohter simpleaf_quant": {}, - }; - utils.add_index_dir_for_simpleaf_index_quant_combo(o) +**Example Usage** -we will get the following JSON configuration: +.. code-block:: jsonnet -.. code-block:: console + # import the utility library + local utils=std.extVar("__utils"); - local o = { - "simpleaf_index": { - "--output": "/simpleaf/index/output" - }, - "simpleaf_quant": { - "--index": "/simpleaf/index/output/index" - } - "anohter simpleaf_quant": {}, + local splici_args = { + gtf : "genes.gtf", + fasta : "genome.fa", + rlen : 91, }; - utils.add_index_dir_for_simpleaf_index_quant_combo(o) + + local splici = utils.splici(splici_args); + local arguments = { + active : true, + "--use-piscem" : true, + }; + + local simpleaf_index = utils.simpleaf_index( + 1, # step number + splici, # ref_type, + arguments, + "./simpleaf_index" # output directory + )}; + + +The `simpleaf_index` object in the above code chunk will be + +.. code-block:: JSON + { + # hidden, system fields + ref_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `ref_type` + arguments :: {active : true, "--use-piscem" : true}, # hidden field + output :: "./simpleaf_index", # hidden field + index :: "./simpleaf_index/index", # hidden field + t2g_map :: "./simpleaf_index/index/t2g_3col.tsv", # hidden field + + # fields shown in in the manifest + program_name : "simpleaf index", + step : 1, + active : true, + "--output": "./workflow_output/simpleaf_index", + "--gtf" : "genes.gtf", + "--fasta" : "genome.fa", + "--rlen" : 91, + "--use-piscem" : true, + } + +map_type +"""""""""""""""""""""""""""""""""""""""""""""" -utils.get(o, f, use_default = false, default = null) -"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" +**Input**: +- `o`: an object with + - a `type` field, and + - an object field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}`. +- `simpleaf_index`: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. -**Input**: o: an object, f: the target field name, use_default: boolean, default: any valid type +**Output**: An object with the `simpleaf quant` arguments that are related to the specified map type in the input object. -**Output**: Return the target field *f* in the given object if the object has a sub-field called *f*. Otherwise, - - if *use_default* is *true*, return the value of the *default* argument (defualtly *null*). - - if *use_default* is false, raise an error. +This function has two modes (map types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The two modes are: +- `map_reads`: Map reads against the provided index or an index built from a previous step. The required fields are + - `reads1`: A string representing the path to a gene annotation GTF file, + - `reads2`: A string representing the path to a reference genome FASTA file. +- `existing_mappings`: Skip mapping and use the existing mapping results. The required fields are + - `map_dir`: A string representing the path to the mapping result directory, + - `t2g_map`: A string representing the path to a transcript-to-gene mapping file. -This function tries to (non-recursively) get a sub-field in the provided object and return it. If the field doesn't exist, then it either returns a default value or raises an error. +**Wrapper functions**: We also provide separate functions for each of the two modes, `map_reads` and `existing_mappings`, which are thin wrappers of `map_type`. These two functions take an object containing their required fields introduced above. -Simpleaf Program Arguments -'''''''''''''''''''''''''' -This section lists the arguments of *simpleaf* command arguments for programs that are supported in *simpleaf workflow*. Usually, these fields are used for obtaining and validating the fields included in a command record. Details about a command record can be found in `protocol estuary `_. +**Example Usage** -utils.SimpleafPrograms["simpleaf index"] -""""""""""""""""""""""""""""""""""""""""""""" -This field contains all command line flags of the *simpleaf index* command. Furthermore, it also includes the identity fields, *Program Name*, *Step*, and *Active*. +.. code-block:: jsonnet + # import the utility library + local utils=std.extVar("__utils"); -utils.SimpleafPrograms["simpleaf quant"] -""""""""""""""""""""""""""""""""""""""""" -This field contains all command line flags of the *simpleaf quant* command. Furthermore, it also includes the identity fields, *Program Name*, *Step*, and *Active*. + local simpleaf_index = ... # The return of object of simpleaf_index function in its example usage -Helper Functions -'''''''''''''''''''''''''''''''''''''''''''' + local map_reads_args = { + reads1 : "reads1.fastq", + reads2 : "reads2.fastq", + }; -utils.flat_arg_groups(o, path = "") -"""""""""""""""""""""""""""""""""""""""""""""" + local map_type = utils.map_type({ + type : "map_reads", + map_reads : map_reads_args, + }) -**Input**: o: an object + local map_reads = utils.map_reads(map_reads_args); -**Output**: An object with the same information and layout as the original object, but all simpleaf command arguments located at a nested layer of the corresponding simpleaf command record are brought to the same layer as the identity fields of the simpleaf command record. +In the above example, the objects `map_type` and `map_reads` are identical and look like the following: -The *combine_main_sections* function calls this function internally. When merging the two main sections, the function will bring all arguments in the nested layers of any simpleaf command record to the same layer as the identity fields of that record live. See our example on `setting the path for showing trajectory `_. +.. code-block:: jsonnet -utils.recursive_get(o, target_name, path = "") + { + # hidden, system fields + type :: "map_reads", # hidden field + arguments :: {reads1 : "reads1.fastq", reads2 : "reads2.fastq"}, # hidden field + + # fields shown in the manifest + "--index" : "./workflow_output/simpleaf_index/index", + "--t2g-map": "./workflow_output/simpleaf_index/index/t2g_3col.tsv", + "--reads1" : "reads1.fastq", + "--reads2" : "reads2.fastq", + } + + +cell_filt_type """""""""""""""""""""""""""""""""""""""""""""" -**Input**: o: an object, target_name: name of the field to look for, path: trajectory path to the object if the object lives in a nested layer -**Output**: The value of the target field if it is in the object, else *null*. +**Input**: +- `o`: an object with a `type` field, and an argument field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "explicit_pl", "explicit_pl": "whitelist.txt"}` -This function recursively traverses the object to find the field with the target name. If it finds it, it will return the value of the field. If not, it will return a *null*. See our example on `setting the path for showing trajectory `_. +**Output**: An object with the `simpleaf quant` arguments that are related to the specified cell filtering type in the input object. -utils.get_output(o) -"""""""""""""""""""""""""""""""""""""""""""""" +This function has five modes (cell filtering types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. For more details, please refer to the online documentation of [simpleaf quant](https://simpleaf.readthedocs.io/en/latest/quant-command.html) and [alevin-fry](https://alevin-fry.readthedocs.io/en/latest/). The five modes are: +- `unfiltered_pl`: No cell filtering but correcting cell barcodes by an external or default (only works for 10X Chromium V2 and V3). The corresponding argument value field can be `true` (using the default whitelist if in `10xv2` and `10xv3` chemistry), or a string representing the path to an unfiltered permit list file. +- `knee`: Knee point-based filtering. The corresponding argument value field must be `true` if selected. +- `forced`: Use a forced number of cells. The corresponding argument field must be an integer representing the number of cells that can pass the filtering. +- `expect`: Use the expected number of cells. The corresponding argument field must be an integer representing the expected number of cells. +- `explicit_pl`: Use a filtered, explicit permit list. The corresponding argument field must be a string representing the path to a cell barcode permit list file. + +**Wrapper functions**: We also provide separate functions for each mode, `unfiltered_pl`, `knee`, `forced`, `expect`, and `explicit_pl`, which are thin wrappers of `cell_filt_type`. These functions take an object containing their required fields introduced above. + +**Example Usage** -**Input**: o: an object +.. code-block:: jsonnet + + # import the utility library + local utils=std.extVar("__utils"); + + local cell_filt_args = { + unfiltered_pl : true, + }; + + local cell_filt_type = utils.cell_filt_type({ + type : "unfiltered_pl", + unfiltered_pl : unfiltered_pl_args, + }) + + local unfiltered_pl = utils.unfiltered_pl(unfiltered_pl_args); -**Output**: a string representing the actual output directory. +In the above example, the objects `cell_filt_type` and `unfiltered_pl` are identical and look like the following: -This function checks two places to decide the output directory and return it as a string. -1. the *__output* built-in variable, which represents the path provided via the ``--output`` argument of ``simpleaf workflow run``. -2. the *output* meta-variable in the *meta_info* main section. +.. code-block:: jsonnet -If the meta-variable is valid, it will be the return value of this function. Otherwise, the built-in variable will be the return value. Notice that if a template uses this function to parse the template out of *simpleaf*, for example, using *jsonnet* or *jrsonnet*, one must manually provide the *__output* variable by doing something like ``jsonnet template.jsonnet --ext-code "__output='/path/to/a/directory'"``. + { + # hidden, system fields + type :: "unfiltered_pl", # hidden field + arguments :: true, # hidden field + + # fields shown in the manifest + "--unfiltered-pl" : true, + } -utils.check_invalid_args(o, path = "") +simpleaf_quant """""""""""""""""""""""""""""""""""""""""""""" -**Input**: o: an object, path: trajectory path to the object if the object lives in a nested layer +**Input**: +- step : An integer indicating the step number (execution order) of this simpleaf command record in the workflow. +- map_type : A ``map_type`` object returned by calling `utils.map_type` or any object with the same format. +- cell_filt_type : A ``cell_filt_type`` object returned by calling `utils.cell_filt_type` or any object with the same format. +- arguments : an object in which each field represents a ``simpleaf quant`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. +- output : A string that will be passed to `simpleaf quant --output`. + +**Output**: A well-defined ``simpleaf quant`` command record. + +**Example Usage** + +.. code-block:: jsonnet + + # import the utility library + local utils=std.extVar("__utils"); -**Output**: If all simpleaf arguments are valid, the original object will be returned. Otherwise, an error will be raised. + local map_type = utils.map_type({...}); # Please refer to the example usage of function `map_reads` for full details. -This function traverses the given object to find simpleaf command records. If the records contain invalid fields that are neither a simpleaf flag field nor an identity field, an error will be raised. If no simpleaf command record contains invalid fields, the original object will be returned. However, we do not recommend validating simpleaf commands in any template because when parsing the resulting workflow manifest, simpleaf itself will validate all simpleaf commands and return clear error messages if encountering invalid command records. + local cell_filt_type = utils.cell_filt_type({...}); # Please refer to the example usage of function `cell_filt_type` for full details. -utils.get_recommended_args(o) + local arguments = { + active : true, + "--chemistry" : "10xv3", + "--resolution" : "cr-like" + }; + + local simpleaf_quant = utils.simpleaf_quant( + 2, # step number + map_type, + cell_filt_type, + arguments, + "./simpleaf_quant" # output directory + )}; + + +The `simpleaf_quant` object in the above code chunk will be + +.. code-block:: JSON + { + # hidden, system fields + map_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `map_reads` + cell_filt_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `cell_filt_type` + arguments :: {active : true, "--chemistry" : "10xv3", "--resolution" : "cr-like"}, # hidden field + output :: "./simpleaf_quant", # hidden field + + # fields shown in in the manifest + program_name : "simpleaf index", + step : 1, + active : true, + "--chemistry": "10xv3", + "--index": "./workflow_output/simpleaf_index/index", + "--min-reads": 10, + "--output": "./workflow_output/simpleaf_quant", + "--reads1": "reads1.fastq", + "--reads2": "reads2.fastq", + "--resolution": "cr-like", + "--t2g-map": "./workflow_output/simpleaf_index/index/t2g_3col.tsv", + "--unfiltered-pl": true, + } + + +feature_barcode_ref """""""""""""""""""""""""""""""""""""""""""""" +I've fixed the typos in your input text: + +**Input**: +- `step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define three command records with incremental step numbers according to the provided step number. +- `csv`: A string representing the path to the "feature_barcode.csv" file of the dataset. +- `name_col`: An integer representing the column index of the feature name column in the feature barcode CSV file. +- `barcode_col`: An integer representing the column index of the feature barcode sequence column in the feature barcode CSV file. +- `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. + +**Output**: An object containing three external command records, including "mkdir," "create_t2g," and "create_fasta," and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, + +This function defines three external command records: +1. "mkdir": This command calls the "mkdir" shell program to create the output directory recursively if it doesn't exist. +2. "create_t2g": This command calls `awk` to create a transcript-to-gene mapping TSV file according to the input "csv" file, in which the transcript ID and gene ID of each feature barcode are identical. The expected output file of this command will be named ".feature_barcode_ref_t2g.tsv" and located in the provided output directory. +3. "create_fasta": This command calls `awk` to create a FASTA file according to the input "csv" file, in which each feature barcode is a FASTA record. The expected output file of this command will be named ".feature_barcode_ref.fa" and located in the provided output directory. + +Please note that the "step" argument represents the starting step of the series of external commands. If "step" is set to 1, then "mkdir" will be assigned step 1, "create_t2g" will be assigned step 2, and "create_fasta" will be assigned step 3. Therefore, the step of any future command after the `feature_barcode_ref` commands should not be less than 4. -**Input**: o: an object +**Example Usage** -**Output**: An object with the same information and layout as the original object's *Recommended Configuration* section but contains only the missing fields with a `null`. +.. code-block:: jsonnet + # import the utility library + local utils=std.extVar("__utils"); + + local feature_barcode_ref = utils.feature_barcode_ref( + 1, # start step number + "feature_barcode.csv", # feature barcode csv + 1, # name_column + 5, # barcode column + "feature_barcode_ref" # output path + ) + +The resulting object will look like the following: + +.. code-block:: jsonnet + + { + # hidden, system fields + step :: 1, + last_step :: 3, + csv :: "feature_barcode.csv", + output :: "./feature_barcode_ref", + ref_seq :: "./feature_barcode_ref/.feature_barcode_ref.fa", + t2g_map :: "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv", + + # external command records + mkdir : { + active : true, + step: step, + program_name: "mkdir", + arguments: ["-p", "./feature_barcode_ref"] + }, + create_t2g : { + active : true, + step: step + 1, + program_name: "awk", + arguments: ["-F","','","'NR>1 {sub(/ /,\"_\",$1);print $1\"\\t\"$1}'", csv, ">", "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv"], + }, + + create_fasta : { + active : true, + step: step + 2, + program_name: "awk", + arguments: ["-F","','","'NR>1 {sub(/ /,\"_\",$1);print \">\"$1\"\\n\"$5}'", csv, ">", "./feature_barcode_ref/.feature_barcode_ref.fa"] + }, + ref_type :: { + type :: "direct_ref", + arguments :: {step: step, csv: csv, output: output}, + t2g_map :: "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv", + "--ref-seq" : "./feature_barcode_ref/.feature_barcode_ref.fa", + } + } -This function will recursively traverse the *Recommended Configuration* main section to find all fields with a null value and return those fields as the original layout of *Recommended Configuration*. -utils.get_missing_args(o) +barcode_translation """""""""""""""""""""""""""""""""""""""""""""" -**Input**: o: an object +**Input**: +- `step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define five command records with incremental step numbers according to the provided step number. +- `url`: A string representing the downloadable URL to the barcode mapping file. You can use [this URL](https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz) for 10xv3 data. +- `quant_cb`: A string representing the path to the cell barcode file. Usually, this is at `af_quant/alevin/quants_mat_rows.txt` in the simpleaf quant command output directory. +- `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. + +**Output**: An object containing five external command records, including "mkdir," "fetch_cb_translation_file," "unzip_cb_translation_file," "backup_bc_file," and "barcode_translation." + +This function defines five external command records: +1. "mkdir": This command calls the "mkdir" shell program to create the output directory recursively if it doesn't exist. +2. "fetch_cb_translation_file": This command calls "wget" to fetch the barcode mapping file. The expected output file of this command will be called ".barcode.txt.gz," located in the provided output directory. +3. "unzip_cb_translation_file": This command calls "gunzip" to decompress the barcode mapping file. The expected output file of this command will be called ".barcode.txt," located in the provided output directory. +4. "backup_bc_file": This command calls "mv" to rename the provided barcode file. The expected output file of this command will have the same path as the provided barcode file but with a ".bkp" suffix. +5. "barcode_translation": This command calls "awk" to convert the barcodes in the provided barcode file according to the barcode translation file. The expected output file will be put at the provided `quant_cb` path. + +Notice that the "step" argument represents the starting step of the series of external commands. If "step" is set to 1, then "mkdir" will be assigned as step 1, "fetch_cb_translation_file" will be assigned step 2, and so on. Therefore, the step of any future command after the `barcode_translation` commands should not be less than 6. +**Example Usage** + +.. code-block:: jsonnet + + # import the utility library + local utils=std.extVar("__utils"); + local url = "https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz"; + local quant_cb = "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt"; + + local barcode_translation = utils.barcode_translation( + 1, # start step number + url, + quant_cb, + "simpeaf_quant/af_quant/alevin" # output path + ) + +The resulting object will look like the following: + +.. code-block:: jsonnet + + { + step :: 1, + last_step :: 5, + url :: "https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz", + quant_cb :: "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt", + output :: "simpeaf_quant/af_quant/alevin", + mkdir : { + active : true, + step : step, + program_name : "mkdir", + arguments : ["-p", "simpeaf_quant/af_quant/alevin"] + }, + + fetch_cb_translation_file : { + active : true, + step : step + 1, + program_name : "wget", + arguments : ["-O", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", "https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz"], + }, + + unzip_cb_translation_file : { + active : true, + step : step + 2, + "program_name" : "gunzip", + "arguments": ["-c", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", ">", "simpeaf_quant/af_quant/alevin/.barcode.txt"], + }, + + backup_bc_file : { + active : true, + step: step + 3, + program_name: "mv", + arguments: ["simpeaf_quant/af_quant/alevin/quants_mat_rows.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp"], + }, + + // Translate RNA barcode to feature barcode + barcode_translation : { + active : true, + step: step + 4, + program_name: "awk", + arguments: ["'FNR==NR {dict[$1]=$2; next} {$1=($1 in dict) ? dict[$1] : $1}1'", "simpeaf_quant/af_quant/alevin/.barcode.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp", ">", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt"], + }, + }, + +utils.get(o, f, use_default = false, default = null) +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +**Input**: o: an object, f: the target field name, use_default: boolean, default: any valid type + +**Output**: Return the target field *f* in the given object if the object has a sub-field called *f*. Otherwise, + - if *use_default* is *true*, return the value of the *default* argument (defualtly *null*). + - if *use_default* is false, raise an error. + +This function tries to (non-recursively) get a sub-field in the provided object and return it. If the field doesn't exist, then it either returns a default value or raises an error. + +**Example Usage** + +.. code-block:: jsonnet + + local utils = std.extVar("__utils"); + + local splici_args = { + gtf : "genes.gtf", + fasta : "genome.fa", + rlen : 91, + }; + + { + default_behavior : utils.get(splici_args, "gtf") # this will return "genes.gtf", -**Output**: An object with the same layout as the original object but only contains the missing fields with a `null`. + not_exist : utils.get(splici_args, "I do not exist") # raise error + + provide_default : utils.get(splici_args, "I do not exist", true, "but I have a default value") # this yields "but I have a default value" -This function will recursively traverse the object to find all fields with a null value and return those fields in the same layout as the original object. + } From 4fa509fcc482577ac3731e4b0b311e5ed6d2fcd7 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 18:33:53 -0400 Subject: [PATCH 04/13] fix typo in doc --- docs/source/workflow-utility-library.rst | 136 ++++++++++++----------- 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 3d69ae8..afacf89 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -4,7 +4,7 @@ Simpleaf workflow utility library To ease the development of *simpleaf* workflow templates, the *simpleaf* team provides not only some `built-in variables `_, but also a workflow utility library, which will be automatically passed to the `internal Jsonnet engine `_ of *simpleaf* when parsing a workflow template as the ``__utils`` external variable. One can receive this variable in their templates by adding ``utils=std.extVar("__utils")``, and use the functions in the utility library by calling ``utils.function_name(args)``, where *function_name* should be replaced by an actual function name listed below. To be consistent with the `Jsonnet official documentation `_, here we will list the function signatures with a brief description. One can find the function definitions on `this page `_. Import the utility library -'''''''''''''''''''''''''' +"""""""""""""""""""""""""""""""""""""""""""""" As the built-in variables are provided by *simpleaf* to its internal Jsonnet engine, they will be unavailable if we want to parse the template directly using Jsonnet or Jrsonnet. Therefore, when debugging templates that utilize the utility library with an external Jsonnet engine, we must manually provide the ``__utils`` external variable to Jsonnet and either copy and paste the library file to the same directory as the template file, which is the default library searching path when calling jsonnet, or provide the directory containing the library as an additional library searching path. Although in the following code chunk, we show the code for both ways, we only need to select one in practice. Here we assume that *simpleaf* has been configured correctly, i.e., the ``ALEVIN_FRY_HOME`` environment variable has been set and a local copy of the protocol-estuary exists. If not, one can directly obtain the library file from its GitHub repository. @@ -29,14 +29,15 @@ To do this, we recommend adding the following code at the beginning of your work local utils=std.extVar("__utils"); -ref_type +utils.ref_type(o) """""""""""""""""""""""""""""""""""""""""""""" -**Input**: `o`: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, `{type: "spliceu", spliceu: {gtf: null, fasta: null}}`. +**Input**: `o`: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, `{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa"}}`. -**Output**: An object with the ``simpleaf index`` arguments that are related to the specified reference type in the input object. +**Output**: An object with the *simpleaf index* arguments that are related to the specified reference type in the input object. This function has four modes (reference types), triggered by the ``type`` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The four modes are: + - *spliceu* (*spliced+unspliced* reference): The required fields are: - ``gtf``: A string representing the path to a gene annotation GTF file. - ``fasta``: A string representing the path to a reference genome FASTA file. @@ -51,10 +52,9 @@ This function has four modes (reference types), triggered by the ``type`` field - ``index``: A string representing the path to an existing index directory. - ``t2g_map``: A string representing the path to a transcript-to-gene mapping file. -**Wrapper functions**: We also provide separate functions for each of the four modes, ``splici``, ``spliceu``, ``direct_ref``, and ``existing_index``, which are thin wrappers of ``ref_type``. These four functions take an object containing their required fields introduced above. +**Wrapper functions**: We also provide separate functions for each of the four modes, ``utils.splici``, ``utils.spliceu``, ``utils.direct_ref``, and ``utils.existing_index``, which are thin wrappers of ``utils.ref_type``. These four functions take an object containing their required fields introduced above. -I've corrected the formatting and typos in the text. **Example Usage** .. code-block:: jsonnet @@ -65,12 +65,12 @@ I've corrected the formatting and typos in the text. local splici_args = { gtf : "genes.gtf", fasta : "genome.fa", - rlen : 91, + rlen : 91 }; local ref_type = utils.ref_type({ type : "splici", - splici : splici_args, + splici : splici_args }) local splici = utils.splici(splici_args); @@ -88,18 +88,19 @@ In the above example, the objects `ref_type` and `splici` are identical and look "--ref-type" : "splici", "--fasta" : "genome.fa", "--gtf" : "genes.gtf", - "--rlen" : 91, + "--rlen" : 91 } -simpleaf_index +utils.simpleaf_index(step, ref_type, arguments, output) """""""""""""""""""""""""""""""""""""""""""""" **Input**: + - *step*: An integer indicating the step number (execution order) of this simpleaf command record in the workflow. -- *ref_type*: A ``ref_type`` object returned by calling `utils.ref_type` or any object with the same format. -- *arguments*: An object in which each field represents a ``simpleaf index`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. -- *output*: A string that will be passed to `simpleaf index --output`. +- *ref_type*: A ``ref_type`` object returned by calling ``utils.ref_type`` or any object with the same format. +- *arguments*: An object in which each field represents a *simpleaf index* argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. +- *output*: A string that will be passed to ``simpleaf index --output``. **Output**: A well-defined ``simpleaf index`` command record. @@ -128,15 +129,16 @@ simpleaf_index splici, # ref_type, arguments, "./simpleaf_index" # output directory - )}; + ); The `simpleaf_index` object in the above code chunk will be -.. code-block:: JSON +.. code-block:: jsonnet + { # hidden, system fields - ref_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `ref_type` + ref_type :: {}, # hidden field. The actual contents are omitted. see above example code for function `ref_type` arguments :: {active : true, "--use-piscem" : true}, # hidden field output :: "./simpleaf_index", # hidden field index :: "./simpleaf_index/index", # hidden field @@ -150,21 +152,24 @@ The `simpleaf_index` object in the above code chunk will be "--gtf" : "genes.gtf", "--fasta" : "genome.fa", "--rlen" : 91, - "--use-piscem" : true, + "--use-piscem" : true } -map_type + +utils.map_type(o, simpleaf_index = {}) """""""""""""""""""""""""""""""""""""""""""""" **Input**: + - `o`: an object with - a `type` field, and - an object field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}`. -- `simpleaf_index`: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. +- `simpleaf_index`: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. The default value is an empty object. **Output**: An object with the `simpleaf quant` arguments that are related to the specified map type in the input object. This function has two modes (map types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The two modes are: + - `map_reads`: Map reads against the provided index or an index built from a previous step. The required fields are - `reads1`: A string representing the path to a gene annotation GTF file, - `reads2`: A string representing the path to a reference genome FASTA file. @@ -172,15 +177,16 @@ This function has two modes (map types), triggered by the `type` field in the in - `map_dir`: A string representing the path to the mapping result directory, - `t2g_map`: A string representing the path to a transcript-to-gene mapping file. -**Wrapper functions**: We also provide separate functions for each of the two modes, `map_reads` and `existing_mappings`, which are thin wrappers of `map_type`. These two functions take an object containing their required fields introduced above. +**Wrapper functions**: We also provide separate functions for each of the two modes, `utils.map_reads` and `utils.existing_mappings`, which are thin wrappers of `map_type`. These two functions take an object containing their required fields introduced above. **Example Usage** .. code-block:: jsonnet + # import the utility library local utils=std.extVar("__utils"); - local simpleaf_index = ... # The return of object of simpleaf_index function in its example usage + local simpleaf_index = {}; # The return of object of simpleaf_index function in its example usage local map_reads_args = { reads1 : "reads1.fastq", @@ -211,23 +217,24 @@ In the above example, the objects `map_type` and `map_reads` are identical and l } -cell_filt_type +utils.cell_filt_type(o) """""""""""""""""""""""""""""""""""""""""""""" - **Input**: + - `o`: an object with a `type` field, and an argument field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "explicit_pl", "explicit_pl": "whitelist.txt"}` **Output**: An object with the `simpleaf quant` arguments that are related to the specified cell filtering type in the input object. -This function has five modes (cell filtering types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. For more details, please refer to the online documentation of [simpleaf quant](https://simpleaf.readthedocs.io/en/latest/quant-command.html) and [alevin-fry](https://alevin-fry.readthedocs.io/en/latest/). The five modes are: -- `unfiltered_pl`: No cell filtering but correcting cell barcodes by an external or default (only works for 10X Chromium V2 and V3). The corresponding argument value field can be `true` (using the default whitelist if in `10xv2` and `10xv3` chemistry), or a string representing the path to an unfiltered permit list file. +This function has five modes (cell filtering types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. For more details, please refer to the online documentation of `simpleaf quant `_ and `alevin-fry `_. The five modes are: + +- `unfiltered_pl`: No cell filtering but correcting cell barcodes by an external or default (only works for 10X Chromium V2 and V3). The corresponding argument value field can be ``true`` (using the default whitelist if in `10xv2` and `10xv3` chemistry), or a string representing the path to an unfiltered permit list file. - `knee`: Knee point-based filtering. The corresponding argument value field must be `true` if selected. - `forced`: Use a forced number of cells. The corresponding argument field must be an integer representing the number of cells that can pass the filtering. - `expect`: Use the expected number of cells. The corresponding argument field must be an integer representing the expected number of cells. - `explicit_pl`: Use a filtered, explicit permit list. The corresponding argument field must be a string representing the path to a cell barcode permit list file. -**Wrapper functions**: We also provide separate functions for each mode, `unfiltered_pl`, `knee`, `forced`, `expect`, and `explicit_pl`, which are thin wrappers of `cell_filt_type`. These functions take an object containing their required fields introduced above. +**Wrapper functions**: We also provide a separate function for each mode, `utils.unfiltered_pl`, `utils.knee`, `utils.forced`, `utils.expect`, and `utils.explicit_pl`, which are thin wrappers of `utils.cell_filt_type`. These functions take an object containing their required fields introduced above. **Example Usage** @@ -236,7 +243,7 @@ This function has five modes (cell filtering types), triggered by the `type` fie # import the utility library local utils=std.extVar("__utils"); - local cell_filt_args = { + local unfiltered_pl_args = { unfiltered_pl : true, }; @@ -257,18 +264,19 @@ In the above example, the objects `cell_filt_type` and `unfiltered_pl` are ident arguments :: true, # hidden field # fields shown in the manifest - "--unfiltered-pl" : true, + "--unfiltered-pl" : true } -simpleaf_quant +simpleaf_quant(step, map_type, cell_filt_type, output) """""""""""""""""""""""""""""""""""""""""""""" **Input**: -- step : An integer indicating the step number (execution order) of this simpleaf command record in the workflow. -- map_type : A ``map_type`` object returned by calling `utils.map_type` or any object with the same format. -- cell_filt_type : A ``cell_filt_type`` object returned by calling `utils.cell_filt_type` or any object with the same format. -- arguments : an object in which each field represents a ``simpleaf quant`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. -- output : A string that will be passed to `simpleaf quant --output`. + +- `step` : An integer indicating the step number (execution order) of this simpleaf command record in the workflow. +- `map_type` : A ``map_type`` object returned by calling ``utils.map_type`` or any object with the same format. +- `cell_filt_type` : A ``cell_filt_type`` object returned by calling ``utils.cell_filt_type`` or any object with the same format. +- `arguments` : an object in which each field represents a ``simpleaf quant`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. +- `output` : A string that will be passed to ``simpleaf quant --output``. **Output**: A well-defined ``simpleaf quant`` command record. @@ -279,10 +287,6 @@ simpleaf_quant # import the utility library local utils=std.extVar("__utils"); - local map_type = utils.map_type({...}); # Please refer to the example usage of function `map_reads` for full details. - - local cell_filt_type = utils.cell_filt_type({...}); # Please refer to the example usage of function `cell_filt_type` for full details. - local arguments = { active : true, "--chemistry" : "10xv3", @@ -291,8 +295,8 @@ simpleaf_quant local simpleaf_quant = utils.simpleaf_quant( 2, # step number - map_type, - cell_filt_type, + map_type, # defined in the example usage of function `map_reads` + cell_filt_type, # defined in the example usage of function `cell_filt_type` arguments, "./simpleaf_quant" # output directory )}; @@ -301,10 +305,11 @@ simpleaf_quant The `simpleaf_quant` object in the above code chunk will be .. code-block:: JSON + { # hidden, system fields - map_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `map_reads` - cell_filt_type :: {...}, # hidden field. The actual contents are omitted. see above example code for function `cell_filt_type` + map_type :: {}, # hidden field. The actual contents are omitted. see above example code for function `map_reads` + cell_filt_type :: {}, # hidden field. The actual contents are omitted. see above example code for function `cell_filt_type` arguments :: {active : true, "--chemistry" : "10xv3", "--resolution" : "cr-like"}, # hidden field output :: "./simpleaf_quant", # hidden field @@ -320,33 +325,35 @@ The `simpleaf_quant` object in the above code chunk will be "--reads2": "reads2.fastq", "--resolution": "cr-like", "--t2g-map": "./workflow_output/simpleaf_index/index/t2g_3col.tsv", - "--unfiltered-pl": true, + "--unfiltered-pl": true } -feature_barcode_ref +feature_barcode_ref(start_step, csv, name_col, barcode_col, output) """""""""""""""""""""""""""""""""""""""""""""" -I've fixed the typos in your input text: **Input**: -- `step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define three command records with incremental step numbers according to the provided step number. + +- `start_step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define three command records with incremental step numbers according to the provided step number. - `csv`: A string representing the path to the "feature_barcode.csv" file of the dataset. - `name_col`: An integer representing the column index of the feature name column in the feature barcode CSV file. - `barcode_col`: An integer representing the column index of the feature barcode sequence column in the feature barcode CSV file. - `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. -**Output**: An object containing three external command records, including "mkdir," "create_t2g," and "create_fasta," and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, +**Output**: An object containing three external command records, including `mkdir`, `create_t2g`, and `create_fasta`, and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, This function defines three external command records: -1. "mkdir": This command calls the "mkdir" shell program to create the output directory recursively if it doesn't exist. -2. "create_t2g": This command calls `awk` to create a transcript-to-gene mapping TSV file according to the input "csv" file, in which the transcript ID and gene ID of each feature barcode are identical. The expected output file of this command will be named ".feature_barcode_ref_t2g.tsv" and located in the provided output directory. -3. "create_fasta": This command calls `awk` to create a FASTA file according to the input "csv" file, in which each feature barcode is a FASTA record. The expected output file of this command will be named ".feature_barcode_ref.fa" and located in the provided output directory. -Please note that the "step" argument represents the starting step of the series of external commands. If "step" is set to 1, then "mkdir" will be assigned step 1, "create_t2g" will be assigned step 2, and "create_fasta" will be assigned step 3. Therefore, the step of any future command after the `feature_barcode_ref` commands should not be less than 4. +1. `mkdir`: This command calls the `mkdir` shell program to create the output directory recursively if it doesn't exist. +2. `create_t2g`: This command calls `awk` to create a transcript-to-gene mapping TSV file according to the input `csv` file, in which the transcript ID and gene ID of each feature barcode are identical. The expected output file of this command will be named ".feature_barcode_ref_t2g.tsv" and located in the provided output directory. +3. `create_fasta`: This command calls `awk` to create a FASTA file according to the input `csv` file, in which each feature barcode is a FASTA record. The expected output file of this command will be named ".feature_barcode_ref.fa" and located in the provided output directory. + +Please note that the `start_step` argument represents the starting step of the series of external commands. If `start_step` is set to 1, then `mkdir` will be assigned step 1, `create_t2g` will be assigned step 2, and so on. Therefore, the step of any future command after the `utils.feature_barcode_ref` commands should not be less than 4. **Example Usage** .. code-block:: jsonnet + # import the utility library local utils=std.extVar("__utils"); @@ -392,33 +399,35 @@ The resulting object will look like the following: arguments: ["-F","','","'NR>1 {sub(/ /,\"_\",$1);print \">\"$1\"\\n\"$5}'", csv, ">", "./feature_barcode_ref/.feature_barcode_ref.fa"] }, ref_type :: { - type :: "direct_ref", - arguments :: {step: step, csv: csv, output: output}, + type : "direct_ref", t2g_map :: "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv", "--ref-seq" : "./feature_barcode_ref/.feature_barcode_ref.fa", } } -barcode_translation +barcode_translation(start_step, url, quant_cb, output) """""""""""""""""""""""""""""""""""""""""""""" **Input**: -- `step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define five command records with incremental step numbers according to the provided step number. -- `url`: A string representing the downloadable URL to the barcode mapping file. You can use [this URL](https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz) for 10xv3 data. + +- `start_step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define five command records with incremental step numbers according to the provided step number. +- `url`: A string representing the downloadable URL to the barcode mapping file. You can use `this URL `_ for 10xv3 data. - `quant_cb`: A string representing the path to the cell barcode file. Usually, this is at `af_quant/alevin/quants_mat_rows.txt` in the simpleaf quant command output directory. - `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. -**Output**: An object containing five external command records, including "mkdir," "fetch_cb_translation_file," "unzip_cb_translation_file," "backup_bc_file," and "barcode_translation." +**Output**: An object containing five external command records, including `mkdir`, `fetch_cb_translation_file`, `unzip_cb_translation_file`, `backup_bc_file`, and `barcode_translation`. This function defines five external command records: -1. "mkdir": This command calls the "mkdir" shell program to create the output directory recursively if it doesn't exist. -2. "fetch_cb_translation_file": This command calls "wget" to fetch the barcode mapping file. The expected output file of this command will be called ".barcode.txt.gz," located in the provided output directory. -3. "unzip_cb_translation_file": This command calls "gunzip" to decompress the barcode mapping file. The expected output file of this command will be called ".barcode.txt," located in the provided output directory. -4. "backup_bc_file": This command calls "mv" to rename the provided barcode file. The expected output file of this command will have the same path as the provided barcode file but with a ".bkp" suffix. -5. "barcode_translation": This command calls "awk" to convert the barcodes in the provided barcode file according to the barcode translation file. The expected output file will be put at the provided `quant_cb` path. -Notice that the "step" argument represents the starting step of the series of external commands. If "step" is set to 1, then "mkdir" will be assigned as step 1, "fetch_cb_translation_file" will be assigned step 2, and so on. Therefore, the step of any future command after the `barcode_translation` commands should not be less than 6. +1. `mkdir`: This command calls the `mkdir` shell program to create the output directory recursively if it doesn't exist. +2. `fetch_cb_translation_file`: This command calls `wget` to fetch the barcode mapping file. The expected output file of this command will be called ".barcode.txt.gz", located in the provided output directory. +3. `unzip_cb_translation_file`: This command calls `gunzip` to decompress the barcode mapping file. The expected output file of this command will be called ".barcode.txt", located in the provided output directory. +4. `backup_bc_file`: This command calls `mv` to rename the provided barcode file. The expected output file of this command will have the same path as the provided barcode file but with a `.bkp` suffix. +5. `barcode_translation`: This command calls `awk` to convert the barcodes in the provided barcode file according to the barcode translation file. The expected output file will be put at the provided `quant_cb` path. + +Notice that the `start_step` argument represents the starting step of the series of external commands. If `start_step` is set to 1, then `mkdir` will be assigned as step 1, `fetch_cb_translation_file` will be assigned step 2, and so on. Therefore, the step of any future command after the `barcode_translation` commands should not be less than 6. + **Example Usage** .. code-block:: jsonnet @@ -488,6 +497,7 @@ utils.get(o, f, use_default = false, default = null) **Input**: o: an object, f: the target field name, use_default: boolean, default: any valid type **Output**: Return the target field *f* in the given object if the object has a sub-field called *f*. Otherwise, + - if *use_default* is *true*, return the value of the *default* argument (defualtly *null*). - if *use_default* is false, raise an error. From 9d65c9659cf01bc5a6b625201c64bbdf7a647262 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 18:45:52 -0400 Subject: [PATCH 05/13] make doc prettier --- docs/source/workflow-utility-library.rst | 91 +++++++++++++----------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index afacf89..594f8e3 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -21,7 +21,7 @@ As the built-in variables are provided by *simpleaf* to its internal Jsonnet eng where ``--ext-code`` is the flag for passing an external variable, and ``--jpath`` specifies the library searching path. -Although *simpleaf* automatically provides the utility library as the external variable `__utils`, we must receive this external variable in our template before starting using the functions provided in this library. +Although *simpleaf* automatically provides the utility library as the external variable ``__utils``, we must receive this external variable in our template before starting using the functions provided in this library. To do this, we recommend adding the following code at the beginning of your workflow template. @@ -32,7 +32,7 @@ To do this, we recommend adding the following code at the beginning of your work utils.ref_type(o) """""""""""""""""""""""""""""""""""""""""""""" -**Input**: `o`: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, `{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa"}}`. +**Input**: ``o``: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, ``{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa"}}``. **Output**: An object with the *simpleaf index* arguments that are related to the specified reference type in the input object. @@ -75,7 +75,7 @@ This function has four modes (reference types), triggered by the ``type`` field local splici = utils.splici(splici_args); -In the above example, the objects `ref_type` and `splici` are identical and look like the following: +In the above example, the objects ``ref_type`` and ``splici`` are identical and look like the following: .. code-block:: jsonnet @@ -97,12 +97,12 @@ utils.simpleaf_index(step, ref_type, arguments, output) **Input**: -- *step*: An integer indicating the step number (execution order) of this simpleaf command record in the workflow. -- *ref_type*: A ``ref_type`` object returned by calling ``utils.ref_type`` or any object with the same format. -- *arguments*: An object in which each field represents a *simpleaf index* argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. -- *output*: A string that will be passed to ``simpleaf index --output``. +- ``step``: An integer indicating the step number (execution order) of this simpleaf command record in the workflow. +- ``ref_type``: A ``ref_type`` object returned by calling ``utils.ref_type`` or any object with the same format. +- ``arguments``: An object in which each field represents a *simpleaf index* argument. Furthermore, there must be a field called ``active`` representing the active state of this `simpleaf index` command. +- ``output``: A string representing the output directory of the `simpleaf index` command. -**Output**: A well-defined ``simpleaf index`` command record. +**Output**: A well-defined *simpleaf index* command record. **Example Usage** @@ -132,7 +132,7 @@ utils.simpleaf_index(step, ref_type, arguments, output) ); -The `simpleaf_index` object in the above code chunk will be +The ``simpleaf_index`` object in the above code chunk will be .. code-block:: jsonnet @@ -161,23 +161,23 @@ utils.map_type(o, simpleaf_index = {}) **Input**: -- `o`: an object with - - a `type` field, and - - an object field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}`. -- `simpleaf_index`: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. The default value is an empty object. +- ``o``: an object with + - a ``type`` field, and + - an object field with the name specified by the ``type`` field. Other fields will be ignored. For example, ``{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}``. +- ``simpleaf_index``: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. The default value is an empty object. **Output**: An object with the `simpleaf quant` arguments that are related to the specified map type in the input object. This function has two modes (map types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The two modes are: - `map_reads`: Map reads against the provided index or an index built from a previous step. The required fields are - - `reads1`: A string representing the path to a gene annotation GTF file, - - `reads2`: A string representing the path to a reference genome FASTA file. + - ``reads1``: A string representing the path to a gene annotation GTF file, + - ``reads2``: A string representing the path to a reference genome FASTA file. - `existing_mappings`: Skip mapping and use the existing mapping results. The required fields are - - `map_dir`: A string representing the path to the mapping result directory, - - `t2g_map`: A string representing the path to a transcript-to-gene mapping file. + - ``map_dir``: A string representing the path to the mapping result directory, + - ``t2g_map``: A string representing the path to a transcript-to-gene mapping file. -**Wrapper functions**: We also provide separate functions for each of the two modes, `utils.map_reads` and `utils.existing_mappings`, which are thin wrappers of `map_type`. These two functions take an object containing their required fields introduced above. +**Wrapper functions**: We also provide separate functions for each of the two modes, ``utils.map_reads`` and ``utils.existing_mappings``, which are thin wrappers of ``utils.map_type``. These two functions take an object containing their required fields introduced above. **Example Usage** @@ -200,7 +200,7 @@ This function has two modes (map types), triggered by the `type` field in the in local map_reads = utils.map_reads(map_reads_args); -In the above example, the objects `map_type` and `map_reads` are identical and look like the following: +In the above example, the objects ``map_type`` and ``map_reads`` are identical and look like the following: .. code-block:: jsonnet @@ -222,7 +222,7 @@ utils.cell_filt_type(o) **Input**: -- `o`: an object with a `type` field, and an argument field with the name specified by the `type` field. Other fields will be ignored. For example, `{"type": "explicit_pl", "explicit_pl": "whitelist.txt"}` +- ``o``: an object with a ``type`` field, and an argument field with the name specified by the ``type`` field. Other fields will be ignored. For example, ``{"type": "explicit_pl", "explicit_pl": "whitelist.txt"}`` **Output**: An object with the `simpleaf quant` arguments that are related to the specified cell filtering type in the input object. @@ -234,7 +234,7 @@ This function has five modes (cell filtering types), triggered by the `type` fie - `expect`: Use the expected number of cells. The corresponding argument field must be an integer representing the expected number of cells. - `explicit_pl`: Use a filtered, explicit permit list. The corresponding argument field must be a string representing the path to a cell barcode permit list file. -**Wrapper functions**: We also provide a separate function for each mode, `utils.unfiltered_pl`, `utils.knee`, `utils.forced`, `utils.expect`, and `utils.explicit_pl`, which are thin wrappers of `utils.cell_filt_type`. These functions take an object containing their required fields introduced above. +**Wrapper functions**: We also provide a separate function for each mode, ``utils.unfiltered_pl``, ``utils.knee``, ``utils.forced``, ``utils.expect``, and ``utils.explicit_pl``, which are thin wrappers of ``utils.cell_filt_type``. These functions take an object containing their required fields introduced above. **Example Usage** @@ -272,13 +272,13 @@ simpleaf_quant(step, map_type, cell_filt_type, output) **Input**: -- `step` : An integer indicating the step number (execution order) of this simpleaf command record in the workflow. -- `map_type` : A ``map_type`` object returned by calling ``utils.map_type`` or any object with the same format. -- `cell_filt_type` : A ``cell_filt_type`` object returned by calling ``utils.cell_filt_type`` or any object with the same format. -- `arguments` : an object in which each field represents a ``simpleaf quant`` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. -- `output` : A string that will be passed to ``simpleaf quant --output``. +- ``step`` : An integer indicating the step number (execution order) of this simpleaf command record in the workflow. +- ``map_type`` : A `map_type` object returned by calling `utils.map_type` or any object with the same format. +- ``cell_filt_type`` : A `cell_filt_type` object returned by calling `utils.cell_filt_type` or any object with the same format. +- ``arguments`` : an object in which each field represents a `simpleaf quant` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. +- ``output`` : A string representing the output directory of this `simpleaf quant` command. -**Output**: A well-defined ``simpleaf quant`` command record. +**Output**: A well-defined `simpleaf quant` command record. **Example Usage** @@ -299,12 +299,12 @@ simpleaf_quant(step, map_type, cell_filt_type, output) cell_filt_type, # defined in the example usage of function `cell_filt_type` arguments, "./simpleaf_quant" # output directory - )}; + ); -The `simpleaf_quant` object in the above code chunk will be +The ``simpleaf_quant`` object in the above code chunk will be -.. code-block:: JSON +.. code-block:: jsonnet { # hidden, system fields @@ -334,11 +334,11 @@ feature_barcode_ref(start_step, csv, name_col, barcode_col, output) **Input**: -- `start_step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define three command records with incremental step numbers according to the provided step number. -- `csv`: A string representing the path to the "feature_barcode.csv" file of the dataset. -- `name_col`: An integer representing the column index of the feature name column in the feature barcode CSV file. -- `barcode_col`: An integer representing the column index of the feature barcode sequence column in the feature barcode CSV file. -- `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. +- ``start_step``: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define three command records with incremental step numbers according to the provided step number. +- ``csv``: A string representing the path to the "feature_barcode.csv" file of the dataset. +- ``name_col``: An integer representing the column index of the feature name column in the feature barcode CSV file. +- ``barcode_col``: An integer representing the column index of the feature barcode sequence column in the feature barcode CSV file. +- ``output``: A string representing the parent output directory of the result files. It will be created if it doesn't exist. **Output**: An object containing three external command records, including `mkdir`, `create_t2g`, and `create_fasta`, and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, @@ -411,10 +411,10 @@ barcode_translation(start_step, url, quant_cb, output) **Input**: -- `start_step`: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define five command records with incremental step numbers according to the provided step number. -- `url`: A string representing the downloadable URL to the barcode mapping file. You can use `this URL `_ for 10xv3 data. -- `quant_cb`: A string representing the path to the cell barcode file. Usually, this is at `af_quant/alevin/quants_mat_rows.txt` in the simpleaf quant command output directory. -- `output`: A string representing the parent output directory of the result files. It will be created if it doesn't exist. +- ``start_step``: An integer indicating the starting step number (execution order) of the series of command records in the workflow. This function will define five command records with incremental step numbers according to the provided step number. +- ``url``: A string representing the downloadable URL to the barcode mapping file. You can use `this URL `_ for 10xv3 data. +- ``quant_cb``: A string representing the path to the cell barcode file. Usually, this is at `af_quant/alevin/quants_mat_rows.txt` in the simpleaf quant command output directory. +- ``output``: A string representing the parent output directory of the result files. It will be created if it doesn't exist. **Output**: An object containing five external command records, including `mkdir`, `fetch_cb_translation_file`, `unzip_cb_translation_file`, `backup_bc_file`, and `barcode_translation`. @@ -494,12 +494,17 @@ The resulting object will look like the following: utils.get(o, f, use_default = false, default = null) """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -**Input**: o: an object, f: the target field name, use_default: boolean, default: any valid type +**Input**: + +- ``o``: an object, +- ``f``: the target field name, +- ``use_default``: boolean, +- ``default``: a default value returned if the target field doesn't exist. -**Output**: Return the target field *f* in the given object if the object has a sub-field called *f*. Otherwise, +**Output**: Return the target field *f* in the given object if the object has a sub-field called ``f``. Otherwise, - - if *use_default* is *true*, return the value of the *default* argument (defualtly *null*). - - if *use_default* is false, raise an error. + - if ``use_default`` is ``true``, return the value of the ``default`` argument. + - if ``use_default`` is false, raise an error. This function tries to (non-recursively) get a sub-field in the provided object and return it. If the field doesn't exist, then it either returns a default value or raises an error. From 5d67af3b5e507f1fb62a6f77291e813d2641242a Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 19:29:30 -0400 Subject: [PATCH 06/13] make doc prettier --- docs/source/workflow-utility-library.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 594f8e3..77411f1 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -489,7 +489,7 @@ The resulting object will look like the following: program_name: "awk", arguments: ["'FNR==NR {dict[$1]=$2; next} {$1=($1 in dict) ? dict[$1] : $1}1'", "simpeaf_quant/af_quant/alevin/.barcode.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp", ">", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt"], }, - }, + } utils.get(o, f, use_default = false, default = null) """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -503,8 +503,8 @@ utils.get(o, f, use_default = false, default = null) **Output**: Return the target field *f* in the given object if the object has a sub-field called ``f``. Otherwise, - - if ``use_default`` is ``true``, return the value of the ``default`` argument. - - if ``use_default`` is false, raise an error. +- if ``use_default`` is ``true``, return the value of the ``default`` argument. +- if ``use_default`` is false, raise an error. This function tries to (non-recursively) get a sub-field in the provided object and return it. If the field doesn't exist, then it either returns a default value or raises an error. From c3eb4b71bd499c2fdc9e8b9c18de6a816798e384 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 20:38:20 -0400 Subject: [PATCH 07/13] make doc prettierr --- docs/source/workflow-utility-library.rst | 48 ++++++++++++++++++------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 77411f1..530dd88 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -32,9 +32,15 @@ To do this, we recommend adding the following code at the beginning of your work utils.ref_type(o) """""""""""""""""""""""""""""""""""""""""""""" -**Input**: ``o``: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. For example, ``{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa"}}``. +**Input**: + +- ``o``: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. + + For example, ``{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa", "field_being_ignored": "ignore me"}}``. + +**Output**: -**Output**: An object with the *simpleaf index* arguments that are related to the specified reference type in the input object. +- An object with the *simpleaf index* arguments that are related to the specified reference type in the input object. This function has four modes (reference types), triggered by the ``type`` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The four modes are: @@ -102,7 +108,9 @@ utils.simpleaf_index(step, ref_type, arguments, output) - ``arguments``: An object in which each field represents a *simpleaf index* argument. Furthermore, there must be a field called ``active`` representing the active state of this `simpleaf index` command. - ``output``: A string representing the output directory of the `simpleaf index` command. -**Output**: A well-defined *simpleaf index* command record. +**Output**: + +- A well-defined *simpleaf index* command record. **Example Usage** @@ -163,10 +171,14 @@ utils.map_type(o, simpleaf_index = {}) - ``o``: an object with - a ``type`` field, and - - an object field with the name specified by the ``type`` field. Other fields will be ignored. For example, ``{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}``. + - an object field with the name specified by the ``type`` field. Other fields will be ignored. + + For example, ``{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}``. - ``simpleaf_index``: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. The default value is an empty object. -**Output**: An object with the `simpleaf quant` arguments that are related to the specified map type in the input object. +**Output**: + +- An object with the `simpleaf quant` arguments that are related to the specified map type in the input object. This function has two modes (map types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. The two modes are: @@ -222,9 +234,13 @@ utils.cell_filt_type(o) **Input**: -- ``o``: an object with a ``type`` field, and an argument field with the name specified by the ``type`` field. Other fields will be ignored. For example, ``{"type": "explicit_pl", "explicit_pl": "whitelist.txt"}`` +- ``o``: an object with a ``type`` field, and an argument field with the name specified by the ``type`` field. Other fields will be ignored. + + For example, ``{"type": "explicit_pl", "explicit_pl": "whitelist.txt", "field_being_ignored": "ignore me"}`` + +**Output**: -**Output**: An object with the `simpleaf quant` arguments that are related to the specified cell filtering type in the input object. +- An object with the `simpleaf quant` arguments that are related to the specified cell filtering type in the input object. This function has five modes (cell filtering types), triggered by the `type` field in the input object. When specifying a mode, the input object must contain an object field named by that mode and contain the required fields. Otherwise, an error will be raised. For more details, please refer to the online documentation of `simpleaf quant `_ and `alevin-fry `_. The five modes are: @@ -267,7 +283,7 @@ In the above example, the objects `cell_filt_type` and `unfiltered_pl` are ident "--unfiltered-pl" : true } -simpleaf_quant(step, map_type, cell_filt_type, output) +simpleaf_quant(step, map_type, cell_filt_type, arguments, output) """""""""""""""""""""""""""""""""""""""""""""" **Input**: @@ -278,7 +294,9 @@ simpleaf_quant(step, map_type, cell_filt_type, output) - ``arguments`` : an object in which each field represents a `simpleaf quant` argument. Furthermore, there must be a field called ``active`` representing the active state of this simpleaf index command. - ``output`` : A string representing the output directory of this `simpleaf quant` command. -**Output**: A well-defined `simpleaf quant` command record. +**Output**: + +- A well-defined `simpleaf quant` command record. **Example Usage** @@ -340,7 +358,9 @@ feature_barcode_ref(start_step, csv, name_col, barcode_col, output) - ``barcode_col``: An integer representing the column index of the feature barcode sequence column in the feature barcode CSV file. - ``output``: A string representing the parent output directory of the result files. It will be created if it doesn't exist. -**Output**: An object containing three external command records, including `mkdir`, `create_t2g`, and `create_fasta`, and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, +**Output**: + +- An object containing three external command records, including `mkdir`, `create_t2g`, and `create_fasta`, and a hidden object that follows the output format of `utils.ref_type` shown above. This `ref_type` object is of the `direct_ref` type. It can be used as the second argument of `utils.simpleaf_index`. In this `ref_type` object, This function defines three external command records: @@ -416,7 +436,9 @@ barcode_translation(start_step, url, quant_cb, output) - ``quant_cb``: A string representing the path to the cell barcode file. Usually, this is at `af_quant/alevin/quants_mat_rows.txt` in the simpleaf quant command output directory. - ``output``: A string representing the parent output directory of the result files. It will be created if it doesn't exist. -**Output**: An object containing five external command records, including `mkdir`, `fetch_cb_translation_file`, `unzip_cb_translation_file`, `backup_bc_file`, and `barcode_translation`. +**Output**: + +- An object containing five external command records, including `mkdir`, `fetch_cb_translation_file`, `unzip_cb_translation_file`, `backup_bc_file`, and `barcode_translation`. This function defines five external command records: @@ -501,7 +523,9 @@ utils.get(o, f, use_default = false, default = null) - ``use_default``: boolean, - ``default``: a default value returned if the target field doesn't exist. -**Output**: Return the target field *f* in the given object if the object has a sub-field called ``f``. Otherwise, +**Output**: + +- Return the target field *f* in the given object if the object has a sub-field called ``f``. Otherwise, - if ``use_default`` is ``true``, return the value of the ``default`` argument. - if ``use_default`` is false, raise an error. From 94b81deda0a2adebdda73192dac4c98535c4b1f2 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 20:48:58 -0400 Subject: [PATCH 08/13] make doc prettierrr --- docs/source/workflow-utility-library.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 530dd88..254b335 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -34,8 +34,9 @@ utils.ref_type(o) **Input**: -- ``o``: an object with a *type* field and an object field with the name specified by the *type* field. Other fields will be ignored. - +- ``o``: an object with + - a *type* field and + - an object field with the name specified by the *type* field. Other fields will be ignored. For example, ``{type: "spliceu", spliceu: {gtf: "genes.gtf", fasta: "genome.fa", "field_being_ignored": "ignore me"}}``. **Output**: @@ -234,7 +235,9 @@ utils.cell_filt_type(o) **Input**: -- ``o``: an object with a ``type`` field, and an argument field with the name specified by the ``type`` field. Other fields will be ignored. +- ``o``: an object with + - a ``type`` field, and + - an argument field with the name specified by the ``type`` field. Other fields will be ignored. For example, ``{"type": "explicit_pl", "explicit_pl": "whitelist.txt", "field_being_ignored": "ignore me"}`` From 7a7e97b67228fd698fdd96ad9e1b808af40d1b2c Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 20:55:06 -0400 Subject: [PATCH 09/13] make doc prettierrrr --- docs/source/workflow-utility-library.rst | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 254b335..aa59b20 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -208,8 +208,8 @@ This function has two modes (map types), triggered by the `type` field in the in local map_type = utils.map_type({ type : "map_reads", - map_reads : map_reads_args, - }) + map_reads : map_reads_args + }); local map_reads = utils.map_reads(map_reads_args); @@ -226,7 +226,7 @@ In the above example, the objects ``map_type`` and ``map_reads`` are identical a "--index" : "./workflow_output/simpleaf_index/index", "--t2g-map": "./workflow_output/simpleaf_index/index/t2g_3col.tsv", "--reads1" : "reads1.fastq", - "--reads2" : "reads2.fastq", + "--reads2" : "reads2.fastq" } @@ -263,12 +263,12 @@ This function has five modes (cell filtering types), triggered by the `type` fie local utils=std.extVar("__utils"); local unfiltered_pl_args = { - unfiltered_pl : true, + unfiltered_pl : true }; local cell_filt_type = utils.cell_filt_type({ type : "unfiltered_pl", - unfiltered_pl : unfiltered_pl_args, + unfiltered_pl : unfiltered_pl_args }) local unfiltered_pl = utils.unfiltered_pl(unfiltered_pl_args); @@ -412,7 +412,7 @@ The resulting object will look like the following: active : true, step: step + 1, program_name: "awk", - arguments: ["-F","','","'NR>1 {sub(/ /,\"_\",$1);print $1\"\\t\"$1}'", csv, ">", "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv"], + arguments: ["-F","','","'NR>1 {sub(/ /,\"_\",$1);print $1\"\\t\"$1}'", csv, ">", "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv"] }, create_fasta : { @@ -424,7 +424,7 @@ The resulting object will look like the following: ref_type :: { type : "direct_ref", t2g_map :: "./feature_barcode_ref/.feature_barcode_ref_t2g.tsv", - "--ref-seq" : "./feature_barcode_ref/.feature_barcode_ref.fa", + "--ref-seq" : "./feature_barcode_ref/.feature_barcode_ref.fa" } } @@ -490,21 +490,21 @@ The resulting object will look like the following: active : true, step : step + 1, program_name : "wget", - arguments : ["-O", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", "https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz"], + arguments : ["-O", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", "https://github.com/10XGenomics/cellranger/raw/master/lib/python/cellranger/barcodes/translation/3M-february-2018.txt.gz"] }, unzip_cb_translation_file : { active : true, step : step + 2, "program_name" : "gunzip", - "arguments": ["-c", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", ">", "simpeaf_quant/af_quant/alevin/.barcode.txt"], + "arguments": ["-c", "simpeaf_quant/af_quant/alevin/.barcode.txt.gz", ">", "simpeaf_quant/af_quant/alevin/.barcode.txt"] }, backup_bc_file : { active : true, step: step + 3, program_name: "mv", - arguments: ["simpeaf_quant/af_quant/alevin/quants_mat_rows.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp"], + arguments: ["simpeaf_quant/af_quant/alevin/quants_mat_rows.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp"] }, // Translate RNA barcode to feature barcode @@ -512,7 +512,7 @@ The resulting object will look like the following: active : true, step: step + 4, program_name: "awk", - arguments: ["'FNR==NR {dict[$1]=$2; next} {$1=($1 in dict) ? dict[$1] : $1}1'", "simpeaf_quant/af_quant/alevin/.barcode.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp", ">", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt"], + arguments: ["'FNR==NR {dict[$1]=$2; next} {$1=($1 in dict) ? dict[$1] : $1}1'", "simpeaf_quant/af_quant/alevin/.barcode.txt", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt.bkp", ">", "simpeaf_quant/af_quant/alevin/quants_mat_rows.txt"] }, } @@ -544,13 +544,13 @@ This function tries to (non-recursively) get a sub-field in the provided object local splici_args = { gtf : "genes.gtf", fasta : "genome.fa", - rlen : 91, + rlen : 91 }; { - default_behavior : utils.get(splici_args, "gtf") # this will return "genes.gtf", + default_behavior : utils.get(splici_args, "gtf"), # this will return "genes.gtf", - not_exist : utils.get(splici_args, "I do not exist") # raise error + not_exist : utils.get(splici_args, "I do not exist"), # raise error provide_default : utils.get(splici_args, "I do not exist", true, "but I have a default value") # this yields "but I have a default value" From 64994a2ea6ed264376fe6fd121c8a1882d0a9ac5 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 20:55:14 -0400 Subject: [PATCH 10/13] make doc prettierrrr --- docs/source/workflow-utility-library.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index aa59b20..4a62438 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -123,14 +123,14 @@ utils.simpleaf_index(step, ref_type, arguments, output) local splici_args = { gtf : "genes.gtf", fasta : "genome.fa", - rlen : 91, + rlen : 91 }; local splici = utils.splici(splici_args); local arguments = { active : true, - "--use-piscem" : true, + "--use-piscem" : true }; local simpleaf_index = utils.simpleaf_index( @@ -173,7 +173,6 @@ utils.map_type(o, simpleaf_index = {}) - ``o``: an object with - a ``type`` field, and - an object field with the name specified by the ``type`` field. Other fields will be ignored. - For example, ``{"type": "map_reads", "map_reads": {"reads1": null, "reads2": null}, "field_being_ignored": "ignore me"}``. - ``simpleaf_index``: An empty object if in `existing_mappings` mode, or the output object of the `simpleaf_index` function if in `map_reads` mode. The default value is an empty object. @@ -203,7 +202,7 @@ This function has two modes (map types), triggered by the `type` field in the in local map_reads_args = { reads1 : "reads1.fastq", - reads2 : "reads2.fastq", + reads2 : "reads2.fastq" }; local map_type = utils.map_type({ @@ -238,7 +237,6 @@ utils.cell_filt_type(o) - ``o``: an object with - a ``type`` field, and - an argument field with the name specified by the ``type`` field. Other fields will be ignored. - For example, ``{"type": "explicit_pl", "explicit_pl": "whitelist.txt", "field_being_ignored": "ignore me"}`` **Output**: From 99b95994957cfb7f3ab850abf275fb96ca7317c7 Mon Sep 17 00:00:00 2001 From: DongzeHe Date: Thu, 26 Oct 2023 20:59:08 -0400 Subject: [PATCH 11/13] make doc prettierrrr --- docs/source/workflow-utility-library.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/workflow-utility-library.rst b/docs/source/workflow-utility-library.rst index 4a62438..c1bad0f 100644 --- a/docs/source/workflow-utility-library.rst +++ b/docs/source/workflow-utility-library.rst @@ -78,7 +78,7 @@ This function has four modes (reference types), triggered by the ``type`` field local ref_type = utils.ref_type({ type : "splici", splici : splici_args - }) + }); local splici = utils.splici(splici_args); @@ -267,7 +267,7 @@ This function has five modes (cell filtering types), triggered by the `type` fie local cell_filt_type = utils.cell_filt_type({ type : "unfiltered_pl", unfiltered_pl : unfiltered_pl_args - }) + }); local unfiltered_pl = utils.unfiltered_pl(unfiltered_pl_args); From aae666e8d0cdb2751fe32cb877317b69aae9dfd3 Mon Sep 17 00:00:00 2001 From: Robert Patro Date: Sat, 28 Oct 2023 11:27:02 -0400 Subject: [PATCH 12/13] fix: bumped roers dep --- Cargo.lock | 1029 +++++++++++++++++++--------------------------------- Cargo.toml | 2 +- 2 files changed, 369 insertions(+), 662 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0bd37e..f6feb43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -19,25 +10,32 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -79,15 +77,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ "utf8parse", ] @@ -113,9 +111,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "argminmax" @@ -142,46 +140,6 @@ dependencies = [ "serde", ] -[[package]] -name = "arrow2" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ae0428d69ab31d7b2adad22a752d6f11fef2e901d2262d0cad4f5cb08b7093" -dependencies = [ - "ahash", - "arrow-format", - "bytemuck", - "chrono", - "dyn-clone", - "either", - "ethnum", - "foreign_vec", - "futures", - "getrandom", - "hash_hasher", - "lexical-core", - "lz4", - "multiversion", - "num-traits", - "regex", - "regex-syntax 0.6.29", - "rustc_version", - "simdutf8", - "strength_reduce", - "zstd", -] - -[[package]] -name = "async-trait" -version = "0.1.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "atoi" version = "2.0.0" @@ -197,35 +155,11 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "backtrace" -version = "0.3.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - [[package]] name = "base64" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" - -[[package]] -name = "bedrs" -version = "0.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e88965de14b05118d0b695991fa0e9867717be1b04e119790cbfb6c4ac1c163" -dependencies = [ - "anyhow", -] +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bit-vec" @@ -278,47 +212,47 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.4" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad152d03a2c813c80bb94fedbf3a3f02b28f793e39e7c214c8a0bcc196343de7" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "bytemuck" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdde5c9cd29ebd706ce1b35600920a33550e402fc998a2e53ad3b42c3c47a192" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2" @@ -373,9 +307,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", @@ -383,9 +317,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -396,21 +330,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmd_lib" @@ -447,9 +381,9 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "comfy-table" -version = "6.2.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ "crossterm", "strum", @@ -465,9 +399,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -515,6 +449,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -526,17 +470,14 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", "crossterm_winapi", "libc", - "mio", - "parking_lot 0.12.1", - "signal-hook", - "signal-hook-mio", + "parking_lot", "winapi", ] @@ -613,9 +554,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "either" @@ -632,7 +573,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] @@ -650,9 +591,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -666,9 +607,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.3.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0198b9d0078e0f30dedc7acbb21c974e838fc8fae3ee170128658a98cb2c1c04" +checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" [[package]] name = "faccess" @@ -689,18 +630,15 @@ checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "libz-sys", @@ -725,95 +663,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "fxhash" version = "0.2.1" @@ -846,12 +695,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gimli" -version = "0.27.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" - [[package]] name = "glob" version = "0.3.1" @@ -860,14 +703,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "grangers" -version = "0.2.1" -source = "git+https://github.com/COMBINE-lab/grangers.git?branch=dev#1d780e1911f1a456b4f9d4117628ce16cb05f8a9" +version = "0.4.0" +source = "git+https://github.com/COMBINE-lab/grangers.git?branch=dev#eb66072cc70ef95dfd53d48943e63c5e0fa446ba" dependencies = [ "anyhow", - "bedrs", "clap", "flate2", + "lazy_static", "noodles", + "nutype", "peak_alloc", "polars", "rust-lapper", @@ -875,12 +719,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "hash_hasher" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" - [[package]] name = "hashbrown" version = "0.12.3" @@ -894,14 +732,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ "ahash", - "rayon", ] [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +dependencies = [ + "ahash", + "allocator-api2", + "rayon", +] [[package]] name = "heck" @@ -911,9 +753,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "home" @@ -932,16 +774,16 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -965,32 +807,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", + "hashbrown 0.14.2", ] [[package]] @@ -1000,7 +822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.19", + "rustix", "windows-sys", ] @@ -1015,15 +837,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -1070,7 +892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c11fb98940a7f8b419619e98ccbf2e094671a5fdd0e277f05acd373071186d57" dependencies = [ "jrsonnet-gcmodule-derive", - "parking_lot 0.12.1", + "parking_lot", ] [[package]] @@ -1256,15 +1078,15 @@ checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libm" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "pkg-config", @@ -1277,12 +1099,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - [[package]] name = "linux-raw-sys" version = "0.4.10" @@ -1291,9 +1107,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1301,9 +1117,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lz4" @@ -1359,15 +1175,15 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" dependencies = [ "libc", ] @@ -1399,23 +1215,11 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys", -] - [[package]] name = "multiversion" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cda45dade5144c2c929bf2ed6c24bebbba784e9198df049ec87d722b9462bd1" +checksum = "b2c7b9d7fe61760ce5ea19532ead98541f6b4c495d87247aff9826445cf6872a" dependencies = [ "multiversion-macros", "target-features", @@ -1423,9 +1227,9 @@ dependencies = [ [[package]] name = "multiversion-macros" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04bffdccbd4798b61dce08c97ce8c66a68976f95541aaf284a6e90c1d1c306e1" +checksum = "26a83d8500ed06d68877e9de1dde76c1dbb83885dcdbda4ef44ccbc3fbda2ac8" dependencies = [ "proc-macro2", "quote", @@ -1449,23 +1253,22 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.7.1", "pin-utils", - "static_assertions", ] [[package]] name = "noodles" -version = "0.44.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57213e65425cd2be8923d39ad3cd13c0847949dc9ed3e1bef73f2733d84c0570" +checksum = "9ba860dd272b95bc6758e9cf1ec9d13d159c928726b020432a88bd897ca75bbb" dependencies = [ "noodles-core", "noodles-fasta", @@ -1475,9 +1278,9 @@ dependencies = [ [[package]] name = "noodles-bgzf" -version = "0.22.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2985a96e0306878a47e223f1bc8b5c988de6ef1516797796e05349dac0a727b2" +checksum = "7d578e5a173cbfac77295db4188c959966ce24a3364e009d363170d1ed44066a" dependencies = [ "byteorder", "bytes", @@ -1493,22 +1296,22 @@ checksum = "94fbe3192fe33acacabaedd387657f39b0fc606f1996d546db0dfe14703b843a" [[package]] name = "noodles-csi" -version = "0.22.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55329e145d9b5ba58299e3f9e36512d143e19379f13c3480db7346bcfaadc679" +checksum = "e0531175d5473e6057c1724c1242b19bfc42dba644fe275b4df89c5b8d31a782" dependencies = [ "bit-vec", "byteorder", - "indexmap 2.0.0", + "indexmap 2.0.2", "noodles-bgzf", "noodles-core", ] [[package]] name = "noodles-fasta" -version = "0.25.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e096096e1c04ab76fe034ea499c692d8a4401a321155b542cd8ea3b1f7d08d12" +checksum = "310dcfb61e8e2cafb65d9da4b329a98a390f2b570c17599a7f4639328cfb3e2c" dependencies = [ "bytes", "memchr", @@ -1518,10 +1321,11 @@ dependencies = [ [[package]] name = "noodles-gff" -version = "0.16.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112d886335b986e5716de8c02291565813fd3be65053e3f5774b95399434af4f" +checksum = "2a70052d45f0bf58198617f282a6564c2c22602aa9bfff293a30b90060148ce6" dependencies = [ + "indexmap 2.0.2", "noodles-bgzf", "noodles-core", "noodles-csi", @@ -1530,9 +1334,9 @@ dependencies = [ [[package]] name = "noodles-gtf" -version = "0.14.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096367bb382b244da2dc087a9628daecae299a5641f6572f28627f2001187465" +checksum = "fc089e0d2a80c447213126e85d52b4408ed7a737f64e4a01521336962a6f67aa" dependencies = [ "noodles-bgzf", "noodles-core", @@ -1569,31 +1373,32 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", "libm", ] [[package]] -name = "num_cpus" -version = "1.16.0" +name = "nutype" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "0ae1c26994c543093dea62dc235aba699a8ea1ba5bcab09449a757449a643a91" dependencies = [ - "hermit-abi", - "libc", + "nutype_macros", ] [[package]] -name = "object" -version = "0.31.1" +name = "nutype_macros" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "1f23758ee2f73e58d93bad0e422ed9cbb9a9609797c2ee3b27f8c1537b38a2ca" dependencies = [ - "memchr", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1640,17 +1445,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -1658,32 +1452,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.8", + "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.6" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", - "instant", "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", "windows-targets", ] @@ -1702,9 +1482,9 @@ checksum = "84d3978b0aa7d46c34452384c28264ac859c652b67635f6acfd598e1b6608de5" [[package]] name = "peg" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07f2cafdc3babeebc087e499118343442b742cc7c31b4d054682cc598508554" +checksum = "400bcab7d219c38abf8bd7cc2054eb9bbbd4312d66f6a5557d572a203f646f61" dependencies = [ "peg-macros", "peg-runtime", @@ -1712,9 +1492,9 @@ dependencies = [ [[package]] name = "peg-macros" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a90084dc05cf0428428e3d12399f39faad19b0909f64fb9170c9fdd6d9cd49b" +checksum = "46e61cce859b76d19090f62da50a9fe92bab7c2a5f09e183763559a2ac392c90" dependencies = [ "peg-runtime", "proc-macro2", @@ -1723,9 +1503,9 @@ dependencies = [ [[package]] name = "peg-runtime" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa00462b37ead6d11a82c9d568b26682d78e0477dc02d1966c013af80969739" +checksum = "36bae92c60fa2398ce4678b98b2c4b5a7c61099961ca1fa305aec04a9ad28922" [[package]] name = "percent-encoding" @@ -1735,19 +1515,20 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.0" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" dependencies = [ + "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.0" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" dependencies = [ "pest", "pest_generator", @@ -1755,22 +1536,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.0" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] name = "pest_meta" -version = "2.7.0" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" dependencies = [ "once_cell", "pest", @@ -1807,7 +1588,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] @@ -1821,9 +1602,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1848,9 +1629,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c464be75f4ec8b1a8527a55583f67961e3514f112a99f789476a7ccb9a397d10" +checksum = "40db657cc67a8dd9fe4b40db5b73027f5f224623545597e1930cbbb9c05b1de5" dependencies = [ "getrandom", "polars-core", @@ -1864,32 +1645,46 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f187a7fd8fbef04d6a810bfd4c181baf584fc4439ec054e0caad4325bf00f95" +checksum = "d1e50c63db77f846ac5119477422f0156f0a1826ceaae7d921f9a6d5ea5f7ca3" dependencies = [ - "arrow2", - "hashbrown 0.13.2", + "ahash", + "arrow-format", + "bytemuck", + "chrono", + "dyn-clone", + "either", + "ethnum", + "foreign_vec", + "getrandom", + "hashbrown 0.14.2", + "lexical-core", + "lz4", "multiversion", "num-traits", "polars-error", - "thiserror", + "rustc_version", + "simdutf8", + "streaming-iterator", + "strength_reduce", + "zstd", ] [[package]] name = "polars-core" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b7216aa3336fd2a7b5ebfa66748f3770682b28cd682930d1abf59b53b670e0" +checksum = "cdfb622b8ca81b4614c64d95e7590d6e0571d7d398b5ad595c1abc4412abe714" dependencies = [ "ahash", - "arrow2", - "bitflags 1.3.2", + "bitflags 2.4.1", + "bytemuck", "chrono", "comfy-table", "either", - "hashbrown 0.13.2", - "indexmap 1.9.3", + "hashbrown 0.14.2", + "indexmap 2.0.2", "num-traits", "once_cell", "polars-arrow", @@ -1902,41 +1697,40 @@ dependencies = [ "regex", "smartstring", "thiserror", - "wasm-timer", + "version_check", "xxhash-rust", ] [[package]] name = "polars-error" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb4f020b2b6d12408cca6b081dbd928fef8bbe9cdbba4c88441e61ade33bc2d" +checksum = "4b6480520ebde0b20935b600483b865513891e36c04942cebdd19e4f338257b4" dependencies = [ - "arrow2", + "arrow-format", "regex", + "simdutf8", "thiserror", ] [[package]] name = "polars-io" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "270650704e08bef37d227a6904b36c4bdad2d013536ea3c57f40007c19ad2ebc" +checksum = "666466a3b151047c76d99b4e4e5f5438895ef97848008cf49b06df8e3d2d395a" dependencies = [ "ahash", - "arrow2", - "async-trait", "bytes", "chrono", "fast-float", - "futures", "home", + "itoa", "lexical", - "lexical-core", "memchr", "memmap2", "num-traits", "once_cell", + "percent-encoding", "polars-arrow", "polars-core", "polars-error", @@ -1944,18 +1738,19 @@ dependencies = [ "polars-utils", "rayon", "regex", + "ryu", "simdutf8", - "tokio", + "smartstring", ] [[package]] name = "polars-lazy" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a8fe41263496b5212098f19d0cdddc11f75c71957a09d2ce300a8fdb4407e3" +checksum = "07e1c2da1ca20106f80d9510090344e7311fd1dcfd6e6b65031e10606c0958c7" dependencies = [ "ahash", - "bitflags 1.3.2", + "bitflags 2.4.1", "glob", "once_cell", "polars-arrow", @@ -1968,32 +1763,43 @@ dependencies = [ "polars-utils", "rayon", "smartstring", + "version_check", ] [[package]] name = "polars-ops" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d63c31565b88f31457abafb224166fdc0824bf2954414f0c5c52b31c09189942" +checksum = "0fe2d37a6a3ef358499d43aecee80740e62dd44e6cfe7a9c4aa0b8db88de8292" dependencies = [ + "ahash", "argminmax", - "arrow2", + "bytemuck", "either", + "hashbrown 0.14.2", + "indexmap 2.0.2", "memchr", + "num-traits", "polars-arrow", "polars-core", + "polars-error", "polars-utils", + "rayon", + "regex", "smartstring", + "version_check", ] [[package]] name = "polars-pipe" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51076665e7e45f536a2f080763db040e38eab9fdb1cd927b93428720ba23c46a" +checksum = "f6aa050d529be01617f54bc60658149da76f97dbea9fdac3c9d60b811f64a2ba" dependencies = [ + "crossbeam-channel", + "crossbeam-queue", "enum_dispatch", - "hashbrown 0.13.2", + "hashbrown 0.14.2", "num-traits", "polars-arrow", "polars-core", @@ -2004,17 +1810,19 @@ dependencies = [ "polars-utils", "rayon", "smartstring", + "version_check", ] [[package]] name = "polars-plan" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7d0f48bdd7fa9474f718ecb99fc12d97671933707091a249bb0da96b30f291" +checksum = "c47e5d62d8f612aab61a6331d04c5c95c9ff301106d8b91131c8833b4ef3def6" dependencies = [ "ahash", - "arrow2", + "bytemuck", "once_cell", + "percent-encoding", "polars-arrow", "polars-core", "polars-io", @@ -2024,29 +1832,33 @@ dependencies = [ "rayon", "regex", "smartstring", + "strum_macros", + "version_check", ] [[package]] name = "polars-row" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b18fa58195b88fdd32c98561cbeff4024b106cdd2a8c975e7d06544324d0e4d1" +checksum = "f05d6544f7d6065fcaa93bc69aac0532ce09aab4f81ec03c9a78dd901bb0c05b" dependencies = [ - "arrow2", + "polars-arrow", "polars-error", "polars-utils", ] [[package]] name = "polars-sql" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8485d0c9664e67123b065c8a7c946a0165b0192c05c24cdcde447b1c9e05c7e5" +checksum = "77f65f9c8bfe7f0b2c08c38c79b92ec4ddaf213fc424d94a6272ed7b2d83987f" dependencies = [ "polars-arrow", "polars-core", + "polars-error", "polars-lazy", "polars-plan", + "rand", "serde", "serde_json", "sqlparser", @@ -2054,17 +1866,17 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c0d11ef8a93b2d204a60da7099510c40fe1ac04d95022ae4daa39df31b345e1" +checksum = "3763af36aeeb85ef083f11c43bc28c5b6222e2aae039c5118d916bc855f2b5b9" dependencies = [ - "arrow2", "atoi", "chrono", "now", "once_cell", "polars-arrow", "polars-core", + "polars-error", "polars-ops", "polars-utils", "regex", @@ -2073,16 +1885,20 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.30.0" +version = "0.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145a59f928f8317fcf543407ef1b83e827448462e05e2fc6444162fcec84386a" +checksum = "55d2c038ff67e4eb6019682c3f66d83f744e285de9c28e816109a61bace824cd" dependencies = [ "ahash", - "hashbrown 0.13.2", + "bytemuck", + "hashbrown 0.14.2", + "num-traits", "once_cell", + "polars-error", "rayon", "smartstring", "sysinfo", + "version_check", ] [[package]] @@ -2187,9 +2003,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -2197,44 +2013,33 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", ] [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.2", - "regex-syntax 0.7.3", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2248,13 +2053,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.3", + "regex-syntax 0.8.2", ] [[package]] @@ -2265,14 +2070,14 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "roers" -version = "0.2.0" -source = "git+https://github.com/COMBINE-lab/roers?branch=main#c7d21fccb05d3ed5173a7a34665a4fbbbeb28a23" +version = "0.3.0" +source = "git+https://github.com/COMBINE-lab/roers?branch=main#a1cb54e79c2cb2c0af1d2c9ba92e1e3b3b6f63cd" dependencies = [ "anyhow", "clap", @@ -2298,12 +2103,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -2321,42 +2120,28 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys", -] - -[[package]] -name = "rustix" -version = "0.38.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags 2.4.1", "errno", "libc", - "linux-raw-sys 0.4.10", + "linux-raw-sys", "windows-sys", ] [[package]] name = "rustversion" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "safemem" @@ -2366,9 +2151,9 @@ checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" @@ -2405,22 +2190,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] @@ -2459,9 +2244,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2480,43 +2265,13 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.4" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ "lazy_static", ] -[[package]] -name = "signal-hook" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - [[package]] name = "simdutf8" version = "0.1.4" @@ -2552,24 +2307,15 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - -[[package]] -name = "slab" -version = "0.4.8" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "smartstring" @@ -2582,21 +2328,11 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "sqlparser" -version = "0.34.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" +checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" dependencies = [ "log", ] @@ -2607,6 +2343,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strength_reduce" version = "0.2.4" @@ -2643,21 +2385,21 @@ dependencies = [ [[package]] name = "strum" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] @@ -2673,9 +2415,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -2684,9 +2426,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.29.4" +version = "0.29.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "751e810399bba86e9326f5762b7f32ac5a085542df78da6a78d94e07d14d7c11" +checksum = "0a18d114d420ada3a891e6bc8e96a2023402203296a47cdd65083377dad18ba5" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2728,21 +2470,20 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-features" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06f6b473c37f9add4cf1df5b4d66a8ef58ab6c895f1a3b3f949cf3e21230140e" +checksum = "cfb5fa503293557c5158bd215fdc225695e567a77e453f5d4452a50a193969bd" [[package]] name = "tempfile" -version = "3.6.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ - "autocfg", "cfg-if", "fastrand", - "redox_syscall 0.3.5", - "rustix 0.37.23", + "redox_syscall", + "rustix", "windows-sys", ] @@ -2761,28 +2502,28 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "rustix 0.38.19", + "rustix", "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] @@ -2830,21 +2571,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tokio" -version = "1.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" -dependencies = [ - "autocfg", - "backtrace", - "libc", - "mio", - "pin-project-lite", - "socket2", - "windows-sys", -] - [[package]] name = "tracing" version = "0.1.40" @@ -2864,7 +2590,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", ] [[package]] @@ -2879,12 +2605,12 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" dependencies = [ - "lazy_static", "log", + "once_cell", "tracing-core", ] @@ -2908,9 +2634,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" @@ -2981,22 +2707,10 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.87" @@ -3015,7 +2729,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3026,31 +2740,6 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" -[[package]] -name = "wasm-timer" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" -dependencies = [ - "futures", - "js-sys", - "parking_lot 0.11.2", - "pin-utils", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "which" version = "5.0.0" @@ -3060,7 +2749,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.19", + "rustix", "windows-sys", ] @@ -3096,10 +2785,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] @@ -3115,9 +2804,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -3130,45 +2819,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "wyz" @@ -3181,9 +2870,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" +checksum = "9828b178da53440fa9c766a3d2f73f7cf5d0ac1fe3980c1e5018d899fd19e07b" [[package]] name = "xz2" @@ -3212,32 +2901,50 @@ dependencies = [ "winapi", ] +[[package]] +name = "zerocopy" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7d7c7970ca2215b8c1ccf4d4f354c4733201dfaaba72d44ae5b37472e4901" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b27b1bb92570f989aac0ab7e9cbfbacdd65973f7ee920d9f0e71ebac878fd0b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 3ae70c7..3e364b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ categories = ["command-line-utilities", "science"] [dependencies] seq_geom_parser = { git = "https://github.com/COMBINE-lab/seq_geom_parser", branch = "dev", version = "0.3.0" } seq_geom_xform = { git = "https://github.com/COMBINE-lab/seq_geom_xform", branch = "dev", version = "0.4.0" } -roers = { git = "https://github.com/COMBINE-lab/roers", branch = "main", version = "0.2.0" } +roers = { git = "https://github.com/COMBINE-lab/roers", branch = "main", version = "0.3.0" } anyhow = "^1.0" clap = { version = "4.4.6", features = ["derive", "cargo", "deprecated", "wrap_help", "help", "usage", "error-context"] } cmd_lib = "^1.6.0" From a80523b3214820ffde25d7d035b36154ed3ef265 Mon Sep 17 00:00:00 2001 From: Rob Patro Date: Sat, 28 Oct 2023 21:36:09 -0400 Subject: [PATCH 13/13] fix: bump other deps --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3e364b5..7632626 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,12 +36,12 @@ seq_geom_parser = { git = "https://github.com/COMBINE-lab/seq_geom_parser", bran seq_geom_xform = { git = "https://github.com/COMBINE-lab/seq_geom_xform", branch = "dev", version = "0.4.0" } roers = { git = "https://github.com/COMBINE-lab/roers", branch = "main", version = "0.3.0" } anyhow = "^1.0" -clap = { version = "4.4.6", features = ["derive", "cargo", "deprecated", "wrap_help", "help", "usage", "error-context"] } +clap = { version = "4.4.7", features = ["derive", "cargo", "deprecated", "wrap_help", "help", "usage", "error-context"] } cmd_lib = "^1.6.0" tracing = "0.1.40" tracing-subscriber = { version = "0.3.17", default-features = true, features = ["env-filter"] } semver = "^1.0.20" -serde = {version = "1.0.189", features = ["derive"]} +serde = {version = "1.0.190", features = ["derive"]} serde_json = "1.0.107" time = {version = "^0.3.30", features = ["macros", "formatting", "parsing", "serde", "serde-human-readable"]} which = "^5.0.0"