push stats on demand

BRB/PushButton.py

@@ -8,7 +8,7 @@
 import stat
 from pathlib import Path
 
-def createPath(config, group, project, organism, libraryType, tuples):
+def createPath(config, group, project, organism, libraryType, tuples, stats):
     """Ensures that the output path exists, creates it otherwise, and return where it is"""
     if tuples[0][3]:
         baseDir = "{}/{}/Analysis_{}".format(config.get('Paths', 'baseData'),
@@ -20,10 +20,14 @@ def createPath(config, group, project, organism, libraryType, tuples):
                                                             BRB.misc.getLatestSeqdir(config.get('Paths','groupData'), group),
                                                             config.get('Options', 'runID'),
                                                             BRB.misc.pacifier(project))
-    os.makedirs(baseDir, mode=0o700, exist_ok=True)
+
+    if not stats:
+        os.makedirs(baseDir, mode=0o700, exist_ok=True)
 
     oDir = os.path.join(baseDir, "{}_{}".format(BRB.misc.pacifier(libraryType), organism.split(' ')[0].lower()))
-    os.makedirs(oDir, mode=0o700, exist_ok=True)
+    if not stats:
+        os.makedirs(oDir, mode=0o700, exist_ok=True)
+
     return oDir
 
 
@@ -564,7 +568,7 @@ def scATAC(config, group, project, organism, libraryType, tuples):
     return outputDir, 0, True
 
 
-def GetResults(config, project, libraries):
+def GetResults(config, project, libraries, stats):
     """
     Project is something like '352_Grzes_PearceEd' and libraries is a dictionary with libraries as keys:
         {'18L005489': ['FAT_first_A',
@@ -591,7 +595,7 @@ def GetResults(config, project, libraries):
         )
         log.info(f"Processing {dataPath}")
     except:
-        print("external data")
+        print(f"GetResults with ignore=True, {project} is external data.")
         ignore = True
     validLibraryTypes = {v: i for i, v in enumerate(config.get('Options', 'validLibraryTypes').split(','))}
     pipelines = config.get('Options', 'pipelines').split(',')
@@ -638,7 +642,12 @@ def GetResults(config, project, libraries):
                 reruncount = 0
                 # RELACS needs the unpacified project name to copy the original sample sheet to the dest dir
                 # hence the pacifier is applied on the project in each pipeline separately
-                outputDir, rv, sambaUpdate = globals()[pipeline](config, group, project, organism, libraryType, tuples)
+                if stats:
+                    outputDir, rv, sambaUpdate = (
+                        createPath(config, group, BRB.misc.pacifier(project), organism, libraryType, tuples, stats),
+                        0, False)
+                else:
+                    outputDir, rv, sambaUpdate = globals()[pipeline](config, group, project, organism, libraryType, tuples)
                 if reruncount == 0 and rv != 0:
                     # Allow for one re-run
                     reruncount += 1

BRB/run.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import glob
 import sys
 import os
 import BRB.getConfig
@@ -13,6 +14,36 @@
 from pathlib import Path
 from rich import print
 
+
+def process_data(config, ParkourDict, stats):
+    bdir = "{}/{}".format(config.get('Paths', 'baseData'), config.get('Options', 'runID'))
+    msg = []
+    for k, v in ParkourDict.items():
+        if not os.path.exists("{}/Project_{}".format(bdir, BRB.misc.pacifier(k))):
+            log.info("{}/Project_{} doesn't exist, probably lives on another lane.".format(bdir, BRB.misc.pacifier(k)))
+            continue
+        try:
+            msg = msg + BRB.PushButton.GetResults(config, k, v, stats)
+        except Exception as e:
+            BRB.email.errorEmail(config, sys.exc_info(), "Received an error running PushButton.GetResults() with {} and {}".format(k, v))
+            log.critical("Received an error running PushButton.GetResults() with {} and {}".format(k, v))
+            print("Received an error running PushButton.GetResults() with {} and {}".format(k, v), file=sys.stderr)
+            raise
+
+    # Email finished message
+    log.info('Create e-mail')
+    log.info(msg)
+    BRB.email.finishedEmail(config, msg)
+
+    return
+
+
+def validate_fcid_with_stats(ctx, param, value):
+    if ctx.params.get('stats') and not value:
+        raise click.UsageError('--fcid is required when --stats standalone run is active.')
+    return value
+
+
 @click.command(
     context_settings=dict(
         help_option_names=["-h", "--help"]
@@ -24,49 +55,78 @@
    type=click.Path(exists=True),
    required=False,
    default=os.path.expanduser('~/configs/BigRedButton.ini'),
-   help='specify a custom ini file.',
+   help='Specify a custom ini file.',
    show_default=True
 )
-def run_brb(configfile):
+@click.option(
+    "-s",
+    "--stats",
+    required=False,
+    is_flag=True,
+    help='Standalone run, use only on finished flowcells. Requires --fcid to indicate target.'
+)
+@click.option('--fcid', callback=validate_fcid_with_stats, help='Flowcell ID to push stats.')
+def run_brb(configfile, stats, fcid):
 
     while True:
-        #Read the config file
+        # Read the config file
         config = BRB.getConfig.getConfig(configfile)
+
+        if not stats:
+            # Get the next flow cell to process, or sleep
+            config, ParkourDict = BRB.findFinishedFlowCells.newFlowCell(config)
+            if (config.get('Options','runID') == '') or ParkourDict is None:
+                sleep(60*60)
+                continue
 
-        #Get the next flow cell to process, or sleep
-        config, ParkourDict = BRB.findFinishedFlowCells.newFlowCell(config)
-        if(config.get('Options','runID') == '') or ParkourDict is None:
-            sleep(60*60)
-            continue
+            # Open log file
+            logFile = Path(
+                config['Paths']['logPath'],
+                config.get('Options','runID') + '.log'
+            )
+            print(f"Logging into: {logFile}")
+            setLog(logFile)
 
-        # Open log file
-        logFile = Path(
-            config['Paths']['logPath'],
-            config.get('Options','runID') + '.log'
-        )
-        print(f"Logging into: {logFile}")
-        setLog(logFile)
+        else:
+            # Push stats on-demand
+            d = [d for d in glob.glob("{}/*/fastq.made".format(config.get('Paths', 'baseData'))) if fcid in d]
+            dual_lane = len(d) == 2
+            if len(d) == 0:
+                print(f"ERROR: No fastq.made files found for {fcid}")
+                return  # Exit BRB if no files found.
+            elif len(d) > 2:
+                print(f"ERROR: How many lanes does {fcid} have?!")
+                return  # Exit BRB this error shouldn't happen at all.
+
+            config.set('Options','runID',d[0].split("/")[-2])
+            ParkourDict = BRB.findFinishedFlowCells.queryParkour(config)
+
+            if dual_lane:
+                config1 = BRB.getConfig.getConfig(configfile)
+                config1.set('Options','runID',d[1].split("/")[-2])
+                ParkourDict1 = BRB.findFinishedFlowCells.queryParkour(config)
 
-        #Process each group's data, ignore cases where the project isn't in the lanes being processed
-        bdir = "{}/{}".format(config.get('Paths', 'baseData'), config.get('Options', 'runID'))
-        msg = []
-        for k, v in ParkourDict.items():
-            if not os.path.exists("{}/Project_{}".format(bdir, BRB.misc.pacifier(k))):
-                log.info("{}/Project_{} doesn't exist, probably lives on another lane.".format(bdir, BRB.misc.pacifier(k)))
-                continue
-            try:
-                msg = msg + BRB.PushButton.GetResults(config, k, v)
-            except Exception as e:
-                BRB.email.errorEmail(config, sys.exc_info(), "Received an error running PushButton.GetResults() with {} and {}".format(k, v))
-                log.critical("Received an error running PushButton.GetResults() with {} and {}".format(k, v))
-                print("Received an error running PushButton.GetResults() with {} and {}".format(k, v), file=sys.stderr)
-                raise
+            # Open log file
+            if not dual_lane:
+                logFile = Path(config['Paths']['logPath'], config.get('Options','runID') + '.log')
+            else:
+                logFile = Path(config['Paths']['logPath'], config.get('Options','runID')[:-1] + 'both' + '.log')
+            print(f"Logging into: {logFile}")
+            setLog(logFile)
+            log.info(f"Pushing stats for flowcell: {fcid}")
+
+        # Process each group's data, ignore cases where the project isn't in the lanes being processed
+        process_data(config, ParkourDict, stats)
+
+        if stats and dual_lane:
+            process_data(config1, ParkourDict1, stats)
+
+
+        if not stats:
+            # Mark the flow cell as having been processed
+            BRB.findFinishedFlowCells.markFinished(config)
+            log.info('=== finished flowcell ===')
 
-        #Email finished message
-        log.info('Create e-mail')
-        log.info(msg)
-        BRB.email.finishedEmail(config, msg)
 
-        #Mark the flow cell as having been processed
-        BRB.findFinishedFlowCells.markFinished(config)
-        log.info('=== finished flowcell ===')
+        if stats:
+            return  # don't do anything else.