Merge pull request #130 from PixelgenTechnologies/feature/tech-70-fix…

…-qc-aggregate-fraction-count

Feature/tech 70 fix qc aggregate fraction count

CHANGELOG.md

@@ -46,6 +46,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Renaming of component metrics in adata
 * Use MPX graph compatible permutation strategy when calculating Moran's I.
 * Marker filtering is now done after count transformation in polarization score calculation.
+* Use the input read count at the annotate stage for the `fraction_antibody_reads_in_outliers` metric denominator
+  instead of the total raw input reads.
 * Use common analysis engine to orchestrate running different "per component" analysis, like
   polarization and colocalization analysis (yielding a roughly 3x speed-up over previous approach).
 
@@ -61,6 +63,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   by switching to a server/client-based logging system.
 * Fix a bug in the amplicon stage where read suffixes were not correctly recognised.
 * Ensure deterministic results from `pmds_layout` (given a set seed).
+* Fix an issue with the `fraction_antibody_reads_usable_per_cell` metric where the denominator
+  read count was not correctly averaged with the cell count.
 
 ### Removed
 

pyproject.toml

@@ -43,12 +43,13 @@ scanpy = "*"
 lxml = "*"
 cssselect = "*"
 typing_extensions = "*"
-scipy = "*"
+scipy = "<1.13.0"
 pyarrow = ">=14,<16"
 semver = "^3.0.0"
 ruamel-yaml = "^0.17.21"
 pydantic = "~2.0.0"
-polars = "^0.20"
+# Pin to 0.20.19 to fix a bug in lazy collect projection_pushdown optimzation
+polars = "0.20.19"
 importlib-resources = "^5.12.0"
 fsspec = "^2023.12.2"
 fastparquet = "^2023.8.0"

src/pixelator/report/models/demux.py

@@ -51,6 +51,14 @@ def per_antibody_read_count_fractions(self) -> dict[str, float]:  # noqa: D102
     def unrecognised_antibody_read_count(self) -> int:  # noqa: D102
         return self.input_read_count - self.output_read_count
 
+    @pydantic.computed_field(  # type: ignore
+        return_type=int,
+        description="Fraction of reads without a recognized antibody barcode.",
+    )
+    @property
+    def fraction_unrecognised_antibody_reads(self) -> float:  # noqa: D102
+        return self.unrecognised_antibody_read_count / self.input_read_count
+
     @classmethod
     def from_json(cls, p: Path) -> DemuxSampleReport:
         """Initialize an :class:`DemuxSampleReport` from a cutadapt report file."""

src/pixelator/report/qcreport/main.py

@@ -51,18 +51,20 @@ def create_dynamic_report(
 
     # Collect antibody metrics
     antibodies_data_values = {
-        "antibody_reads": demux_metrics.output_read_count,
+        "antibody_reads": reads_flow.valid_antibody_read_count,
         "antibody_reads_usable_per_cell": annotate_metrics.read_count_per_cell_stats.mean,
         "antibody_reads_in_outliers": annotate_metrics.reads_in_aggregates_count,
         "unrecognized_antibodies": demux_metrics.unrecognised_antibody_read_count,
     }
 
-    antibodies_data_fractions = (
-        pd.Series(antibodies_data_values) / reads_flow.input_read_count
-    ).to_dict()
-
+    total_reads_per_cell = reads_flow.input_read_count / annotate_metrics.cell_count
     antibodies_data_fractions = {
-        f"fraction_{k}": v for k, v in antibodies_data_fractions.items()
+        "fraction_antibody_reads": reads_flow.fraction_valid_antibody_reads,
+        "fraction_antibody_reads_usable_per_cell": (
+            annotate_metrics.read_count_per_cell_stats.mean / total_reads_per_cell
+        ),
+        "fraction_antibody_reads_in_outliers": annotate_metrics.fraction_reads_in_aggregates,
+        "fraction_unrecognized_antibodies": demux_metrics.fraction_unrecognised_antibody_reads,
     }
 
     placeholder_cell_predictions = {

tests/pixeldataset/test_precomputed_layouts.py

@@ -17,7 +17,7 @@
 from tests.utils import dna_seqs
 
 
-def layout_df() -> pl.DataFrame:
+def layout_df() -> pl.LazyFrame:
     nbr_of_rows = 300
     components = [
         "PXLCMP0000000",
@@ -84,7 +84,7 @@ def layout_df_generator() -> Iterable[pl.LazyFrame]:
 @pytest.fixture(name="precomputed_layouts", params=["layout_df", "layout_df_generator"])
 def precomputed_layouts_fixture(request) -> PreComputedLayouts:
     if request.param == "layout_df":
-        return PreComputedLayouts(layout_df())
+        return PreComputedLayouts(layout_df())  # type: ignore
     if request.param == "layout_df_generator":
         return PreComputedLayouts(layout_df_generator())
     raise Exception("We should never get here!")

tests/report/snapshots/test_demux/test_collapse_metrics_lookup/pbmcs_unstimulated/pbmcs_unstimulated_demux_metrics.json

@@ -166,5 +166,6 @@
         "mIgG2a": 0.001540503389107456,
         "mIgG2b": 0.00047400104280229414
     },
-    "unrecognised_antibody_read_count": 2321
+    "unrecognised_antibody_read_count": 2321,
+    "fraction_unrecognised_antibody_reads": 0.013565404419715132
 }

tests/report/snapshots/test_demux/test_collapse_metrics_lookup/uropod_control/uropod_control_demux_metrics.json

@@ -166,5 +166,6 @@
         "mIgG2a": 0.00048114010322642217,
         "mIgG2b": 0.00041751827139482916
     },
-    "unrecognised_antibody_read_count": 3788
+    "unrecognised_antibody_read_count": 3788,
+    "fraction_unrecognised_antibody_reads": 0.01483895735562572
 }

tests/report/snapshots/test_demux/test_demux_summary/demux_summary.csv

@@ -1,3 +1,3 @@
-sample_id,input_read_count,output_read_count,unrecognised_antibody_read_count,read_counts_ACTB,read_counts_B2M,read_counts_CD102,read_counts_CD11a,read_counts_CD11b,read_counts_CD11c,read_counts_CD127,read_counts_CD137,read_counts_CD14,read_counts_CD150,read_counts_CD152,read_counts_CD154,read_counts_CD158,read_counts_CD16,read_counts_CD161,read_counts_CD162,read_counts_CD163,read_counts_CD18,read_counts_CD19,read_counts_CD197,read_counts_CD1d,read_counts_CD2,read_counts_CD20,read_counts_CD200,read_counts_CD22,read_counts_CD229,read_counts_CD244,read_counts_CD25,read_counts_CD26,read_counts_CD268,read_counts_CD27,read_counts_CD274,read_counts_CD278,read_counts_CD279,read_counts_CD29,read_counts_CD314,read_counts_CD32,read_counts_CD328,read_counts_CD33,read_counts_CD337,read_counts_CD35,read_counts_CD36,read_counts_CD37,read_counts_CD38,read_counts_CD3E,read_counts_CD4,read_counts_CD40,read_counts_CD41,read_counts_CD43,read_counts_CD44,read_counts_CD45,read_counts_CD45RA,read_counts_CD45RB,read_counts_CD47,read_counts_CD48,read_counts_CD49D,read_counts_CD5,read_counts_CD50,read_counts_CD52,read_counts_CD53,read_counts_CD54,read_counts_CD55,read_counts_CD59,read_counts_CD62P,read_counts_CD64,read_counts_CD69,read_counts_CD7,read_counts_CD71,read_counts_CD72,read_counts_CD8,read_counts_CD82,read_counts_CD84,read_counts_CD86,read_counts_CD9,read_counts_HLA-ABC,read_counts_HLA-DR,read_counts_TCRVb5,read_counts_mIgG1,read_counts_mIgG2a,read_counts_mIgG2b,read_count_fraction_ACTB,read_count_fraction_B2M,read_count_fraction_CD102,read_count_fraction_CD11a,read_count_fraction_CD11b,read_count_fraction_CD11c,read_count_fraction_CD127,read_count_fraction_CD137,read_count_fraction_CD14,read_count_fraction_CD150,read_count_fraction_CD152,read_count_fraction_CD154,read_count_fraction_CD158,read_count_fraction_CD16,read_count_fraction_CD161,read_count_fraction_CD162,read_count_fraction_CD163,read_count_fraction_CD18,read_count_fraction_CD19,read_count_fraction_CD197,read_count_fraction_CD1d,read_count_fraction_CD2,read_count_fraction_CD20,read_count_fraction_CD200,read_count_fraction_CD22,read_count_fraction_CD229,read_count_fraction_CD244,read_count_fraction_CD25,read_count_fraction_CD26,read_count_fraction_CD268,read_count_fraction_CD27,read_count_fraction_CD274,read_count_fraction_CD278,read_count_fraction_CD279,read_count_fraction_CD29,read_count_fraction_CD314,read_count_fraction_CD32,read_count_fraction_CD328,read_count_fraction_CD33,read_count_fraction_CD337,read_count_fraction_CD35,read_count_fraction_CD36,read_count_fraction_CD37,read_count_fraction_CD38,read_count_fraction_CD3E,read_count_fraction_CD4,read_count_fraction_CD40,read_count_fraction_CD41,read_count_fraction_CD43,read_count_fraction_CD44,read_count_fraction_CD45,read_count_fraction_CD45RA,read_count_fraction_CD45RB,read_count_fraction_CD47,read_count_fraction_CD48,read_count_fraction_CD49D,read_count_fraction_CD5,read_count_fraction_CD50,read_count_fraction_CD52,read_count_fraction_CD53,read_count_fraction_CD54,read_count_fraction_CD55,read_count_fraction_CD59,read_count_fraction_CD62P,read_count_fraction_CD64,read_count_fraction_CD69,read_count_fraction_CD7,read_count_fraction_CD71,read_count_fraction_CD72,read_count_fraction_CD8,read_count_fraction_CD82,read_count_fraction_CD84,read_count_fraction_CD86,read_count_fraction_CD9,read_count_fraction_HLA-ABC,read_count_fraction_HLA-DR,read_count_fraction_TCRVb5,read_count_fraction_mIgG1,read_count_fraction_mIgG2a,read_count_fraction_mIgG2b
-pbmcs_unstimulated,171097,168776,2321,46,20357,262,5583,26,255,553,22,49,58,157,20,14,262,85,175,39,2649,80,872,118,3043,385,36,768,325,70,123,1948,210,1108,314,19,63,1244,65,1091,218,68,33,1801,1857,1477,1466,1248,4788,301,3254,582,3836,18730,1383,1471,1938,639,496,1127,239,750,116,2429,470,2964,115,69,263,1912,74,723,1555,1721,256,488,53,60699,3881,387,65,260,80,0.00027255059961131913,0.12061549035407877,0.0015523534151775135,0.0330793477745651,0.0001540503389107456,0.0015108783239323126,0.0032765322083708583,0.0001303502867706309,0.0002903256387164052,0.00034365075603166325,0.0009302270464995023,0.00011850026070057354,8.295018249040148e-05,0.0015523534151775135,0.0005036261079774375,0.0010368772811300184,0.0002310755083661184,0.015695359529790967,0.00047400104280229414,0.005166611366545007,0.0006991515381333838,0.018029814665592264,0.0022811300184860405,0.00021330046926103238,0.004550410010902024,0.00192562923638432,0.00041475091245200737,0.0007287766033085273,0.011541925392235863,0.001244252737356022,0.006564914442811774,0.0018604540929990046,0.00011257524766554486,0.00037327582120680666,0.007370716215575674,0.000385125847276864,0.006464189221216287,0.0012916528416362517,0.00040290088638195,0.00019552543015594634,0.010670948476086647,0.011002749206048254,0.008751244252737355,0.00868606910935204,0.007394416267715789,0.028368962411717307,0.0017834289235436319,0.019279992415983317,0.00344835758638669,0.022728350002370005,0.11097549414608712,0.008194293027444661,0.008715694174527184,0.011482675261885577,0.003786083329383325,0.002938806465374224,0.006677489690477319,0.0014160781153718539,0.004443759776271507,0.0006873015120633265,0.014391856662084656,0.0027847561264634784,0.017561738635824997,0.0006813764990282979,0.0004088258994169787,0.001558278428212542,0.011328624922974831,0.0004384509645921221,0.004283784424325733,0.009213395269469593,0.010196947433284352,0.0015168033369673413,0.0028914063610939946,0.0003140256908565199,0.35964236621320567,0.022994975588946295,0.002292980044556098,0.000385125847276864,0.001540503389107456,0.00047400104280229414
-uropod_control,255274,251486,3788,77,35500,923,6620,47,99,618,23,21,44,62,30,33,617,47,1013,30,9293,53,1215,5,7695,127,46,180,521,110,296,9481,87,1101,551,814,99,3285,104,153,53,25,37,49,21,738,8778,6089,2764,568,40,1379,14384,27160,524,2995,1001,823,1743,1939,380,639,97,2600,258,1796,30,13,446,4405,207,92,17232,3736,323,1237,50,56939,8231,339,80,121,105,0.00030618006568954134,0.141160939376347,0.0036701844237850218,0.026323532920321607,0.00018688913100530448,0.0003936600844579818,0.00245739325449528,9.145638325791495e-05,8.350365427896582e-05,0.00017496003753688077,0.0002465345983474229,0.0001192909346842369,0.00013122002815266058,0.0024534168900058055,0.00018688913100530448,0.004028057227837733,0.0001192909346842369,0.03695235520068712,0.00021074731794215186,0.004831282854711594,1.9881822447372815e-05,0.030598124746506764,0.0005049982901632695,0.0001829127665158299,0.0007157456081054213,0.0020716858990162475,0.000437400093842202,0.0011770038888844706,0.03769991172470833,0.00034594371058428697,0.004377977302911494,0.0021909768337004844,0.0032367606944322946,0.0003936600844579818,0.01306235734792394,0.0004135419069053546,0.0006083837668896081,0.00021074731794215186,9.940911223686409e-05,0.00014712548611055883,0.0001948418599842536,8.350365427896582e-05,0.0029345569932322274,0.034904527488607714,0.024212083376410617,0.010990671448907693,0.0022585750300215517,0.00015905457957898252,0.0054834066309854225,0.057196026816602116,0.10799805953412914,0.002083614992484671,0.011909211645976317,0.003980340853964038,0.0032725479748375655,0.006930803305154163,0.007710170745091178,0.0015110185060003341,0.002540896908774246,0.00038570735547903266,0.010338547672633864,0.0010259020382844372,0.007141550623096316,0.0001192909346842369,5.1692738363169325e-05,0.0017734585623056553,0.01751588557613545,0.0008231074493212346,0.0003658255330316598,0.06852071288262568,0.014855697732676969,0.0012843657301002838,0.004918762873480035,0.00019881822447372817,0.22641021766619215,0.03272945611286513,0.0013479875619318768,0.00031810915915796503,0.00048114010322642217,0.00041751827139482916
+sample_id,input_read_count,output_read_count,unrecognised_antibody_read_count,fraction_unrecognised_antibody_reads,read_counts_ACTB,read_counts_B2M,read_counts_CD102,read_counts_CD11a,read_counts_CD11b,read_counts_CD11c,read_counts_CD127,read_counts_CD137,read_counts_CD14,read_counts_CD150,read_counts_CD152,read_counts_CD154,read_counts_CD158,read_counts_CD16,read_counts_CD161,read_counts_CD162,read_counts_CD163,read_counts_CD18,read_counts_CD19,read_counts_CD197,read_counts_CD1d,read_counts_CD2,read_counts_CD20,read_counts_CD200,read_counts_CD22,read_counts_CD229,read_counts_CD244,read_counts_CD25,read_counts_CD26,read_counts_CD268,read_counts_CD27,read_counts_CD274,read_counts_CD278,read_counts_CD279,read_counts_CD29,read_counts_CD314,read_counts_CD32,read_counts_CD328,read_counts_CD33,read_counts_CD337,read_counts_CD35,read_counts_CD36,read_counts_CD37,read_counts_CD38,read_counts_CD3E,read_counts_CD4,read_counts_CD40,read_counts_CD41,read_counts_CD43,read_counts_CD44,read_counts_CD45,read_counts_CD45RA,read_counts_CD45RB,read_counts_CD47,read_counts_CD48,read_counts_CD49D,read_counts_CD5,read_counts_CD50,read_counts_CD52,read_counts_CD53,read_counts_CD54,read_counts_CD55,read_counts_CD59,read_counts_CD62P,read_counts_CD64,read_counts_CD69,read_counts_CD7,read_counts_CD71,read_counts_CD72,read_counts_CD8,read_counts_CD82,read_counts_CD84,read_counts_CD86,read_counts_CD9,read_counts_HLA-ABC,read_counts_HLA-DR,read_counts_TCRVb5,read_counts_mIgG1,read_counts_mIgG2a,read_counts_mIgG2b,read_count_fraction_ACTB,read_count_fraction_B2M,read_count_fraction_CD102,read_count_fraction_CD11a,read_count_fraction_CD11b,read_count_fraction_CD11c,read_count_fraction_CD127,read_count_fraction_CD137,read_count_fraction_CD14,read_count_fraction_CD150,read_count_fraction_CD152,read_count_fraction_CD154,read_count_fraction_CD158,read_count_fraction_CD16,read_count_fraction_CD161,read_count_fraction_CD162,read_count_fraction_CD163,read_count_fraction_CD18,read_count_fraction_CD19,read_count_fraction_CD197,read_count_fraction_CD1d,read_count_fraction_CD2,read_count_fraction_CD20,read_count_fraction_CD200,read_count_fraction_CD22,read_count_fraction_CD229,read_count_fraction_CD244,read_count_fraction_CD25,read_count_fraction_CD26,read_count_fraction_CD268,read_count_fraction_CD27,read_count_fraction_CD274,read_count_fraction_CD278,read_count_fraction_CD279,read_count_fraction_CD29,read_count_fraction_CD314,read_count_fraction_CD32,read_count_fraction_CD328,read_count_fraction_CD33,read_count_fraction_CD337,read_count_fraction_CD35,read_count_fraction_CD36,read_count_fraction_CD37,read_count_fraction_CD38,read_count_fraction_CD3E,read_count_fraction_CD4,read_count_fraction_CD40,read_count_fraction_CD41,read_count_fraction_CD43,read_count_fraction_CD44,read_count_fraction_CD45,read_count_fraction_CD45RA,read_count_fraction_CD45RB,read_count_fraction_CD47,read_count_fraction_CD48,read_count_fraction_CD49D,read_count_fraction_CD5,read_count_fraction_CD50,read_count_fraction_CD52,read_count_fraction_CD53,read_count_fraction_CD54,read_count_fraction_CD55,read_count_fraction_CD59,read_count_fraction_CD62P,read_count_fraction_CD64,read_count_fraction_CD69,read_count_fraction_CD7,read_count_fraction_CD71,read_count_fraction_CD72,read_count_fraction_CD8,read_count_fraction_CD82,read_count_fraction_CD84,read_count_fraction_CD86,read_count_fraction_CD9,read_count_fraction_HLA-ABC,read_count_fraction_HLA-DR,read_count_fraction_TCRVb5,read_count_fraction_mIgG1,read_count_fraction_mIgG2a,read_count_fraction_mIgG2b
+pbmcs_unstimulated,171097,168776,2321,0.013565404419715132,46,20357,262,5583,26,255,553,22,49,58,157,20,14,262,85,175,39,2649,80,872,118,3043,385,36,768,325,70,123,1948,210,1108,314,19,63,1244,65,1091,218,68,33,1801,1857,1477,1466,1248,4788,301,3254,582,3836,18730,1383,1471,1938,639,496,1127,239,750,116,2429,470,2964,115,69,263,1912,74,723,1555,1721,256,488,53,60699,3881,387,65,260,80,0.00027255059961131913,0.12061549035407877,0.0015523534151775135,0.0330793477745651,0.0001540503389107456,0.0015108783239323126,0.0032765322083708583,0.0001303502867706309,0.0002903256387164052,0.00034365075603166325,0.0009302270464995023,0.00011850026070057354,8.295018249040148e-05,0.0015523534151775135,0.0005036261079774375,0.0010368772811300184,0.0002310755083661184,0.015695359529790967,0.00047400104280229414,0.005166611366545007,0.0006991515381333838,0.018029814665592264,0.0022811300184860405,0.00021330046926103238,0.004550410010902024,0.00192562923638432,0.00041475091245200737,0.0007287766033085273,0.011541925392235863,0.001244252737356022,0.006564914442811774,0.0018604540929990046,0.00011257524766554486,0.00037327582120680666,0.007370716215575674,0.000385125847276864,0.006464189221216287,0.0012916528416362517,0.00040290088638195,0.00019552543015594634,0.010670948476086647,0.011002749206048254,0.008751244252737355,0.00868606910935204,0.007394416267715789,0.028368962411717307,0.0017834289235436319,0.019279992415983317,0.00344835758638669,0.022728350002370005,0.11097549414608712,0.008194293027444661,0.008715694174527184,0.011482675261885577,0.003786083329383325,0.002938806465374224,0.006677489690477319,0.0014160781153718539,0.004443759776271507,0.0006873015120633265,0.014391856662084656,0.0027847561264634784,0.017561738635824997,0.0006813764990282979,0.0004088258994169787,0.001558278428212542,0.011328624922974831,0.0004384509645921221,0.004283784424325733,0.009213395269469593,0.010196947433284352,0.0015168033369673413,0.0028914063610939946,0.0003140256908565199,0.35964236621320567,0.022994975588946295,0.002292980044556098,0.000385125847276864,0.001540503389107456,0.00047400104280229414
+uropod_control,255274,251486,3788,0.01483895735562572,77,35500,923,6620,47,99,618,23,21,44,62,30,33,617,47,1013,30,9293,53,1215,5,7695,127,46,180,521,110,296,9481,87,1101,551,814,99,3285,104,153,53,25,37,49,21,738,8778,6089,2764,568,40,1379,14384,27160,524,2995,1001,823,1743,1939,380,639,97,2600,258,1796,30,13,446,4405,207,92,17232,3736,323,1237,50,56939,8231,339,80,121,105,0.00030618006568954134,0.141160939376347,0.0036701844237850218,0.026323532920321607,0.00018688913100530448,0.0003936600844579818,0.00245739325449528,9.145638325791495e-05,8.350365427896582e-05,0.00017496003753688077,0.0002465345983474229,0.0001192909346842369,0.00013122002815266058,0.0024534168900058055,0.00018688913100530448,0.004028057227837733,0.0001192909346842369,0.03695235520068712,0.00021074731794215186,0.004831282854711594,1.9881822447372815e-05,0.030598124746506764,0.0005049982901632695,0.0001829127665158299,0.0007157456081054213,0.0020716858990162475,0.000437400093842202,0.0011770038888844706,0.03769991172470833,0.00034594371058428697,0.004377977302911494,0.0021909768337004844,0.0032367606944322946,0.0003936600844579818,0.01306235734792394,0.0004135419069053546,0.0006083837668896081,0.00021074731794215186,9.940911223686409e-05,0.00014712548611055883,0.0001948418599842536,8.350365427896582e-05,0.0029345569932322274,0.034904527488607714,0.024212083376410617,0.010990671448907693,0.0022585750300215517,0.00015905457957898252,0.0054834066309854225,0.057196026816602116,0.10799805953412914,0.002083614992484671,0.011909211645976317,0.003980340853964038,0.0032725479748375655,0.006930803305154163,0.007710170745091178,0.0015110185060003341,0.002540896908774246,0.00038570735547903266,0.010338547672633864,0.0010259020382844372,0.007141550623096316,0.0001192909346842369,5.1692738363169325e-05,0.0017734585623056553,0.01751588557613545,0.0008231074493212346,0.0003658255330316598,0.06852071288262568,0.014855697732676969,0.0012843657301002838,0.004918762873480035,0.00019881822447372817,0.22641021766619215,0.03272945611286513,0.0013479875619318768,0.00031810915915796503,0.00048114010322642217,0.00041751827139482916