From 6c6b780c53f183f4383e768bc8f8d107cb38f135 Mon Sep 17 00:00:00 2001 From: RalfG Date: Wed, 8 Jul 2026 14:34:00 +0200 Subject: [PATCH 1/2] Add reader for DIA-NN report.parquet format DIA-NN 2.0+ writes its main report as report.parquet instead of a .tsv file, with a different schema (no CScore or MS2.Scan, but adds Decoy and Precursor.Mz). --- README.rst | 1 + psm_utils/io/__init__.py | 6 +++ psm_utils/io/diann.py | 77 ++++++++++++++++++++++++++++- tests/test_data/test_diann.parquet | Bin 0 -> 11755 bytes tests/test_io/test_diann.py | 42 +++++++++++++++- 5 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 tests/test_data/test_diann.parquet diff --git a/README.rst b/README.rst index 2949b22..c954146 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,7 @@ Supported file formats `AlphaDIA precursors TSV `_ ``alphadia`` ✅ ❌ `CBOR `_ ``cbor`` ✅ ✅ Requires the optional ``cbor2`` dependency (``pip install psm-utils[cbor]``) `DIA-NN TSV `_ ``diann`` ✅ ❌ + `DIA-NN report.parquet `_ ``diann_parquet`` ✅ ❌ `FlashLFQ generic TSV `_ ``flashlfq`` ✅ ✅ `FragPipe PSM TSV `_ ``fragpipe`` ✅ ❌ `ionbot CSV `_ ``ionbot`` ✅ ❌ diff --git a/psm_utils/io/__init__.py b/psm_utils/io/__init__.py index bb39378..f2fae8a 100644 --- a/psm_utils/io/__init__.py +++ b/psm_utils/io/__init__.py @@ -182,6 +182,12 @@ class FileType(TypedDict): "extension": ".tsv", "filename_pattern": r"^.*(?:_|\.)?diann\.tsv$", }, + "diann_parquet": { + "reader": diann.DIANNParquetReader, + "writer": None, + "extension": ".parquet", + "filename_pattern": r"^.*(?:_|\.)?report\.parquet$", + }, "parquet": { # List after more specific Parquet patterns to avoid matching conflicts "reader": parquet.ParquetReader, "writer": parquet.ParquetWriter, diff --git a/psm_utils/io/diann.py b/psm_utils/io/diann.py index 2d46a5b..c71102b 100644 --- a/psm_utils/io/diann.py +++ b/psm_utils/io/diann.py @@ -2,14 +2,20 @@ Reader for PSM files from DIA-NN. Reads the '.tsv' file as defined on the -`DIA-NN documentation page `_. +`DIA-NN documentation page `_, +or the ``report.parquet`` main report written by DIA-NN 2.0 and later. Notes ----- - DIA-NN calculates q-values at both the run and library level. The run-level q-value is used as the PSM q-value. -- DIA-NN currently does not return precursor m/z values. - DIA-NN currently does not support C-terminal modifications in its searches. +- The classic ``.tsv`` main report does not contain precursor m/z values or decoy status; these + are set to ``None``/``False``, respectively. The ``report.parquet`` main report does contain + both, and they are used when available. +- The ``report.parquet`` main report does not contain a PSM-level score (no ``CScore`` column) + or an MS2 scan number (no ``MS2.Scan`` column). The score is set to ``None`` and + ``Precursor.Id`` is used as the spectrum identifier instead. """ @@ -22,6 +28,7 @@ from typing import Any, cast import pandas as pd +import pyarrow.parquet as pq # type: ignore[import] from psm_utils.io._base_classes import ReaderBase from psm_utils.io._utils import set_csv_field_size_limit @@ -130,3 +137,69 @@ def from_dataframe(cls, dataframe: pd.DataFrame) -> PSMList: """Create a PSMList from a DIA-NN Pandas DataFrame.""" records = cast(list[dict[str, str]], dataframe.to_dict(orient="records")) return PSMList(psm_list=[cls._get_peptide_spectrum_match(entry) for entry in records]) + + +class DIANNParquetReader(ReaderBase): + """Reader for the DIA-NN 2.0+ ``report.parquet`` main report.""" + + def __init__(self, filename: str | Path, *args: Any, **kwargs: Any) -> None: + """ + Reader for DIA-NN ``report.parquet`` file. + + Parameters + ---------- + filename : str or Path + Path to PSM file. + *args + Additional positional arguments passed to the base class. + **kwargs + Additional keyword arguments passed to the base class. + + """ + super().__init__(filename, *args, **kwargs) + + def __iter__(self) -> Iterator[PSM]: + """Iterate over file and return PSMs one-by-one.""" + with pq.ParquetFile(self.filename) as pq_file: + for batch in pq_file.iter_batches(): + for row in batch.to_pylist(): + yield self._get_peptide_spectrum_match(row, self.filename) + + @staticmethod + def _get_peptide_spectrum_match( + psm_dict: dict[str, Any], filename: str | Path | None = None + ) -> PSM: + """Parse a single PSM from a DIA-NN ``report.parquet`` file.""" + rescoring_features: dict[str, Any] = {} + for ft in RESCORING_FEATURES: + try: + rescoring_features[ft] = psm_dict[ft] + except KeyError: + continue + + return PSM( + peptidoform=DIANNTSVReader._parse_peptidoform( + psm_dict["Modified.Sequence"], str(psm_dict["Precursor.Charge"]) + ), + spectrum_id=psm_dict["Precursor.Id"], + run=psm_dict["Run"], + is_decoy=bool(psm_dict["Decoy"]), + qvalue=psm_dict["Q.Value"], + pep=float(psm_dict["PEP"]), + score=None, # Not returned by DIA-NN in the `report.parquet` main report :( + precursor_mz=float(psm_dict["Precursor.Mz"]), + retention_time=float(psm_dict["RT"]), + ion_mobility=float(psm_dict["IM"]), + protein_list=psm_dict["Protein.Ids"].split(";"), + source="diann", + rank=None, + provenance_data=({"diann_filename": str(filename)} if filename else {}), + rescoring_features=rescoring_features, + metadata={}, + ) + + @classmethod + def from_dataframe(cls, dataframe: pd.DataFrame) -> PSMList: + """Create a PSMList from a DIA-NN ``report.parquet`` Pandas DataFrame.""" + records = cast(list[dict[str, Any]], dataframe.to_dict(orient="records")) + return PSMList(psm_list=[cls._get_peptide_spectrum_match(entry) for entry in records]) diff --git a/tests/test_data/test_diann.parquet b/tests/test_data/test_diann.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1b7241feee8749b89b4946c155e19da7518d5014 GIT binary patch literal 11755 zcmeHNOKcm*8K!ARVH7!#9h0S6B^8u->!6OrhiF?ylVNPr$%AcvsnAwYp3D17Mu&+c%S zyGxTzBF)qdO&$>vVTKey7cO-s!pwo#&l1&Y8t4wo40RJG;lL&Yc64CzkB+X+L$!ZUYi& zXO@!q+TG_saVgi3Qq9VB?+Ywkk$5`)YIyV2NH`hU-pc30kyv)kg3fbO40BJ@$%10i zxk)K9cb2E3Xu)uKGDG(ieJuFRJ1R*;4=*DFw}Wc5mdkd@LOKaSncoV5YTORm zLIm2@q>~Wr_Z~ZcM}GYkvkg?^cF-0g(6%PEK)B#^Q!mXvKLeyT{nxyKD@F`R?HNVm zR=+Lg8u7UB{SRIm+|OFDr5d+`wva7tYtl&wudgP5HT%gc76?@1cF-0g(6%O>gz(V^ zZ~y1pf3mWHYTORmLIm2@q>~Wt|0Vyg)qh*rKs9a$Z6N|}Ytl&wG|!9?)$(rGx`ha| zzb5@Z0{TSr$7l1>#Xr9B>z};**KeBRKs9a$Z6N|}Ytm`raOvY0STh3hM~z|wwx|uG z(n$#a{PZ8Ue)b7#VFT5;9khiAw5>@eAw2lS+1Wea{gDL%)wmtBg$T5*NzDig4=x03 z?wO9TC)PP9V{^`3w7XAddyR{>K_=*={TII8e;CMNYYW zJhPJL%Hf=8Z!s5b%ry1L#b8aDEo0(rAPIcN{cGnN=t~cxq&p19$O|)l>LD8xSa#btT)u^w1;&PTk{(s(;CO|=|IY6Ey`8yrn)#_62~1PqOj`w+NkuJ5 znbU?a`>oyej-7e;G3Na<$ZU$rX3FZv%qu*?>x>#AQ^FEj_fEOqJ;l8DICJ;$MXeCh zTDa8Ag%T_^6YLB62vJaFIp~mXKf(OX0Z6zp;ia_@Ns;NL6Om?B^K`k8I?@5&pJjeI z3xMXD2m#H~iGVca^-G!Hy?M%Y_Y`w?j`{FOjIg_5DdwH00LNS`A&yx(HI8mO8Ug&xJoCeO05sQ52xyj04XEGj zMi4(e$GmqA5Y4p|BATUBBf=)E^W?p!ncGhTj47MPmszIN7+FnP#71y#Ihp$xP69_4 zu@S&`UCbTVNdW32HiGzLH}n3NP6E;N@YBWb4-3rC7XZ{;h+&$TrBkQL@X@LRd-E&I z-@mL{I9ih-7^cvubZRjA;MQUNd69W*5wOhF8Dg2GQ)6LSpu@TQEc4MbfMc%F5XUTi zD30s)vo1U1aJ}_ac;>igT3n;Tb=)7Gzqu?mD&pae(3E@iMrZl@(zm{Id1+a>-61;B zASJgqmm6GNgzn2w2rk12DDmXVjzOrv>~se$(zVx+gl0v=?u}l(H5e+I{Jto3QC=6j zT!rg$Fmt0P%dcHt(vY-TFpeyD|L#8*{M5VSYZ!OAE-1yf6gS&ZkquW znE^*<$!H*Tiiep9b&DBE6&^Db1Z#K(Gh-UHCkAe5v4(y!w#QRuixt^1+bLNgslsE1 z(tKuxY1Ez=xTVD!`pMXCJnoyLYt`eJeL!C>ScBth2h$t*-4XW2drG@%CY=f z(U(5pH}YM+k&$1$pvb$C~Ar=Mf=uy$14G6Iu;IdMs5NMTr>ou4MF?yAP zw=d?`Jz_B^BX+#LE^+nzekDZo-EuLz2XlBzh#L!d`PkulGK6*o_V|J_XKo`Qm;CvG z;9DOgYjQ0vp>5?>xo}u3`X=;i1oy!9J;AHv(8%|=Li(T#oY6+Uk?#r{@@{2A?yL46 zlli~zR-QOUV}sSv1@ifMg3^>`LJ zDfcSjl1GV^SFXgOFh0A7`3kww;WuEtY>cdrjDR^@OdRq4cyGJd9LLMDpH+Oz^}~Zw zap)5Zj^z&Ti)ucdCMO|gygw^*g|*&RA$v3~7h!!JRATuSF9j0ijTEuPisOl_ZCS|? zt9>S~wIi|Pm3As8^vHizYiJwIn_SHZsZz|!!T@UMjV{#BAooKo zi&?1EPzJS2OqPUtbO1FH*7tp}5FJ2%`juFgWvJ^~yl;)v@TvV&FBMwr#~b9Fp!u_o zHMo4hpIXS_h3vb9L z`9NJX7|pREadmff74AZ8Dtv8q^%^uIKoTyXf;ae~TmpOrhq=e5+UrXHID6HAw?PJI zcwh+3q^%dC-qgl#Z2Cv|;=xbPmyl4#8}|^>Gj(uP8D94R5OUtMNARMd(q!SeKwtA|BP@8&WE)&ZqJvNc#9=(gcmQ70$ut zz-E$f)u2bUpZed(pSTWd9d@G)gD~IXY{S6cl&=`YD2!o4(vWXzoAQk+`HKAXd^>Q9 zeAl5~lzVti7W$!U$nTl~&zNsoe23?|ve7Jet87!{6J+-gURjE|co9?7 zvl^(w5w$-!4Z8oO28hVtY^hP z_@YGbpCEcInC5-i7M?YrR^2Waquz2MuwTZmWc+G|@(X5hxB{ZH50jW3ttL}?JtMwvz fVYQ^bkXlk+Rl!e=;741Z?Vs%|13z$upLzcmIk6v! literal 0 HcmV?d00001 diff --git a/tests/test_io/test_diann.py b/tests/test_io/test_diann.py index 29884b8..c78d2bb 100644 --- a/tests/test_io/test_diann.py +++ b/tests/test_io/test_diann.py @@ -1,6 +1,6 @@ """Tests for psm_utils.io.diann.""" -from psm_utils.io.diann import DIANNTSVReader +from psm_utils.io.diann import DIANNParquetReader, DIANNTSVReader from psm_utils.psm import PSM test_psm = PSM( @@ -52,3 +52,43 @@ def test__parse_peptidoform(self): reader = DIANNTSVReader("./tests/test_data/test_diann.tsv") for (peptide, charge), expected in test_cases: assert reader._parse_peptidoform(peptide, charge) == expected + + +test_parquet_psm = PSM( + peptidoform="[UNIMOD:1]-AAAAEINVKDPKEDLETSVVDEGR/4", + spectrum_id="(UniMod:1)AAAAEINVKDPKEDLETSVVDEGR4", + run="LFQ_Orbitrap_AIF_Yeast_03", + collection=None, + spectrum=None, + is_decoy=False, + score=None, + qvalue=0.000548193, + pep=0.0104343, + precursor_mz=621.5723, + retention_time=75.2574, + ion_mobility=0.0, + protein_list=["P38156"], + rank=None, + source="diann", + metadata={}, + rescoring_features={ + "RT": 75.2574, + "Predicted.RT": 75.2713, + "iRT": 33.9222, + "Predicted.iRT": 33.8999, + "Ms1.Profile.Corr": 0.347567, + "Ms1.Area": 849940.0, + "IM": 0.0, + "iIM": 0.0, + "Predicted.IM": 0.0, + "Predicted.iIM": 0.0, + }, +) + + +class TestDIANNParquetReader: + def test_iter(self): + with DIANNParquetReader("./tests/test_data/test_diann.parquet") as reader: + for psm in reader: + psm.provenance_data = {} + assert psm == test_parquet_psm From 1356c358f6c51495be82df14813a53511fd1b858 Mon Sep 17 00:00:00 2001 From: RalfG Date: Tue, 14 Jul 2026 14:35:27 +0200 Subject: [PATCH 2/2] fix: remove superfluous type conversions --- psm_utils/io/diann.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/psm_utils/io/diann.py b/psm_utils/io/diann.py index c71102b..6f24d09 100644 --- a/psm_utils/io/diann.py +++ b/psm_utils/io/diann.py @@ -97,11 +97,11 @@ def _get_peptide_spectrum_match( run=psm_dict["Run"], is_decoy=False, qvalue=psm_dict["Q.Value"], - pep=float(psm_dict["PEP"]), - score=float(psm_dict["CScore"]), + pep=psm_dict["PEP"], + score=psm_dict["CScore"], precursor_mz=None, # Not returned by DIA-NN :( - retention_time=float(psm_dict["RT"]), - ion_mobility=float(psm_dict["IM"]), + retention_time=psm_dict["RT"], + ion_mobility=psm_dict["IM"], protein_list=psm_dict["Protein.Ids"].split(";"), source="diann", rank=None, @@ -185,11 +185,11 @@ def _get_peptide_spectrum_match( run=psm_dict["Run"], is_decoy=bool(psm_dict["Decoy"]), qvalue=psm_dict["Q.Value"], - pep=float(psm_dict["PEP"]), + pep=psm_dict["PEP"], score=None, # Not returned by DIA-NN in the `report.parquet` main report :( - precursor_mz=float(psm_dict["Precursor.Mz"]), - retention_time=float(psm_dict["RT"]), - ion_mobility=float(psm_dict["IM"]), + precursor_mz=psm_dict["Precursor.Mz"], + retention_time=psm_dict["RT"], + ion_mobility=psm_dict["IM"], protein_list=psm_dict["Protein.Ids"].split(";"), source="diann", rank=None,