Repository navigation
Expand file tree
/
Copy pathprocess_ms_spectra.py
More file actions
391 lines (317 loc) · 11.5 KB
/
Copy pathprocess_ms_spectra.py
File metadata and controls
391 lines (317 loc) · 11.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
'''Process local raw MS spectra into JSONL or JSON peak-list files.'''
from __future__ import annotations
import argparse
import json
import os
import re
import zipfile
from collections import defaultdict
from pathlib import Path
from typing import Any, Literal
from tqdm import tqdm
from common import (
ensure_parent,
load_webbook_index,
require_data_terms_acknowledgement,
split_id_argument,
)
DEFAULT_INPUT = 'local-data/raw/spectra/nist_MS.zip'
DEFAULT_OUTPUT = 'local-data/processed/nist_ms.jsonl'
MEMBER_RE = re.compile(
r'^(?P<compound_id>[^_]+)_(?P<spec_type>[^_]+)_(?P<spec_idx>[^_]+)\.jdx$'
)
PEAK_RE = re.compile(r'(?<![\w.])(\d+)\s*,\s*(\d+)(?![\w.])')
def _index_sort_key(value: str) -> tuple[int, str]:
'''Return a stable sort key for spectrum indexes.'''
try:
return (int(value), value)
except ValueError:
return (10**9, value)
def jdx_text_to_spectra(text: str) -> tuple[list[int], list[int]]:
'''Extract an MS peak list from a JDX text block.
Args:
text: Raw JDX file text.
Returns:
Two lists: m/z values and relative intensities, sorted by m/z.
Raises:
ValueError: If no integer ``m/z,intensity`` pairs are found.
'''
pairs = [(int(mz), int(intensity)) for mz, intensity in PEAK_RE.findall(text)]
if not pairs:
raise ValueError('no integer m/z,intensity peak pairs found')
pairs = sorted(pairs)
mz_values, intensities = zip(*pairs)
return list(mz_values), list(intensities)
def infer_output_format(
path_out: str | Path,
output_format: str | None = None,
) -> Literal['jsonl', 'json']:
'''Infer or validate the processed MS output format.
Args:
path_out: Output path.
output_format: Optional explicit output format.
Returns:
Either ``'jsonl'`` or ``'json'``.
Raises:
ValueError: If the explicit output format is unsupported.
'''
if output_format is not None:
if output_format not in {'jsonl', 'json'}:
raise ValueError(f'Unsupported output format: {output_format}')
return output_format
suffix = Path(path_out).suffix.lower()
if suffix == '.json':
return 'json'
return 'jsonl'
def write_ms_records(
records: list[dict[str, Any]],
path_out: str | Path,
output_format: Literal['jsonl', 'json'],
) -> None:
'''Write processed MS records to JSONL or JSON.
Args:
records: Processed spectrum records.
path_out: Output path.
output_format: Output format.
'''
ensure_parent(path_out)
with Path(path_out).open('w', encoding='utf-8') as out_file:
if output_format == 'jsonl':
for record in records:
line = json.dumps(
record,
ensure_ascii=False,
separators=(',', ':'),
)
out_file.write(line + '\n')
else:
json.dump(records, out_file, ensure_ascii=False, indent=0)
def parse_ms_member_name(member_name: str) -> tuple[str, str] | None:
'''Parse an MS JDX archive member name.
Args:
member_name: ZIP member name.
Returns:
``(compound_id, spectrum_index)`` for MS JDX files, otherwise ``None``.
'''
basename = os.path.basename(member_name)
match = MEMBER_RE.match(basename)
if match is None:
return None
if match.group('spec_type').upper() != 'MS':
return None
return match.group('compound_id'), match.group('spec_idx')
def load_compound_metadata(
index_path: str | Path | None = None,
) -> dict[str, dict[str, Any]]:
'''Load compound names and InChI strings from the NistChemPy index.
Args:
index_path: Optional NistChemPy local index directory or CSV path.
Returns:
Mapping from WebBook compound ID to a metadata dictionary.
'''
df = load_webbook_index(index_path)
required = ['ID', 'name', 'inchi']
missing = [column for column in required if column not in df.columns]
if missing:
raise ValueError(f'NistChemPy index is missing required columns: {missing}')
metadata = {}
for _, row in df[required].iterrows():
compound_id = str(row['ID'])
metadata[compound_id] = {
'name': '' if row['name'] != row['name'] else row['name'],
'inchi': '' if row['inchi'] != row['inchi'] else row['inchi'],
}
return metadata
def collect_ms_members(
zipf: zipfile.ZipFile,
ids: list[str] | None = None,
spectrum_policy: str = 'first',
limit: int | None = None,
) -> list[tuple[str, str, str]]:
'''Collect archive members to process.
Args:
zipf: Open ZIP archive.
ids: Optional ordered compound-ID filter.
spectrum_policy: ``first`` to process one spectrum per compound, or
``all`` to process every MS JDX member.
limit: Optional maximum number of selected members or compounds.
Returns:
Tuples of ``(compound_id, spectrum_index, member_name)``.
'''
id_order = None
if ids is not None:
id_order = {compound_id: idx for idx, compound_id in enumerate(ids)}
grouped: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
for info in zipf.infolist():
if info.is_dir():
continue
parsed = parse_ms_member_name(info.filename)
if parsed is None:
continue
compound_id, spec_idx = parsed
if id_order is not None and compound_id not in id_order:
continue
grouped[compound_id].append((compound_id, spec_idx, info.filename))
if id_order is not None:
compound_ids = sorted(grouped, key=lambda item: id_order[item])
else:
compound_ids = sorted(grouped)
selected = []
for compound_id in compound_ids:
members = sorted(
grouped[compound_id],
key=lambda item: _index_sort_key(item[1]),
)
if spectrum_policy == 'first':
selected.append(members[0])
else:
selected.extend(members)
if limit is not None and len(selected) >= limit:
selected = selected[:limit]
break
return selected
def process_ms_spectra(
path_zip: str | Path,
path_out: str | Path,
spectrum_policy: str = 'first',
ids: list[str] | None = None,
limit: int | None = None,
output_format: str | None = None,
index_path: str | Path | None = None,
) -> None:
'''Process a local raw MS archive and write peak-list records.
Args:
path_zip: ZIP archive containing raw MS JDX files.
path_out: Output JSONL or JSON path.
spectrum_policy: ``first`` for one spectrum per compound, or ``all`` for
all MS spectra in the archive.
ids: Optional ordered compound-ID filter.
limit: Optional maximum number of selected members to process.
output_format: Optional explicit output format. If omitted, ``.json``
outputs a JSON array and all other suffixes use JSON Lines.
index_path: Optional NistChemPy local index directory or CSV path.
Raises:
ValueError: If ``spectrum_policy`` or ``output_format`` is unsupported.
'''
if spectrum_policy not in {'first', 'all'}:
raise ValueError(f'Unsupported spectrum policy: {spectrum_policy}')
resolved_output_format = infer_output_format(path_out, output_format)
metadata = load_compound_metadata(index_path)
data = []
with zipfile.ZipFile(path_zip, 'r') as zipf:
members = collect_ms_members(
zipf,
ids=ids,
spectrum_policy=spectrum_policy,
limit=limit,
)
for compound_id, spec_idx, member_name in tqdm(members, total=len(members)):
try:
text = zipf.read(member_name).decode('utf-8', errors='replace')
mz_values, intensities = jdx_text_to_spectra(text)
compound_metadata = metadata.get(compound_id, {'name': '', 'inchi': ''})
if compound_id not in metadata:
tqdm.write(
f'Metadata not found for {compound_id}; using empty fields'
)
item = {
'ID': compound_id,
'name': compound_metadata['name'],
'inchi': compound_metadata['inchi'],
'mz': mz_values,
'intensities': intensities,
}
if spectrum_policy == 'all':
item['spectrum_index'] = spec_idx
item['source_member'] = member_name
data.append(item)
except Exception as exc:
raise RuntimeError(
f'Error while processing {member_name}: {exc}'
) from exc
write_ms_records(data, path_out, resolved_output_format)
def get_arguments() -> argparse.Namespace:
'''Parse command-line arguments.
Returns:
Parsed CLI arguments.
'''
parser = argparse.ArgumentParser(
description='Process local raw MS JDX archive into JSONL or JSON peak lists.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'path_zip',
nargs='?',
default=DEFAULT_INPUT,
help='input ZIP archive containing raw MS JDX files',
)
parser.add_argument(
'path_out',
nargs='?',
default=DEFAULT_OUTPUT,
help='output JSONL or JSON file',
)
parser.add_argument(
'--format',
dest='output_format',
choices=['jsonl', 'json'],
help='output format; inferred from output suffix when omitted',
)
parser.add_argument(
'--spectrum-policy',
choices=['first', 'all'],
default='first',
help='process the first MS spectrum per compound or all MS spectra',
)
parser.add_argument(
'--ids',
help='comma-separated compound IDs to process instead of all available IDs',
)
parser.add_argument(
'--index-path',
help='NistChemPy local index directory or CSV path',
)
parser.add_argument(
'--limit',
type=int,
help='maximum number of selected members to process',
)
parser.add_argument(
'--accept-data-terms',
action='store_true',
help='acknowledge that generated files are local artifacts',
)
return parser.parse_args()
def check_arguments(args: argparse.Namespace) -> None:
'''Validate command-line arguments.
Args:
args: Parsed CLI arguments.
Raises:
ValueError: If an argument is invalid.
'''
if not Path(args.path_zip).exists():
raise ValueError(f'Input ZIP archive does not exist: {args.path_zip}')
if args.limit is not None and args.limit <= 0:
raise ValueError(f'--limit must be positive: {args.limit}')
def main() -> None:
'''Process local raw MS spectra into JSONL or JSON peak lists.'''
args = get_arguments()
check_arguments(args)
require_data_terms_acknowledgement(args.accept_data_terms)
ids = split_id_argument(args.ids)
output_format = infer_output_format(args.path_out, args.output_format)
print('\nProcessing MS spectra ...')
print(f'Input archive: {args.path_zip}')
print(f'Output {output_format.upper()}: {args.path_out}')
process_ms_spectra(
args.path_zip,
args.path_out,
spectrum_policy=args.spectrum_policy,
ids=ids,
limit=args.limit,
output_format=args.output_format,
index_path=args.index_path,
)
print()
if __name__ == '__main__':
main()