OSCR

The heartbeat evoked potential and the prediction of functional seizure semiology.

Code ↔ Paper

The paper beside its authors' code: matches between them have not been computed for this paper yet.

Paper

Loaded from Europe PMC by your browser, not stored by OSCR: doi.org · Europe PMC

The paper is loaded when this pane is shown.

The authors' code

Python · 1,795 lines · 66 KB · MIT

  1. #!/usr/bin/env python3
  2. # -*- coding: utf-8 -*-
  3. """
  4. EEG preprocessing with robust handling of non-10–20 aux channels:
  5. - Standardise names (incl. P7→T5, P8→T6; T7→T3, T8→T4)
  6. - Classify aux channels (ROC/LOC→eog, EMG→emg, PHOTIC→stim, IBI/BURSTS/SUPPR→misc, unknown non-10–20→misc)
  7. - Apply montage without dropping channels
  8. - Run PyPREP on EEG-only with a NaN-free trimmed montage
  9. - RANSAC interpolation + reset bads
  10. - Optional ASR; ICA + ICLabel pruning (optionally keep/strip cardiac)
  11. - ECG R-peak events (with gap interpolation)
  12. - Save and TSV log
  13. """
  14. import os
  15. import csv
  16. import gc
  17. import inspect
  18. import logging
  19. import re
  20. from datetime import datetime
  21. from pathlib import Path
  22. from contextlib import redirect_stdout, redirect_stderr
  23. import mne
  24. import numpy as np
  25. import neurokit2 as nk
  26. import pandas as pd
  27. from mne.preprocessing import ICA
  28. try:
  29. import psutil
  30. except Exception: # optional; used only for native-memory breadcrumbs
  31. psutil = None
  32. try:
  33. import asrpy
  34. except Exception: # optional; only required when ASR is enabled
  35. asrpy = None
  36. try:
  37. import pyprep
  38. except Exception: # optional; only required when PyPREP is enabled
  39. pyprep = None
  40. _PYPREP_SAFE_QUANTILE_PATCHED = False
  41. try:
  42. from mne_icalabel import label_components
  43. except Exception: # optional; only required when ICA+ICLabel is enabled
  44. label_components = None
  45. try:
  46. from mne_icalabel.iclabel import iclabel_label_components
  47. except Exception: # older mne-icalabel versions do not expose backend selection
  48. iclabel_label_components = None
  49. # ------------------------- Logging helpers ------------------------- #
  50. def _now_utc_iso() -> str:
  51. return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
  52. def _stage(name: str) -> None:
  53. # The parent process records stdout. This breadcrumb survives ordinary
  54. # exceptions and identifies the native call active immediately before a
  55. # Windows access violation. RSS is more useful than tracemalloc here
  56. # because NumPy, BLAS, ONNX and PyTorch allocate outside Python's heap.
  57. rss = ""
  58. if psutil is not None:
  59. try:
  60. rss = f"\trss_gib={psutil.Process().memory_info().rss / 2**30:.3f}"
  61. except Exception:
  62. pass
  63. print(f"HEPPY_STAGE\t{name}{rss}", flush=True)
  64. logging.info("HEPPY stage: %s%s", name, rss)
  65. # Fixed TSV schema so all runs (success/fail) align.
  66. _PREPROC_TSV_FIELDS = [
  67. "utc", "status", "stage",
  68. "input", "output",
  69. "n_channels", "n_events", "bad_channels",
  70. "remove_cfa", "remove_cfa_mode", "flip_ecg",
  71. "asr_threshold", "n_comp", "stim_keep",
  72. "montage_name", "ica_path", "ica_bads", "ica_bad_labels",
  73. "spirometry_channel", "spirometry_status", "spirometry_peaks",
  74. "spirometry_troughs", "spirometry_qc_png", "spirometry_signals_csv",
  75. "spirometry_summary_csv",
  76. "error_type", "error",
  77. ]
  78. def _append_preproc_tsv(logging_path: str | None, rec: dict) -> None:
  79. """Append a preprocessing record (success or failure) to a TSV with stable columns."""
  80. if not logging_path:
  81. return
  82. try:
  83. logp = Path(logging_path)
  84. if str(logp.parent) not in ("", "."):
  85. logp.parent.mkdir(parents=True, exist_ok=True)
  86. file_exists = logp.exists() and logp.stat().st_size > 0
  87. row = {k: "" for k in _PREPROC_TSV_FIELDS}
  88. for k, v in (rec or {}).items():
  89. if k in row:
  90. row[k] = "" if v is None else str(v)
  91. with open(str(logp), "a", newline="") as f:
  92. w = csv.DictWriter(f, fieldnames=_PREPROC_TSV_FIELDS, delimiter="\t")
  93. if not file_exists:
  94. w.writeheader()
  95. w.writerow(row)
  96. except Exception:
  97. logging.exception("Failed to append preprocessing TSV record to %s", logging_path)
  98. # ------------------------- Naming & typing helpers ------------------------- #
  99. TEN_TWENTY_SET = {
  100. 'Fp1','Fp2','F7','F3','Fz','F4','F8',
  101. 'T3','C3','Cz','C4','T4',
  102. 'T5','P3','Pz','P4','T6',
  103. 'O1','O2','A1','A2'
  104. }
  105. from dataclasses import dataclass
  106. from pathlib import Path
  107. # Light wrapper so we can keep names below unchanged
  108. @dataclass
  109. class _RuntimeCfg:
  110. output_root: Path
  111. target_sfreq: float | None
  112. use_asr: float | None
  113. use_pyprep: bool
  114. remove_cfa: bool
  115. remove_cfa_mode: str
  116. log_file: str | None
  117. ref_chs: str | list
  118. reref_chs: str | list
  119. high_pass: float
  120. low_pass: float
  121. prep_ransac: bool
  122. line_freqs: tuple | list
  123. montage_name: str | None = None
  124. rename_to_1020: bool = True
  125. use_ica: bool = True
  126. spirometry_channel: str | None = None
  127. pyprep_channel_wise: bool = False
  128. pyprep_max_chunk_size: int | None = 8
  129. pyprep_by_segment: bool = False
  130. pyprep_safe_quantile_patch: bool = False
  131. ica_fit_hz: float = 100.0
  132. iclabel_backend: str | None = "auto"
  133. _CFG_RT: _RuntimeCfg | None = None
  134. # fallback defaults in case set_runtime_config is not called
  135. prep_params = {
  136. "line_freqs": (50.0, 100.0),
  137. "ref_chs": "eeg",
  138. "reref_chs": "eeg",
  139. "l_freq": 1.0,
  140. "h_freq": 100.0,
  141. "ransac": True,
  142. }
  143. def set_runtime_config(cfg) -> None:
  144. """Call this once from the CLI or notebook to bind the active config."""
  145. global _CFG_RT, output_dir, target_sfreq, do_asr, do_pyprep, remove_cfa, remove_cfa_mode, log_file, prep_params
  146. # derive ICA cardiac strategy
  147. mode = getattr(cfg, "remove_cfa_mode", None)
  148. legacy_flag = getattr(cfg, "remove_cfa", None)
  149. if mode is None:
  150. # backward compatibility: infer mode from legacy bool
  151. if legacy_flag is None:
  152. mode = "remove"
  153. legacy_flag = True
  154. else:
  155. mode = "remove" if bool(legacy_flag) else "keep"
  156. if legacy_flag is None:
  157. legacy_flag = True if str(mode).lower() != "keep" else False
  158. _CFG_RT = _RuntimeCfg(
  159. output_root=Path(cfg.output_root),
  160. target_sfreq=getattr(cfg, "target_sfreq", None),
  161. use_asr=getattr(cfg, "use_asr", None),
  162. use_pyprep=bool(getattr(cfg, "use_pyprep", getattr(cfg, "run_pyprep", True))),
  163. remove_cfa=bool(legacy_flag),
  164. remove_cfa_mode=str(mode).lower(),
  165. log_file=getattr(cfg, "log_file", None),
  166. ref_chs=getattr(cfg, "ref_chs", "eeg"),
  167. reref_chs=getattr(cfg, "reref_chs", "eeg"),
  168. high_pass=getattr(cfg, "high_pass", 1.0),
  169. low_pass=getattr(cfg, "low_pass", 100.0),
  170. prep_ransac=getattr(cfg, "prep_ransac", True),
  171. line_freqs=getattr(cfg, "line_freqs", (50.0, 100.0)),
  172. montage_name=getattr(cfg, "montage_name", None),
  173. rename_to_1020=getattr(cfg, "rename_to_1020", True),
  174. use_ica=bool(getattr(cfg, "use_ica", getattr(cfg, "run_ica", True))),
  175. spirometry_channel=getattr(cfg, "spirometry_channel", None),
  176. pyprep_channel_wise=bool(getattr(cfg, "pyprep_channel_wise", False)),
  177. pyprep_max_chunk_size=getattr(cfg, "pyprep_max_chunk_size", 8),
  178. pyprep_by_segment=bool(getattr(cfg, "pyprep_by_segment", False)),
  179. pyprep_safe_quantile_patch=bool(getattr(cfg, "pyprep_safe_quantile_patch", False)),
  180. ica_fit_hz=float(getattr(cfg, "ica_fit_hz", 100.0)),
  181. iclabel_backend=getattr(cfg, "iclabel_backend", "auto"),
  182. )
  183. # keep legacy names used in the rest of this module
  184. output_dir = str(_CFG_RT.output_root)
  185. target_sfreq = float(_CFG_RT.target_sfreq) if _CFG_RT.target_sfreq else None
  186. do_asr = _CFG_RT.use_asr
  187. do_pyprep = _CFG_RT.use_pyprep
  188. remove_cfa = _CFG_RT.remove_cfa
  189. remove_cfa_mode = _CFG_RT.remove_cfa_mode
  190. log_file = _CFG_RT.log_file or ""
  191. prep_params = {
  192. "line_freqs": _CFG_RT.line_freqs,
  193. "ref_chs": _CFG_RT.ref_chs,
  194. "reref_chs": _CFG_RT.reref_chs,
  195. "l_freq": _CFG_RT.high_pass,
  196. "h_freq": _CFG_RT.low_pass,
  197. "ransac": _CFG_RT.prep_ransac,
  198. }
  199. def _canonicalise_name(ch: str) -> str:
  200. nm = ch.strip()
  201. nm = re.sub(r'^\s*EEG\s+', '', nm, flags=re.IGNORECASE)
  202. parts = [p.strip() for p in nm.split('-') if p.strip()]
  203. if len(parts) > 1:
  204. nm = parts[1] if parts[0].isdigit() else parts[0]
  205. nm = nm.upper()
  206. nm = nm.replace('FP', 'Fp').replace('Z', 'z')
  207. repl = {'T7': 'T3', 'T8': 'T4', 'P7': 'T5', 'P8': 'T6'}
  208. return repl.get(nm, nm)
  209. def _canonical_name_mapping(ch_names: list[str]) -> dict[str, str]:
  210. mapping, used, counts = {}, set(), {}
  211. for ch in ch_names:
  212. base = _canonicalise_name(ch)
  213. name = base
  214. while name in used:
  215. counts[base] = counts.get(base, 0) + 1
  216. name = f"{base}-{counts[base]}"
  217. mapping[ch] = name
  218. used.add(name)
  219. return mapping
  220. AUX_EOG = {'ROC', 'LOC'}
  221. AUX_STIM = {'PHOTIC', 'MKR', 'MKR+', 'MARKER'}
  222. AUX_MISC = {'IBI', 'BURSTS', 'SUPPR'}
  223. RSP_PEAK_DESC = "RSP_Peak"
  224. RSP_TROUGH_DESC = "RSP_Trough"
  225. ECG_TOKENS = ("ECG", "EKG", "CARD", "EXG")
  226. def _optional_name(value: str | None) -> str | None:
  227. text = str(value or "").strip()
  228. return text or None
  229. def _find_channel(raw: mne.io.BaseRaw, preferred: str | None) -> str | None:
  230. preferred = _optional_name(preferred)
  231. if not preferred:
  232. return None
  233. if preferred in raw.ch_names:
  234. return preferred
  235. wanted = preferred.casefold()
  236. for ch in raw.ch_names:
  237. if ch.casefold() == wanted:
  238. return ch
  239. canonical = _canonicalise_name(preferred)
  240. if canonical in raw.ch_names:
  241. return canonical
  242. wanted = canonical.casefold()
  243. for ch in raw.ch_names:
  244. if ch.casefold() == wanted:
  245. return ch
  246. if _canonicalise_name(ch).casefold() == wanted:
  247. return ch
  248. return None
  249. def _active_spirometry_channel(spirometry_channel: str | None = None) -> str | None:
  250. return _optional_name(spirometry_channel) or (_optional_name(_CFG_RT.spirometry_channel) if _CFG_RT else None)
  251. def _mark_spirometry_channel(raw: mne.io.BaseRaw, spirometry_channel: str | None) -> str | None:
  252. ch = _find_channel(raw, _active_spirometry_channel(spirometry_channel))
  253. if ch:
  254. raw.set_channel_types({ch: "resp"})
  255. return ch
  256. def _mark_ecg_channels(raw: mne.io.BaseRaw, ecg_channel: str | None = None) -> list[str]:
  257. names = []
  258. preferred = _find_channel(raw, ecg_channel)
  259. if preferred:
  260. names.append(preferred)
  261. for ch in raw.ch_names:
  262. if any(tok in ch.upper() for tok in ECG_TOKENS) and ch not in names:
  263. names.append(ch)
  264. for ch in names:
  265. try:
  266. raw.set_channel_types({ch: "ecg"})
  267. except Exception:
  268. pass
  269. return names
  270. def _flip_ecg_channels(raw: mne.io.BaseRaw, ecg_channel: str | None = None) -> list[str]:
  271. names = _mark_ecg_channels(raw, ecg_channel)
  272. if not names:
  273. names = [raw.ch_names[i] for i in mne.pick_types(raw.info, ecg=True)]
  274. idx = [raw.ch_names.index(ch) for ch in names if ch in raw.ch_names]
  275. if idx:
  276. raw._data[idx] *= -1
  277. return [raw.ch_names[i] for i in idx]
  278. def _preproc_base_from_output(path: str | Path) -> str:
  279. stem = Path(path).stem
  280. for suffix in ("_pp_raw_keepcfa", "_pp_raw", "_raw_keepcfa", "_raw"):
  281. if stem.endswith(suffix):
  282. return stem[:-len(suffix)]
  283. return stem
  284. def _has_spirometry_annotations(raw: mne.io.BaseRaw) -> bool:
  285. if raw.annotations is None or len(raw.annotations) == 0:
  286. return False
  287. descriptions = set(np.asarray(raw.annotations.description).astype(str))
  288. return bool({RSP_PEAK_DESC, RSP_TROUGH_DESC} & descriptions)
  289. def _add_spirometry_annotations(raw: mne.io.BaseRaw, peaks: np.ndarray, troughs: np.ndarray) -> None:
  290. if raw.annotations is not None and len(raw.annotations):
  291. old = [i for i, desc in enumerate(raw.annotations.description) if desc in (RSP_PEAK_DESC, RSP_TROUGH_DESC)]
  292. if old:
  293. annotations = raw.annotations.copy()
  294. annotations.delete(old)
  295. raw.set_annotations(annotations)
  296. samples = np.concatenate([peaks, troughs]).astype(int)
  297. if samples.size == 0:
  298. return
  299. descriptions = [RSP_PEAK_DESC] * len(peaks) + [RSP_TROUGH_DESC] * len(troughs)
  300. order = np.argsort(samples)
  301. annotations = mne.Annotations(
  302. onset=samples[order] / float(raw.info["sfreq"]),
  303. duration=np.zeros(samples.size),
  304. description=[descriptions[i] for i in order],
  305. orig_time=raw.annotations.orig_time,
  306. )
  307. raw.set_annotations(raw.annotations + annotations)
  308. def save_spirometry_analysis(
  309. raw: mne.io.BaseRaw,
  310. spirometry_channel: str | None,
  311. output_dir: str | Path | None,
  312. base: str,
  313. redo: bool = False,
  314. ) -> dict:
  315. requested = _active_spirometry_channel(spirometry_channel)
  316. if not requested:
  317. return {"spirometry_status": "off"}
  318. ch = _find_channel(raw, requested)
  319. if not ch:
  320. return {"spirometry_channel": requested, "spirometry_status": "missing"}
  321. sfreq = float(raw.info["sfreq"])
  322. signals, info = nk.rsp_process(raw.get_data(picks=ch)[0], sampling_rate=sfreq)
  323. peaks = np.asarray(info.get("RSP_Peaks", []), dtype=int)
  324. troughs = np.asarray(info.get("RSP_Troughs", []), dtype=int)
  325. _add_spirometry_annotations(raw, peaks, troughs)
  326. rec = {
  327. "spirometry_channel": ch,
  328. "spirometry_status": "ok",
  329. "spirometry_peaks": str(len(peaks)),
  330. "spirometry_troughs": str(len(troughs)),
  331. }
  332. if output_dir:
  333. out_dir = Path(output_dir)
  334. out_dir.mkdir(parents=True, exist_ok=True)
  335. signals_csv = out_dir / f"{base}_rsp_signals.csv"
  336. summary_csv = out_dir / f"{base}_rsp_summary.csv"
  337. events_csv = out_dir / f"{base}_rsp_events.csv"
  338. qc_png = out_dir / f"{base}_rsp_qc.png"
  339. if redo or not signals_csv.exists():
  340. signals.to_csv(signals_csv, index=False)
  341. try:
  342. summary = nk.rsp_analyze(signals, sampling_rate=sfreq, method="interval-related")
  343. except Exception as exc:
  344. summary = pd.DataFrame([{"error": str(exc)}])
  345. if redo or not summary_csv.exists():
  346. summary.to_csv(summary_csv, index=False)
  347. if redo or not events_csv.exists():
  348. event_rows = (
  349. [{"sample": int(s), "time_s": float(s / sfreq), "kind": "peak"} for s in peaks]
  350. + [{"sample": int(s), "time_s": float(s / sfreq), "kind": "trough"} for s in troughs]
  351. )
  352. pd.DataFrame(event_rows, columns=["sample", "time_s", "kind"]).sort_values("sample").to_csv(events_csv, index=False)
  353. if redo or not qc_png.exists():
  354. import matplotlib.pyplot as plt
  355. nk.rsp_plot(signals, info, static=True)
  356. fig = plt.gcf()
  357. fig.savefig(qc_png, dpi=150, bbox_inches="tight")
  358. plt.close(fig)
  359. rec.update({
  360. "spirometry_qc_png": str(qc_png),
  361. "spirometry_signals_csv": str(signals_csv),
  362. "spirometry_summary_csv": str(summary_csv),
  363. })
  364. return rec
  365. def spirometry_phase_at_samples(raw: mne.io.BaseRaw, samples: np.ndarray) -> pd.DataFrame | None:
  366. if raw.annotations is None or len(raw.annotations) == 0:
  367. return None
  368. sfreq = float(raw.info["sfreq"])
  369. desc = np.asarray(raw.annotations.description).astype(str)
  370. ann_samples = np.rint(np.asarray(raw.annotations.onset) * sfreq).astype(int)
  371. peaks = ann_samples[desc == RSP_PEAK_DESC]
  372. troughs = ann_samples[desc == RSP_TROUGH_DESC]
  373. if len(peaks) == 0 or len(troughs) == 0:
  374. return None
  375. phase = nk.rsp_phase(peaks=peaks, troughs=troughs, desired_length=raw.n_times)
  376. idx = np.clip(np.asarray(samples, dtype=int), 0, raw.n_times - 1)
  377. rows = phase.iloc[idx].reset_index(drop=True).rename(columns={
  378. "RSP_Phase": "resp_phase",
  379. "RSP_Phase_Completion": "resp_phase_completion",
  380. })
  381. vals = pd.to_numeric(rows["resp_phase"], errors="coerce").to_numpy()
  382. rows["resp_phase_label"] = np.where(
  383. np.isfinite(vals),
  384. np.where(vals == 1, "inspiratory", "expiratory"),
  385. "unknown",
  386. )
  387. return rows
  388. def _classify_aux_channels(raw: mne.io.BaseRaw) -> None:
  389. """Classify obvious auxiliary channels but preserve all EEG leads.
  390. Previous logic demoted any EEG channel not in the 10-20 set to 'misc', causing
  391. large arrays (e.g., dense caps, BioSemi, EGI) to be dropped from EEG processing.
  392. We now only re-type clearly non-EEG auxiliary channels (EOG/stim/misc/ECG/EMG) and
  393. leave all remaining channels as EEG.
  394. """
  395. for ch in list(raw.ch_names):
  396. up = ch.upper()
  397. if up in AUX_EOG:
  398. raw.set_channel_types({ch: 'eog'})
  399. elif up in AUX_STIM:
  400. raw.set_channel_types({ch: 'stim'})
  401. elif up in AUX_MISC:
  402. raw.set_channel_types({ch: 'misc'})
  403. elif any(tok in up for tok in ECG_TOKENS):
  404. raw.set_channel_types({ch: 'ecg'})
  405. elif up == 'EMG':
  406. raw.set_channel_types({ch: 'emg'})
  407. # Do not demote remaining EEG channels; keep full set intact.
  408. _DEF_MONTAGES = [
  409. "standard_1020", "standard_1005", "biosemi32", "biosemi64", "biosemi128",
  410. "GSN-HydroCel-32", "GSN-HydroCel-64", "GSN-HydroCel-128", "GSN-HydroCel-256",
  411. "easycap-M1", "easycap-M10", "egi256"
  412. ]
  413. def _auto_detect_montage(raw: mne.io.BaseRaw, candidates: list[str] | None = None) -> str | None:
  414. """Optional montage auto-detection: choose built-in montage with highest channel name coverage."""
  415. if candidates is None:
  416. candidates = _DEF_MONTAGES
  417. ch_set = {
  418. _canonicalise_name(ch)
  419. for ch, kind in zip(raw.ch_names, raw.get_channel_types())
  420. if kind == "eeg" and not any(tok in _canonicalise_name(ch).upper() for tok in ECG_TOKENS)
  421. }
  422. best_name, best_cov = None, 0.0
  423. for name in candidates:
  424. try:
  425. mont = mne.channels.make_standard_montage(name)
  426. except Exception:
  427. continue
  428. mont_chs = set(mont.ch_names)
  429. inter = ch_set & mont_chs
  430. if not ch_set:
  431. continue
  432. cov = len(inter) / max(1, len(ch_set))
  433. if cov > best_cov:
  434. best_cov, best_name = cov, name
  435. # require minimal coverage threshold
  436. if best_cov >= 0.5:
  437. return best_name
  438. return None
  439. def standardise_and_montage(raw: mne.io.BaseRaw) -> mne.io.BaseRaw:
  440. """Rename channels, apply montage early, then classify auxiliaries.
  441. Applying the montage before auxiliary classification avoids losing EEG leads
  442. when dense arrays are used. All EEG channels are preserved.
  443. """
  444. # Canonicalise channel names first to maximize montage matching
  445. raw.rename_channels(_canonical_name_mapping(raw.ch_names))
  446. # Decide montage: explicit, auto-detect, or fallback
  447. m_name = _CFG_RT.montage_name if _CFG_RT else None
  448. if m_name is None or str(m_name).lower() in ("", "auto", "none"):
  449. detected = _auto_detect_montage(raw)
  450. m_name = detected or "standard_1020"
  451. mont = mne.channels.make_standard_montage(m_name)
  452. raw.set_montage(mont, match_case=False, on_missing='ignore')
  453. logging.info(f"Applied montage: {m_name}")
  454. # Classify auxiliaries AFTER montage so EEG leads remain EEG
  455. eeg_before = sum(1 for t in raw.get_channel_types() if t == 'eeg')
  456. _classify_aux_channels(raw)
  457. eeg_after = sum(1 for t in raw.get_channel_types() if t == 'eeg')
  458. if eeg_after < eeg_before:
  459. logging.warning(f"EEG channel count decreased from {eeg_before} to {eeg_after}. Check classification rules.")
  460. # Drop EEG channels not present in montage (post-application)
  461. mont_chs = set(mont.ch_names)
  462. drop_eeg = [ch for ch, t in zip(raw.ch_names, raw.get_channel_types()) if t == "eeg" and ch not in mont_chs]
  463. eeg_chs = [ch for ch, t in zip(raw.ch_names, raw.get_channel_types()) if t == "eeg"]
  464. if drop_eeg and len(drop_eeg) == len(eeg_chs):
  465. logging.warning("No EEG channels matched montage %s; preserving unmatched EEG names.", m_name)
  466. elif drop_eeg:
  467. raw.drop_channels(drop_eeg)
  468. # force the lead name types to be str (to avoid issues with numpy.str_)
  469. raw.rename_channels({ch: str(ch) for ch in raw.ch_names})
  470. return raw
  471. def _finite_ch_pos(ch_pos: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
  472. ok = {}
  473. for k, v in (ch_pos or {}).items():
  474. arr = np.asarray(v, float)
  475. if arr.shape and np.all(np.isfinite(arr)):
  476. ok[k] = arr
  477. return ok
  478. def _trim_eeg_montage_no_nan(raw_eeg: mne.io.BaseRaw) -> mne.channels.DigMontage | None:
  479. mont = raw_eeg.get_montage()
  480. if mont is None:
  481. return None
  482. pos = mont.get_positions()
  483. ch_pos = _finite_ch_pos(pos.get('ch_pos', {}))
  484. present = [ch for ch in raw_eeg.ch_names if ch in ch_pos]
  485. if not present:
  486. return None
  487. return mne.channels.make_dig_montage(
  488. ch_pos={k: ch_pos[k] for k in present},
  489. nasion=pos.get('nasion'), lpa=pos.get('lpa'), rpa=pos.get('rpa'),
  490. hpi=pos.get('hpi'), coord_frame=pos.get('coord_frame', 'head'),
  491. )
  492. def _prune_prep_params_for_raw(params: dict, raw: mne.io.BaseRaw) -> dict:
  493. import copy
  494. pruned = copy.deepcopy(params)
  495. present = set(raw.ch_names)
  496. keys = {
  497. "ref_chs","reref_chs","eog_chs","corr_chs",
  498. "ransac_channel_picks","interpolation_channel_picks",
  499. "exclude","include","target_channels","bad_channel_prior"
  500. }
  501. def _walk(d):
  502. for k, v in list(d.items()):
  503. if isinstance(v, dict):
  504. _walk(v)
  505. elif isinstance(v, (list, tuple)) and any(isinstance(x, str) for x in v):
  506. d[k] = [x for x in v if isinstance(x, str) and x in present]
  507. if k in keys:
  508. vv = d.get(k, None)
  509. if isinstance(vv, (list, tuple)):
  510. d[k] = [x for x in vv if isinstance(x, str) and x in present]
  511. return d
  512. return _walk(pruned)
  513. def _add_channels_by_name(
  514. target: mne.io.BaseRaw,
  515. source: mne.io.BaseRaw,
  516. channel_names: list[str],
  517. ) -> mne.io.BaseRaw:
  518. channel_names = [ch for ch in channel_names if ch in source.ch_names and ch not in target.ch_names]
  519. if not channel_names:
  520. return target
  521. picks = mne.pick_channels(source.ch_names, include=channel_names, ordered=True)
  522. data = source.get_data(picks=picks)
  523. info = mne.pick_info(source.info, picks, copy=True)
  524. selected = mne.io.RawArray(
  525. data,
  526. info,
  527. first_samp=source.first_samp,
  528. copy="auto",
  529. verbose=False,
  530. )
  531. target.add_channels([selected], force_update_info=True)
  532. return target
  533. def _add_non_eeg_channels(target: mne.io.BaseRaw, source: mne.io.BaseRaw) -> mne.io.BaseRaw:
  534. non_eeg_chs = [
  535. ch for ch, kind in zip(source.ch_names, source.get_channel_types()) if kind != "eeg"
  536. ]
  537. return _add_channels_by_name(target, source, non_eeg_chs)
  538. def _snip_sample_bounds_for_input(input_path: str, raw: mne.io.BaseRaw) -> list[tuple[int, int]] | None:
  539. manifest = Path(input_path).with_name("snip_keyfile.csv")
  540. if not manifest.exists():
  541. return None
  542. sfreq = float(raw.info["sfreq"])
  543. target = Path(input_path).name
  544. bounds: list[tuple[int, int]] = []
  545. with manifest.open(newline="", encoding="utf-8-sig") as handle:
  546. for row in csv.DictReader(handle):
  547. if Path(str(row.get("output_edf", ""))).name != target:
  548. continue
  549. try:
  550. start = float(row["output_start_seconds"])
  551. end = float(row["output_end_seconds"])
  552. except Exception:
  553. continue
  554. if not (np.isfinite(start) and np.isfinite(end) and end > start):
  555. continue
  556. start_samp = max(0, min(raw.n_times, int(round(start * sfreq))))
  557. end_samp = max(0, min(raw.n_times, int(round(end * sfreq))))
  558. if end_samp > start_samp:
  559. bounds.append((start_samp, end_samp))
  560. if len(bounds) <= 1:
  561. return None
  562. bounds.sort()
  563. tol = max(2, int(round(0.01 * sfreq)))
  564. expected = 0
  565. for start, stop in bounds:
  566. if abs(start - expected) > tol:
  567. logging.warning("Snip manifest for %s has a gap/overlap near sample %s; using whole-record PyPREP.", target, expected)
  568. return None
  569. expected = stop
  570. if abs(expected - raw.n_times) > tol:
  571. logging.warning("Snip manifest for %s covers %s/%s samples; using whole-record PyPREP.", target, expected, raw.n_times)
  572. return None
  573. return bounds
  574. def _safe_mat_quantile_1d(values, q: float) -> float:
  575. vals = np.asarray(values, dtype=float).ravel()
  576. vals = vals[np.isfinite(vals)]
  577. n = vals.size
  578. if n == 0:
  579. return float("nan")
  580. if n == 1:
  581. return float(vals[0])
  582. q_adj = ((float(q) - 0.5) * n / (n - 1)) + 0.5
  583. exact_idx = (n - 1) * np.clip(q_adj, 0, 1)
  584. pre_idx = int(np.floor(exact_idx))
  585. post_idx = int(np.ceil(exact_idx))
  586. vals.partition((pre_idx, post_idx))
  587. pre = vals[pre_idx]
  588. post = vals[post_idx]
  589. return float(pre + (post - pre) * (exact_idx - pre_idx))
  590. def _safe_mat_quantile(arr, q, axis=None):
  591. data = np.asarray(arr)
  592. if data.size == 0:
  593. return np.nan
  594. if data.ndim > 2:
  595. raise ValueError(f"Only 1D and 2D arrays are supported (input has {data.ndim} dimensions)")
  596. if axis is None or data.ndim == 1:
  597. return _safe_mat_quantile_1d(data, q)
  598. if axis < 0:
  599. axis += data.ndim
  600. if axis == 0:
  601. return np.asarray([_safe_mat_quantile_1d(data[:, idx], q) for idx in range(data.shape[1])])
  602. if axis == 1:
  603. return np.asarray([_safe_mat_quantile_1d(data[idx, :], q) for idx in range(data.shape[0])])
  604. raise ValueError(f"axis {axis} is out of bounds for array of dimension {data.ndim}")
  605. def _safe_mat_iqr(arr, axis=None):
  606. return _safe_mat_quantile(arr, 0.75, axis) - _safe_mat_quantile(arr, 0.25, axis)
  607. def _install_pyprep_safe_quantile_patch() -> None:
  608. global _PYPREP_SAFE_QUANTILE_PATCHED
  609. if _PYPREP_SAFE_QUANTILE_PATCHED or pyprep is None:
  610. return
  611. try:
  612. import pyprep.find_noisy_channels as noisy_channels
  613. import pyprep.utils as utils
  614. except Exception:
  615. return
  616. utils._mat_quantile = _safe_mat_quantile
  617. utils._mat_iqr = _safe_mat_iqr
  618. noisy_channels._mat_quantile = _safe_mat_quantile
  619. noisy_channels._mat_iqr = _safe_mat_iqr
  620. _PYPREP_SAFE_QUANTILE_PATCHED = True
  621. logging.info("Patched PyPREP quantile helper for bounded-memory channel IQR.")
  622. # ------------------------------- Core steps -------------------------------- #
  623. def run_pyprep(
  624. raw: mne.io.BaseRaw,
  625. param_dict: dict | None = None,
  626. random_seed: int = 42,
  627. ransac: bool = True,
  628. channel_wise: bool | None = None,
  629. max_chunk_size: int | None = None,
  630. segment_bounds: list[tuple[int, int]] | None = None,
  631. ) -> mne.io.BaseRaw:
  632. if pyprep is None:
  633. raise RuntimeError(
  634. "PyPREP is enabled but the 'pyprep' package is not installed. "
  635. "Disable 'Run PyPREP' or install pyprep."
  636. )
  637. if _CFG_RT is not None and _CFG_RT.pyprep_safe_quantile_patch:
  638. _install_pyprep_safe_quantile_patch()
  639. if param_dict is None:
  640. param_dict = prep_params
  641. if channel_wise is None:
  642. channel_wise = bool(getattr(_CFG_RT, "pyprep_channel_wise", False))
  643. if max_chunk_size is None and _CFG_RT is not None:
  644. max_chunk_size = _CFG_RT.pyprep_max_chunk_size
  645. # Segment-wise PyPREP is retained as an opt-in compatibility mode. It is
  646. # substantially slower and keeps several complete Raw objects alive, so the
  647. # stable default is one fit over the patient recording.
  648. if segment_bounds and len(segment_bounds) > 1:
  649. sfreq = float(raw.info["sfreq"])
  650. chunks = []
  651. logging.info("Running PyPREP over %d snip segments.", len(segment_bounds))
  652. for idx, (start, stop) in enumerate(segment_bounds, start=1):
  653. start = max(0, min(raw.n_times, int(start)))
  654. stop = max(0, min(raw.n_times, int(stop)))
  655. if stop <= start:
  656. continue
  657. chunk = raw.copy().crop(
  658. tmin=start / sfreq,
  659. tmax=(stop - 1) / sfreq,
  660. include_tmax=True,
  661. )
  662. try:
  663. chunks.append(
  664. run_pyprep(
  665. chunk,
  666. param_dict,
  667. random_seed=random_seed + idx,
  668. ransac=ransac,
  669. channel_wise=channel_wise,
  670. max_chunk_size=max_chunk_size,
  671. segment_bounds=None,
  672. )
  673. )
  674. except Exception as exc:
  675. logging.warning(
  676. "PyPREP failed on segment %d/%d (%d:%d); keeping raw segment: %s",
  677. idx,
  678. len(segment_bounds),
  679. start,
  680. stop,
  681. exc,
  682. )
  683. chunks.append(chunk)
  684. if chunks:
  685. return mne.concatenate_raws(chunks, verbose=False)
  686. raw_eeg = raw.copy().pick("eeg")
  687. safe_params = _prune_prep_params_for_raw(param_dict, raw_eeg)
  688. trimmed_mont = _trim_eeg_montage_no_nan(raw_eeg)
  689. if trimmed_mont is None:
  690. logging.warning("PyPREP skipped; no EEG channels have usable montage positions.")
  691. _add_non_eeg_channels(raw_eeg, raw)
  692. return raw_eeg
  693. safe_params.setdefault("line_freqs", ())
  694. safe_params.setdefault("l_freq", None)
  695. safe_params.setdefault("h_freq", None)
  696. nyquist = float(raw_eeg.info["sfreq"]) / 2.0
  697. safe_params["line_freqs"] = tuple(
  698. float(freq)
  699. for freq in safe_params.get("line_freqs", ())
  700. if 0.0 < float(freq) < nyquist
  701. )
  702. def _fit_prep(use_ransac: bool):
  703. kwargs = {
  704. "raw": raw_eeg,
  705. "montage": trimmed_mont,
  706. "prep_params": safe_params,
  707. "random_state": random_seed,
  708. "ransac": use_ransac,
  709. "channel_wise": bool(channel_wise),
  710. }
  711. try:
  712. parameters = inspect.signature(pyprep.PrepPipeline).parameters
  713. except Exception:
  714. parameters = {}
  715. if "max_chunk_size" in parameters:
  716. kwargs["max_chunk_size"] = max_chunk_size
  717. if "reject_by_annotation" in parameters:
  718. kwargs["reject_by_annotation"] = "omit"
  719. prep = pyprep.PrepPipeline(**kwargs)
  720. _stage(
  721. f"pyprep_fit:r={int(use_ransac)}:channel_wise={int(bool(channel_wise))}"
  722. )
  723. with open(os.devnull, "w") as fnull, redirect_stdout(fnull), redirect_stderr(fnull):
  724. return prep.fit()
  725. result = None
  726. try:
  727. result = _fit_prep(ransac)
  728. except Exception as exc:
  729. failure = exc
  730. msg = str(failure)
  731. ransac_exhausted = ransac and (
  732. "Too few channels in the original data to reliably perform RANSAC" in msg
  733. or "Too many noisy channels in the data to reliably perform RANSAC" in msg
  734. or "arrays used as indices must be of integer (or boolean) type" in msg
  735. )
  736. if ransac_exhausted:
  737. logging.warning("PyPREP RANSAC skipped after failure: %s", failure)
  738. try:
  739. result = _fit_prep(False)
  740. except Exception as fallback_exc:
  741. failure = fallback_exc
  742. else:
  743. failure = None
  744. if failure is not None and "RobustReference:TooManyBad" in str(failure):
  745. logging.warning(
  746. "PyPREP skipped; too many bad EEG channels for robust reference: %s",
  747. failure,
  748. )
  749. failure = None
  750. if failure is not None:
  751. print("EEG picks:", sorted(raw_eeg.ch_names))
  752. print(
  753. "trimmed EEG montage ch_pos:",
  754. sorted(
  755. _finite_ch_pos(
  756. trimmed_mont.get_positions().get("ch_pos", {})
  757. ).keys()
  758. ),
  759. )
  760. print(
  761. "params (ref/reref/eog/ransac/interp/corr):",
  762. {
  763. key: safe_params.get(key)
  764. for key in (
  765. "ref_chs",
  766. "reref_chs",
  767. "eog_chs",
  768. "ransac_channel_picks",
  769. "interpolation_channel_picks",
  770. "corr_chs",
  771. )
  772. },
  773. )
  774. raise RuntimeError(f"PyPREP failed: {failure}") from failure
  775. cleaned_eeg = result.raw_eeg if result is not None else raw_eeg
  776. if cleaned_eeg.info.get("bads"):
  777. try:
  778. cleaned_eeg.interpolate_bads(reset_bads=True)
  779. except Exception as exc:
  780. raise RuntimeError("PyPREP bad channel interpolation failed.") from exc
  781. cleaned_eeg.info["bads"] = []
  782. _add_non_eeg_channels(cleaned_eeg, raw)
  783. return cleaned_eeg
  784. def _effective_filter_bounds(
  785. raw: mne.io.BaseRaw,
  786. l_freq: float | None,
  787. h_freq: float | None,
  788. ) -> tuple[float | None, float | None]:
  789. nyquist = float(raw.info["sfreq"]) / 2.0
  790. low = float(l_freq) if l_freq is not None else None
  791. high = float(h_freq) if h_freq is not None else None
  792. if low is not None and low <= 0:
  793. low = None
  794. if high is not None and high >= nyquist:
  795. # Sampling already imposes this upper bound; asking MNE to design a
  796. # filter exactly at Nyquist is invalid.
  797. high = None
  798. if low is not None and low >= nyquist:
  799. raise ValueError(f"High-pass {low} Hz is not below Nyquist {nyquist} Hz")
  800. if low is not None and high is not None and low >= high:
  801. raise ValueError(f"Invalid EEG band-pass: {low}–{high} Hz")
  802. return low, high
  803. def _filter_eeg_once(
  804. raw: mne.io.BaseRaw,
  805. l_freq: float | None,
  806. h_freq: float | None,
  807. ) -> mne.io.BaseRaw:
  808. low, high = _effective_filter_bounds(raw, l_freq, h_freq)
  809. # MNE records the effective high- and low-pass limits in info. The old
  810. # implementation compared these fields to the wrong requested cut-offs,
  811. # so already-filtered data were filtered again.
  812. existing_low = float(raw.info.get("highpass", 0.0) or 0.0)
  813. existing_high = float(raw.info.get("lowpass", np.inf) or np.inf)
  814. tolerance = 1e-6
  815. if low is not None and existing_low >= low - tolerance:
  816. if existing_low > low + tolerance:
  817. logging.warning(
  818. "Input is already high-passed at %.3f Hz, above requested %.3f Hz; "
  819. "the removed lower frequencies cannot be restored.",
  820. existing_low,
  821. low,
  822. )
  823. low = None
  824. if high is not None and existing_high <= high + tolerance:
  825. if existing_high < high - tolerance:
  826. logging.warning(
  827. "Input is already low-passed at %.3f Hz, below requested %.3f Hz; "
  828. "the removed higher frequencies cannot be restored.",
  829. existing_high,
  830. high,
  831. )
  832. high = None
  833. if low is None and high is None:
  834. return raw
  835. _stage(f"eeg_filter:{low}:{high}")
  836. raw.filter(
  837. l_freq=low,
  838. h_freq=high,
  839. picks="eeg",
  840. fir_design="firwin",
  841. n_jobs=1,
  842. verbose=False,
  843. )
  844. return raw
  845. def _normalise_asr_cutoff(value) -> float | None:
  846. if value is None or value is False:
  847. return None
  848. if value is True:
  849. return 20.0
  850. try:
  851. cutoff = float(value)
  852. except (TypeError, ValueError):
  853. return None
  854. return cutoff if cutoff > 0 else None
  855. def _ica_decimation(raw: mne.io.BaseRaw) -> int:
  856. target_hz = float(getattr(_CFG_RT, "ica_fit_hz", 100.0) or 100.0)
  857. return max(1, int(np.ceil(float(raw.info["sfreq"]) / target_hz)))
  858. _ICLABEL_CLASSES = (
  859. "brain",
  860. "muscle artifact",
  861. "eye blink",
  862. "heart beat",
  863. "line noise",
  864. "channel noise",
  865. "other",
  866. )
  867. def _normalise_iclabel_label(label: str) -> str:
  868. text = str(label).strip().lower().replace("_", " ")
  869. aliases = {
  870. "heart": "heart beat",
  871. "cardiac": "heart beat",
  872. "eye": "eye blink",
  873. "muscle": "muscle artifact",
  874. }
  875. return aliases.get(text, text)
  876. def _label_ica_components(inst: mne.io.BaseRaw, ica: ICA) -> list[str]:
  877. backend = str(getattr(_CFG_RT, "iclabel_backend", "auto") or "").strip().lower()
  878. if backend in {"", "auto", "none"}:
  879. backend = ""
  880. if backend and iclabel_label_components is not None:
  881. _stage(f"iclabel:{backend}")
  882. try:
  883. probabilities = iclabel_label_components(
  884. inst, ica, inplace=True, backend=backend
  885. )
  886. except Exception as exc:
  887. raise RuntimeError(
  888. f"ICLabel backend {backend!r} failed. Install a compatible "
  889. f"backend or set preprocessing_tuning.iclabel_backend: {exc}"
  890. ) from exc
  891. probabilities = np.asarray(probabilities)
  892. if probabilities.ndim != 2 or probabilities.shape[1] != len(_ICLABEL_CLASSES):
  893. raise RuntimeError(
  894. f"Unexpected ICLabel probability shape: {probabilities.shape}"
  895. )
  896. return [
  897. _ICLABEL_CLASSES[int(index)]
  898. for index in np.argmax(probabilities, axis=1)
  899. ]
  900. if label_components is None:
  901. raise RuntimeError(
  902. "ICA+ICLabel is enabled but the 'mne_icalabel' package is not installed. "
  903. "Disable 'Run ICA + ICLabel' or install mne-icalabel."
  904. )
  905. _stage("iclabel:auto")
  906. result = label_components(inst, ica, method="iclabel")
  907. labels = [_normalise_iclabel_label(value) for value in result.get("labels", [])]
  908. if not labels:
  909. raise RuntimeError("ICLabel returned no component labels")
  910. return labels
  911. def _iclabel_instance(raw_eeg: mne.io.BaseRaw) -> mne.io.BaseRaw:
  912. highpass = float(raw_eeg.info.get("highpass", 0.0) or 0.0)
  913. lowpass = float(raw_eeg.info.get("lowpass", np.inf) or np.inf)
  914. required_high = min(100.0, np.nextafter(float(raw_eeg.info["sfreq"]) / 2.0, 0.0))
  915. if highpass >= 0.95 and lowpass <= required_high + 0.5:
  916. return raw_eeg
  917. inst = raw_eeg.copy()
  918. return _filter_eeg_once(inst, 1.0, required_high)
  919. def _max_eeg_time_std(inst: mne.io.BaseRaw) -> float:
  920. picks = mne.pick_types(inst.info, eeg=True, exclude=[])
  921. maximum = 0.0
  922. for pick in picks:
  923. if inst.preload:
  924. values = inst._data[pick]
  925. else:
  926. values = inst.get_data(picks=[pick])[0]
  927. maximum = max(maximum, float(np.nanstd(values)))
  928. return maximum
  929. def _apply_ica_excludes(
  930. inst: mne.io.BaseRaw,
  931. fitted_ica: ICA,
  932. exclude: list[int],
  933. ) -> mne.io.BaseRaw:
  934. fitted_ica.exclude = list(exclude)
  935. if not fitted_ica.exclude:
  936. return inst.copy()
  937. before = _max_eeg_time_std(inst)
  938. _stage(f"ica_apply:n_exclude={len(fitted_ica.exclude)}")
  939. cleaned = fitted_ica.apply(inst.copy(), verbose=False)
  940. if before and _max_eeg_time_std(cleaned) <= before * 1e-6:
  941. logging.warning("ICA application flattened EEG; keeping pre-ICA data.")
  942. fitted_ica.exclude = []
  943. return inst.copy()
  944. return cleaned
  945. def _apply_asr(
  946. raw_eeg: mne.io.BaseRaw,
  947. asr_thresh: float | int | bool | None,
  948. ) -> mne.io.BaseRaw:
  949. cutoff = _normalise_asr_cutoff(asr_thresh)
  950. if cutoff is None:
  951. return raw_eeg
  952. if asrpy is None:
  953. raise RuntimeError(
  954. "ASR is enabled but the 'asrpy' package is not installed. "
  955. "Disable 'Run ASR' or install asrpy."
  956. )
  957. # ASRpy binds the model to the sampling rate supplied at construction.
  958. # Fitting at 100 Hz and then transforming a higher-rate Raw is not a valid
  959. # speed shortcut and can fail inside NumPy/BLAS code. Keep fit and
  960. # transform at exactly the same sampling rate.
  961. sfreq = float(raw_eeg.info["sfreq"])
  962. _stage(f"asr_fit:cutoff={cutoff}:sfreq={sfreq}")
  963. asr = asrpy.ASR(sfreq=sfreq, cutoff=cutoff)
  964. asr.fit(raw_eeg)
  965. _stage("asr_transform")
  966. return asr.transform(raw_eeg)
  967. def _fit_ica_and_labels(
  968. raw: mne.io.BaseRaw,
  969. asr_thresh: float | int | bool | None,
  970. random_seed: int,
  971. n_comp: int | None,
  972. ):
  973. raw_eeg = raw.copy().pick("eeg")
  974. raw_eeg, _ = mne.set_eeg_reference(raw_eeg, ref_channels="average")
  975. raw_eeg = _apply_asr(raw_eeg, asr_thresh)
  976. n_channels = len(raw_eeg.ch_names) - len(raw_eeg.info.get("bads", []))
  977. if n_channels < 3:
  978. raise RuntimeError(f"ICA requires at least 3 usable EEG channels; found {n_channels}")
  979. if n_comp is None:
  980. n_comp = min(n_channels - 1, 48)
  981. if not 1 < int(n_comp) < n_channels:
  982. raise ValueError(
  983. f"ICA n_components must be between 2 and {n_channels - 1}; got {n_comp}"
  984. )
  985. ica = ICA(
  986. n_components=int(n_comp),
  987. method="infomax",
  988. random_state=random_seed,
  989. fit_params={"extended": True},
  990. max_iter="auto",
  991. )
  992. decim = _ica_decimation(raw_eeg)
  993. _stage(f"ica_fit:n={n_comp}:decim={decim}")
  994. ica.fit(raw_eeg, decim=decim, reject_by_annotation=True, verbose=False)
  995. inst_for_iclabel = _iclabel_instance(raw_eeg)
  996. labels = _label_ica_components(inst_for_iclabel, ica)
  997. if inst_for_iclabel is not raw_eeg:
  998. del inst_for_iclabel
  999. gc.collect()
  1000. eog_inds: list[int] = []
  1001. eog_chs = [
  1002. ch for ch, kind in zip(raw.ch_names, raw.get_channel_types()) if kind == "eog"
  1003. ]
  1004. if eog_chs:
  1005. try:
  1006. eog_inst = raw_eeg.copy()
  1007. _add_channels_by_name(eog_inst, raw, eog_chs)
  1008. eog_inds, _ = ica.find_bads_eog(eog_inst, verbose=False)
  1009. del eog_inst
  1010. except Exception:
  1011. eog_inds = []
  1012. return raw_eeg, ica, labels, list(eog_inds)
  1013. def run_asr_ica(
  1014. raw: mne.io.BaseRaw,
  1015. asr_thresh: float | int | bool | None = 20,
  1016. random_seed: int = 420,
  1017. n_comp: int | None = None,
  1018. remove_cfa_flag: bool = False,
  1019. ) -> tuple[mne.io.BaseRaw, ICA, dict[int, str]]:
  1020. raw_eeg, ica, labels, eog_inds = _fit_ica_and_labels(
  1021. raw, asr_thresh, random_seed, n_comp
  1022. )
  1023. keep_labels = {"brain", "other"}
  1024. if not remove_cfa_flag:
  1025. keep_labels.add("heart beat")
  1026. bads = [
  1027. index
  1028. for index, label in enumerate(labels)
  1029. if _normalise_iclabel_label(label) not in keep_labels
  1030. ]
  1031. bads = sorted(set(bads) | set(eog_inds))
  1032. bad_label_dict = {
  1033. index: ("eog" if index in eog_inds else labels[index])
  1034. for index in bads
  1035. if index < len(labels)
  1036. }
  1037. logging.info(
  1038. "ICLabel-based ICA pruning (%s cardiac): %s",
  1039. "removing" if remove_cfa_flag else "keeping",
  1040. bad_label_dict,
  1041. )
  1042. eeg_clean = _apply_ica_excludes(raw_eeg, ica, bads)
  1043. if not ica.exclude:
  1044. bad_label_dict = {}
  1045. if eeg_clean.info.get("bads"):
  1046. eeg_clean.interpolate_bads(reset_bads=True)
  1047. _add_non_eeg_channels(eeg_clean, raw)
  1048. return eeg_clean, ica, bad_label_dict
  1049. def run_asr_only(
  1050. raw: mne.io.BaseRaw,
  1051. asr_thresh: float | int | bool | None = 20,
  1052. ) -> mne.io.BaseRaw:
  1053. """Apply ASR without ICA/ICLabel, preserving non-EEG channels."""
  1054. if _normalise_asr_cutoff(asr_thresh) is None:
  1055. return raw
  1056. raw_eeg = raw.copy().pick("eeg")
  1057. raw_eeg, _ = mne.set_eeg_reference(raw_eeg, ref_channels="average")
  1058. eeg_clean = _apply_asr(raw_eeg, asr_thresh)
  1059. _add_non_eeg_channels(eeg_clean, raw)
  1060. return eeg_clean
  1061. def detect_r_peaks(
  1062. raw: mne.io.BaseRaw,
  1063. stim_channel: str = "STI 014",
  1064. gap_threshold_factor: float = 2.0,
  1065. ):
  1066. sfreq = float(raw.info["sfreq"])
  1067. ecg_chs = [
  1068. ch for ch, kind in zip(raw.ch_names, raw.get_channel_types()) if kind == "ecg"
  1069. ]
  1070. if not ecg_chs:
  1071. return raw, []
  1072. if stim_channel in raw.ch_names:
  1073. logging.warning(
  1074. "Stim channel %s already exists; leaving it unchanged and skipping ECG events.",
  1075. stim_channel,
  1076. )
  1077. return raw, []
  1078. _stage("ecg_r_peaks")
  1079. ecg = raw.get_data(picks=ecg_chs[0])[0]
  1080. ecg_cleaned = nk.ecg_clean(ecg, sampling_rate=sfreq)
  1081. _signals, info = nk.ecg_peaks(ecg_cleaned, sampling_rate=sfreq)
  1082. peaks = np.asarray(info.get("ECG_R_Peaks", []), dtype=int)
  1083. if peaks.size < 2:
  1084. return raw, []
  1085. rr_sec = np.diff(peaks) / sfreq
  1086. median_rr = float(np.median(rr_sec))
  1087. if not np.isfinite(median_rr) or median_rr <= 0:
  1088. return raw, []
  1089. max_gap = gap_threshold_factor * median_rr
  1090. all_peaks = list(peaks)
  1091. for idx, interval in enumerate(rr_sec):
  1092. if interval > max_gap:
  1093. n_missing = max(0, int(np.round(interval / median_rr)) - 1)
  1094. all_peaks.extend(
  1095. peaks[idx] + int(j * median_rr * sfreq)
  1096. for j in range(1, n_missing + 1)
  1097. )
  1098. all_peaks = np.unique(all_peaks).astype(int)
  1099. all_peaks = all_peaks[(all_peaks >= 0) & (all_peaks < raw.n_times)]
  1100. events = [(int(sample), 0, 1) for sample in all_peaks]
  1101. stim_data = np.zeros((1, raw.n_times), dtype=np.float64)
  1102. stim_data[0, all_peaks] = 1.0
  1103. info_stim = mne.create_info([stim_channel], sfreq, ch_types=["stim"])
  1104. stim_raw = mne.io.RawArray(stim_data, info_stim, verbose=False)
  1105. raw.add_channels([stim_raw], force_update_info=True)
  1106. return raw, events
  1107. def _read_raw_any(input_path: str, *, preload=True, verbose=False) -> mne.io.BaseRaw:
  1108. suffix = Path(input_path).suffix.lower()
  1109. if suffix == ".bdf":
  1110. return mne.io.read_raw_bdf(input_path, preload=preload, verbose=verbose)
  1111. if suffix == ".edf":
  1112. return mne.io.read_raw_edf(input_path, preload=preload, verbose=verbose)
  1113. if suffix == ".fif":
  1114. return mne.io.read_raw_fif(input_path, preload=preload, verbose=verbose)
  1115. if suffix == ".vhdr":
  1116. return mne.io.read_raw_brainvision(input_path, preload=preload, verbose=verbose)
  1117. raise ValueError(f"Unsupported input format: {input_path}")
  1118. def _part_fif_path(path: Path, kind: str) -> Path:
  1119. path = Path(path)
  1120. token = f".part-{os.getpid()}"
  1121. name = path.name
  1122. if kind == "raw":
  1123. if name.endswith("_raw.fif"):
  1124. name = name[:-8] + token + "_raw.fif"
  1125. else:
  1126. name = path.stem + token + "_raw.fif"
  1127. elif kind == "ica":
  1128. if name.endswith("_ica.fif"):
  1129. name = name[:-8] + token + "_ica.fif"
  1130. elif name.endswith("-ica.fif"):
  1131. name = name[:-8] + token + "-ica.fif"
  1132. else:
  1133. name = path.stem + token + "_ica.fif"
  1134. else:
  1135. raise ValueError(f"Unknown FIF kind: {kind}")
  1136. return path.with_name(name)
  1137. def _atomic_save_raw(raw: mne.io.BaseRaw, path: str | Path) -> Path:
  1138. final = Path(path)
  1139. final.parent.mkdir(parents=True, exist_ok=True)
  1140. temporary = _part_fif_path(final, "raw")
  1141. temporary.unlink(missing_ok=True)
  1142. _stage(f"save_raw:{final.name}")
  1143. try:
  1144. raw.save(str(temporary), overwrite=True, verbose=False)
  1145. os.replace(temporary, final)
  1146. finally:
  1147. temporary.unlink(missing_ok=True)
  1148. return final
  1149. def _atomic_save_ica(ica: ICA, path: str | Path) -> Path:
  1150. final = Path(path)
  1151. final.parent.mkdir(parents=True, exist_ok=True)
  1152. temporary = _part_fif_path(final, "ica")
  1153. temporary.unlink(missing_ok=True)
  1154. _stage(f"save_ica:{final.name}")
  1155. try:
  1156. ica.save(str(temporary), overwrite=True, verbose=False)
  1157. os.replace(temporary, final)
  1158. finally:
  1159. temporary.unlink(missing_ok=True)
  1160. return final
  1161. def _pyprep_segment_bounds(input_path: str, raw: mne.io.BaseRaw):
  1162. if _CFG_RT is None or not _CFG_RT.pyprep_by_segment:
  1163. return None
  1164. return _snip_sample_bounds_for_input(input_path, raw)
  1165. def preprocess_edf(
  1166. input_path: str,
  1167. output_path: str | None = None,
  1168. redo: bool = False,
  1169. pyprep_dict: dict | None = None,
  1170. asr_threshold=None,
  1171. random_seed: int = 42,
  1172. n_comp: int | None = None,
  1173. logging_path: str | None = None,
  1174. remove_cfa_override: bool | None = None,
  1175. flip_ecg: bool = False,
  1176. ecg_channel: str | None = None,
  1177. stim_keep: list | None = None,
  1178. spirometry_channel: str | None = None,
  1179. spirometry_dir: str | Path | None = None,
  1180. ):
  1181. if pyprep_dict is None:
  1182. pyprep_dict = dict(prep_params)
  1183. if asr_threshold is None:
  1184. asr_threshold = (
  1185. getattr(_CFG_RT, "asr_cutoff", getattr(_CFG_RT, "use_asr", None))
  1186. if _CFG_RT is not None
  1187. else globals().get("do_asr")
  1188. )
  1189. if output_path is None:
  1190. base = Path(input_path).stem + "_pp_raw.fif"
  1191. output_path = str(Path(output_dir) / base)
  1192. if logging_path is None or str(logging_path).strip() == "":
  1193. logging_path = str(Path(output_dir) / "logs" / "preprocessing.tsv")
  1194. outp = Path(output_path)
  1195. if not redo and outp.exists():
  1196. try:
  1197. raw_cached = mne.io.read_raw_fif(outp, preload=True, verbose=False)
  1198. except Exception as exc:
  1199. logging.warning("Existing output is unreadable and will be replaced: %s", exc)
  1200. outp.unlink(missing_ok=True)
  1201. else:
  1202. logging.info("Output exists and is readable, skipping: %s", outp)
  1203. had_rsp = _has_spirometry_annotations(raw_cached)
  1204. spirometry_rec = save_spirometry_analysis(
  1205. raw_cached,
  1206. spirometry_channel,
  1207. spirometry_dir or (Path(output_dir) / "spirometry"),
  1208. _preproc_base_from_output(outp),
  1209. redo=False,
  1210. )
  1211. if spirometry_rec.get("spirometry_status") == "ok" and not had_rsp:
  1212. _atomic_save_raw(raw_cached, outp)
  1213. rec = {
  1214. "utc": _now_utc_iso(),
  1215. "status": "OK",
  1216. "stage": "cached",
  1217. "input": input_path,
  1218. "output": str(outp),
  1219. "n_channels": len(raw_cached.ch_names),
  1220. "remove_cfa": str(remove_cfa_override),
  1221. "remove_cfa_mode": str(getattr(_CFG_RT, "remove_cfa_mode", "")),
  1222. "flip_ecg": str(bool(flip_ecg)),
  1223. }
  1224. rec.update(spirometry_rec)
  1225. _append_preproc_tsv(logging_path, rec)
  1226. return raw_cached
  1227. try:
  1228. _stage("read_raw")
  1229. raw = _read_raw_any(input_path, preload=True, verbose=False)
  1230. aux_like = ["IBI", "BURSTS", "SUPPR", "T1", "T2", "26", "27", "28", "29", "30"]
  1231. present_aux = [ch for ch in aux_like if ch in raw.ch_names]
  1232. if present_aux:
  1233. raw.set_channel_types({ch: "misc" for ch in present_aux})
  1234. _mark_spirometry_channel(raw, spirometry_channel)
  1235. _mark_ecg_channels(raw, ecg_channel)
  1236. if flip_ecg:
  1237. _flip_ecg_channels(raw, ecg_channel)
  1238. if target_sfreq and not np.isclose(raw.info["sfreq"], float(target_sfreq)):
  1239. _stage(f"resample:{raw.info['sfreq']}->{target_sfreq}")
  1240. raw.resample(float(target_sfreq), npad="auto", n_jobs=1)
  1241. if stim_keep:
  1242. for ch in stim_keep:
  1243. if ch in raw.ch_names:
  1244. raw.set_channel_types({ch: "stim"})
  1245. _mark_ecg_channels(raw, ecg_channel)
  1246. _stage("montage")
  1247. raw = standardise_and_montage(raw)
  1248. if sum(kind == "eeg" for kind in raw.get_channel_types()) == 0:
  1249. raise ValueError(
  1250. "No EEG channels remain after montage/drop; check montage name and channel labels."
  1251. )
  1252. do_pyprep_flag = bool(
  1253. getattr(_CFG_RT, "use_pyprep", globals().get("do_pyprep", True))
  1254. )
  1255. if do_pyprep_flag:
  1256. raw = run_pyprep(
  1257. raw,
  1258. pyprep_dict,
  1259. random_seed=random_seed,
  1260. ransac=bool(
  1261. getattr(_CFG_RT, "prep_ransac", pyprep_dict.get("ransac", False))
  1262. ),
  1263. channel_wise=bool(getattr(_CFG_RT, "pyprep_channel_wise", False)),
  1264. max_chunk_size=getattr(_CFG_RT, "pyprep_max_chunk_size", 8),
  1265. segment_bounds=_pyprep_segment_bounds(input_path, raw),
  1266. )
  1267. # Filtering is independent of PyPREP. In the previous implementation,
  1268. # disabling PyPREP also silently disabled the configured band-pass.
  1269. raw = _filter_eeg_once(
  1270. raw,
  1271. pyprep_dict.get("l_freq", getattr(_CFG_RT, "high_pass", 1.0)),
  1272. pyprep_dict.get("h_freq", getattr(_CFG_RT, "low_pass", 100.0)),
  1273. )
  1274. remove_flag = (
  1275. bool(remove_cfa)
  1276. if remove_cfa_override is None
  1277. else bool(remove_cfa_override)
  1278. )
  1279. use_ica_flag = bool(getattr(_CFG_RT, "use_ica", True))
  1280. if use_ica_flag:
  1281. raw, ica_model, bad_label_dict = run_asr_ica(
  1282. raw,
  1283. asr_thresh=asr_threshold,
  1284. random_seed=random_seed,
  1285. n_comp=n_comp,
  1286. remove_cfa_flag=remove_flag,
  1287. )
  1288. else:
  1289. raw = run_asr_only(raw, asr_threshold)
  1290. ica_model = None
  1291. bad_label_dict = {}
  1292. raw, events = detect_r_peaks(raw)
  1293. _stage("spirometry")
  1294. spirometry_rec = save_spirometry_analysis(
  1295. raw,
  1296. spirometry_channel,
  1297. spirometry_dir or (Path(output_dir) / "spirometry"),
  1298. _preproc_base_from_output(outp),
  1299. redo=redo,
  1300. )
  1301. raw.info["bads"] = []
  1302. raw.set_meas_date(1)
  1303. _atomic_save_raw(raw, outp)
  1304. ica_path = ""
  1305. if ica_model is not None:
  1306. stem_no_pp = outp.stem.replace("_pp_raw", "")
  1307. ica_path_obj = outp.with_name(stem_no_pp + "_ica.fif")
  1308. _atomic_save_ica(ica_model, ica_path_obj)
  1309. ica_path = str(ica_path_obj)
  1310. rec = {
  1311. "utc": _now_utc_iso(),
  1312. "status": "OK",
  1313. "stage": "save",
  1314. "input": input_path,
  1315. "output": str(outp),
  1316. "n_channels": len(raw.ch_names),
  1317. "n_events": len(events),
  1318. "bad_channels": ";".join(raw.info.get("bads", [])),
  1319. "remove_cfa": str(remove_flag),
  1320. "remove_cfa_mode": str(getattr(_CFG_RT, "remove_cfa_mode", "")),
  1321. "flip_ecg": str(bool(flip_ecg)),
  1322. "asr_threshold": str(_normalise_asr_cutoff(asr_threshold) or ""),
  1323. "n_comp": str(n_comp or ""),
  1324. "stim_keep": ";".join(stim_keep) if stim_keep else "",
  1325. "montage_name": str(getattr(_CFG_RT, "montage_name", "auto")),
  1326. "ica_path": ica_path,
  1327. "ica_bads": ";".join(str(key) for key in sorted(bad_label_dict)),
  1328. "ica_bad_labels": ";".join(
  1329. str(bad_label_dict[key]) for key in sorted(bad_label_dict)
  1330. ),
  1331. }
  1332. rec.update(spirometry_rec)
  1333. _append_preproc_tsv(logging_path, rec)
  1334. return raw
  1335. except Exception as exc:
  1336. _append_preproc_tsv(
  1337. logging_path,
  1338. {
  1339. "utc": _now_utc_iso(),
  1340. "status": "FAIL",
  1341. "stage": "preprocess_edf",
  1342. "input": input_path,
  1343. "output": str(outp),
  1344. "error_type": type(exc).__name__,
  1345. "error": str(exc),
  1346. },
  1347. )
  1348. raise
  1349. def _fit_asr_ica_iclabel_once(
  1350. raw: mne.io.BaseRaw,
  1351. asr_thresh: float | int | bool | None = 20,
  1352. random_seed: int = 420,
  1353. n_comp: int | None = None,
  1354. ):
  1355. """Run optional ASR, fit ICA once, and derive both ICLabel rules."""
  1356. raw_eeg, ica, labels, eog_inds = _fit_ica_and_labels(
  1357. raw, asr_thresh, random_seed, n_comp
  1358. )
  1359. labels = [_normalise_iclabel_label(label) for label in labels]
  1360. bads_remove = [
  1361. index for index, label in enumerate(labels) if label not in {"brain", "other"}
  1362. ]
  1363. bads_keep = [
  1364. index
  1365. for index, label in enumerate(labels)
  1366. if label not in {"brain", "heart beat", "other"}
  1367. ]
  1368. bads_remove = sorted(set(bads_remove) | set(eog_inds))
  1369. bads_keep = sorted(set(bads_keep) | set(eog_inds))
  1370. bad_map_remove = {
  1371. index: ("eog" if index in eog_inds else labels[index])
  1372. for index in bads_remove
  1373. if index < len(labels)
  1374. }
  1375. bad_map_keep = {
  1376. index: ("eog" if index in eog_inds else labels[index])
  1377. for index in bads_keep
  1378. if index < len(labels)
  1379. }
  1380. return (
  1381. raw_eeg,
  1382. ica,
  1383. labels,
  1384. bads_remove,
  1385. bads_keep,
  1386. bad_map_remove,
  1387. bad_map_keep,
  1388. eog_inds,
  1389. )
  1390. def preprocess_edf_both(
  1391. input_path: str,
  1392. output_path_remove: str,
  1393. output_path_keep: str,
  1394. redo: bool = False,
  1395. pyprep_dict: dict | None = None,
  1396. asr_threshold=None,
  1397. random_seed: int = 42,
  1398. n_comp: int | None = None,
  1399. logging_path: str | None = None,
  1400. flip_ecg: bool = False,
  1401. ecg_channel: str | None = None,
  1402. stim_keep: list | None = None,
  1403. spirometry_channel: str | None = None,
  1404. spirometry_dir: str | Path | None = None,
  1405. ):
  1406. """Preprocess once, then save cardiac-removed and cardiac-retained variants."""
  1407. if logging_path is None or str(logging_path).strip() == "":
  1408. logging_path = str(Path(output_dir) / "logs" / "preprocessing.tsv")
  1409. if pyprep_dict is None:
  1410. pyprep_dict = dict(prep_params)
  1411. if asr_threshold is None:
  1412. asr_threshold = (
  1413. getattr(_CFG_RT, "asr_cutoff", getattr(_CFG_RT, "use_asr", None))
  1414. if _CFG_RT is not None
  1415. else globals().get("do_asr")
  1416. )
  1417. output_remove = Path(output_path_remove)
  1418. output_keep = Path(output_path_keep)
  1419. if not redo and output_remove.exists() and output_keep.exists():
  1420. readable = True
  1421. for out_path in (output_remove, output_keep):
  1422. try:
  1423. cached = mne.io.read_raw_fif(out_path, preload=True, verbose=False)
  1424. except Exception as exc:
  1425. logging.warning("Unreadable cached output %s: %s", out_path, exc)
  1426. out_path.unlink(missing_ok=True)
  1427. readable = False
  1428. continue
  1429. had_rsp = _has_spirometry_annotations(cached)
  1430. spirometry_rec = save_spirometry_analysis(
  1431. cached,
  1432. spirometry_channel,
  1433. spirometry_dir or (Path(output_dir) / "spirometry"),
  1434. _preproc_base_from_output(out_path),
  1435. redo=False,
  1436. )
  1437. if spirometry_rec.get("spirometry_status") == "ok" and not had_rsp:
  1438. _atomic_save_raw(cached, out_path)
  1439. rec = {
  1440. "utc": _now_utc_iso(),
  1441. "status": "OK",
  1442. "stage": "cached",
  1443. "input": input_path,
  1444. "output": str(out_path),
  1445. "n_channels": len(cached.ch_names),
  1446. "remove_cfa_mode": "both",
  1447. "flip_ecg": str(bool(flip_ecg)),
  1448. }
  1449. rec.update(spirometry_rec)
  1450. _append_preproc_tsv(logging_path, rec)
  1451. del cached
  1452. if readable:
  1453. return (
  1454. mne.io.read_raw_fif(output_remove, preload=False, verbose=False),
  1455. mne.io.read_raw_fif(output_keep, preload=False, verbose=False),
  1456. )
  1457. try:
  1458. _stage("read_raw")
  1459. raw = _read_raw_any(input_path, preload=True, verbose=False)
  1460. aux_like = ["IBI", "BURSTS", "SUPPR", "T1", "T2", "26", "27", "28", "29", "30"]
  1461. present_aux = [ch for ch in aux_like if ch in raw.ch_names]
  1462. if present_aux:
  1463. raw.set_channel_types({ch: "misc" for ch in present_aux})
  1464. _mark_spirometry_channel(raw, spirometry_channel)
  1465. _mark_ecg_channels(raw, ecg_channel)
  1466. if flip_ecg:
  1467. _flip_ecg_channels(raw, ecg_channel)
  1468. if target_sfreq and not np.isclose(raw.info["sfreq"], float(target_sfreq)):
  1469. _stage(f"resample:{raw.info['sfreq']}->{target_sfreq}")
  1470. raw.resample(float(target_sfreq), npad="auto", n_jobs=1)
  1471. if stim_keep:
  1472. for ch in stim_keep:
  1473. if ch in raw.ch_names:
  1474. raw.set_channel_types({ch: "stim"})
  1475. _mark_ecg_channels(raw, ecg_channel)
  1476. _stage("montage")
  1477. raw = standardise_and_montage(raw)
  1478. if sum(kind == "eeg" for kind in raw.get_channel_types()) == 0:
  1479. raise ValueError(
  1480. "No EEG channels remain after montage/drop; check montage name and channel labels."
  1481. )
  1482. if bool(getattr(_CFG_RT, "use_pyprep", globals().get("do_pyprep", True))):
  1483. raw = run_pyprep(
  1484. raw,
  1485. pyprep_dict,
  1486. random_seed=random_seed,
  1487. ransac=bool(
  1488. getattr(_CFG_RT, "prep_ransac", pyprep_dict.get("ransac", False))
  1489. ),
  1490. channel_wise=bool(getattr(_CFG_RT, "pyprep_channel_wise", False)),
  1491. max_chunk_size=getattr(_CFG_RT, "pyprep_max_chunk_size", 8),
  1492. segment_bounds=_pyprep_segment_bounds(input_path, raw),
  1493. )
  1494. raw = _filter_eeg_once(
  1495. raw,
  1496. pyprep_dict.get("l_freq", getattr(_CFG_RT, "high_pass", 1.0)),
  1497. pyprep_dict.get("h_freq", getattr(_CFG_RT, "low_pass", 100.0)),
  1498. )
  1499. # ECG and respiration depend only on auxiliary channels, so calculate
  1500. # them once rather than once for each ICA exclusion rule.
  1501. raw, events = detect_r_peaks(raw)
  1502. _stage("spirometry")
  1503. spirometry_rec = save_spirometry_analysis(
  1504. raw,
  1505. spirometry_channel,
  1506. spirometry_dir or (Path(output_dir) / "spirometry"),
  1507. _preproc_base_from_output(output_remove),
  1508. redo=redo,
  1509. )
  1510. use_ica_flag = bool(getattr(_CFG_RT, "use_ica", True))
  1511. if use_ica_flag:
  1512. (
  1513. raw_eeg_proc,
  1514. ica,
  1515. _labels,
  1516. bads_remove,
  1517. bads_keep,
  1518. bad_map_remove,
  1519. bad_map_keep,
  1520. _eog_inds,
  1521. ) = _fit_asr_ica_iclabel_once(
  1522. raw,
  1523. asr_thresh=asr_threshold,
  1524. random_seed=random_seed,
  1525. n_comp=n_comp,
  1526. )
  1527. variants = (
  1528. (output_remove, "remove", bads_remove, bad_map_remove),
  1529. (output_keep, "keep", bads_keep, bad_map_keep),
  1530. )
  1531. for outp, mode, excludes, label_map in variants:
  1532. ica_local = ica.copy()
  1533. raw_out = _apply_ica_excludes(raw_eeg_proc, ica_local, list(excludes))
  1534. if not ica_local.exclude:
  1535. label_map = {}
  1536. if raw_out.info.get("bads"):
  1537. raw_out.interpolate_bads(reset_bads=True)
  1538. _add_non_eeg_channels(raw_out, raw)
  1539. raw_out.info["bads"] = []
  1540. raw_out.set_meas_date(1)
  1541. _atomic_save_raw(raw_out, outp)
  1542. stem_no_pp = outp.stem.replace("_pp_raw", "")
  1543. ica_path = outp.with_name(stem_no_pp + "_ica.fif")
  1544. _atomic_save_ica(ica_local, ica_path)
  1545. rec = {
  1546. "utc": _now_utc_iso(),
  1547. "status": "OK",
  1548. "stage": "save",
  1549. "input": input_path,
  1550. "output": str(outp),
  1551. "n_channels": len(raw_out.ch_names),
  1552. "n_events": len(events),
  1553. "bad_channels": "",
  1554. "remove_cfa": str(mode == "remove"),
  1555. "remove_cfa_mode": "both",
  1556. "flip_ecg": str(bool(flip_ecg)),
  1557. "asr_threshold": str(_normalise_asr_cutoff(asr_threshold) or ""),
  1558. "n_comp": str(n_comp or ""),
  1559. "stim_keep": ";".join(stim_keep) if stim_keep else "",
  1560. "montage_name": str(getattr(_CFG_RT, "montage_name", "auto")),
  1561. "ica_path": str(ica_path),
  1562. "ica_bads": ";".join(str(key) for key in sorted(label_map)),
  1563. "ica_bad_labels": ";".join(
  1564. str(label_map[key]) for key in sorted(label_map)
  1565. ),
  1566. }
  1567. rec.update(spirometry_rec)
  1568. _append_preproc_tsv(logging_path, rec)
  1569. del raw_out, ica_local
  1570. gc.collect()
  1571. else:
  1572. processed = run_asr_only(raw, asr_threshold)
  1573. processed.info["bads"] = []
  1574. processed.set_meas_date(1)
  1575. for outp, mode in ((output_remove, "remove"), (output_keep, "keep")):
  1576. _atomic_save_raw(processed, outp)
  1577. rec = {
  1578. "utc": _now_utc_iso(),
  1579. "status": "OK",
  1580. "stage": "save_no_ica",
  1581. "input": input_path,
  1582. "output": str(outp),
  1583. "n_channels": len(processed.ch_names),
  1584. "n_events": len(events),
  1585. "bad_channels": "",
  1586. "remove_cfa": str(mode == "remove"),
  1587. "remove_cfa_mode": "both",
  1588. "flip_ecg": str(bool(flip_ecg)),
  1589. "asr_threshold": str(_normalise_asr_cutoff(asr_threshold) or ""),
  1590. "n_comp": "",
  1591. "stim_keep": ";".join(stim_keep) if stim_keep else "",
  1592. "montage_name": str(getattr(_CFG_RT, "montage_name", "auto")),
  1593. }
  1594. rec.update(spirometry_rec)
  1595. _append_preproc_tsv(logging_path, rec)
  1596. return (
  1597. mne.io.read_raw_fif(output_remove, preload=False, verbose=False),
  1598. mne.io.read_raw_fif(output_keep, preload=False, verbose=False),
  1599. )
  1600. except Exception as exc:
  1601. logging.exception("preprocess_edf_both failed for %s", input_path)
  1602. _append_preproc_tsv(
  1603. logging_path,
  1604. {
  1605. "utc": _now_utc_iso(),
  1606. "status": "FAIL",
  1607. "stage": "preprocess_edf_both",
  1608. "input": input_path,
  1609. "output": f"{output_remove} | {output_keep}",
  1610. "error_type": type(exc).__name__,
  1611. "error": str(exc),
  1612. },
  1613. )
  1614. raise
  1615. # ----------------------------------- CLI (optional) ----------------------- #
  1616. if __name__ == "__main__":
  1617. import argparse
  1618. parser = argparse.ArgumentParser(description="EDF→FIF EEG preprocessing")
  1619. parser.add_argument("edf", help="Path to input .edf file")
  1620. parser.add_argument("--out", help="Output .fif path (default: output_dir/<name>_pp_raw.fif)")
  1621. parser.add_argument("--redo", action="store_true", help="Overwrite existing outputs")
  1622. parser.add_argument("--n_comp", type=int, default=None, help="ICA components (default: auto)")
  1623. parser.add_argument("--asr", type=float, default=None, help="ASR cutoff (default: config.do_asr)")
  1624. args = parser.parse_args()
  1625. logging.basicConfig(level=logging.INFO)
  1626. preprocess_edf(
  1627. args.edf,
  1628. output_path=args.out,
  1629. redo=args.redo,
  1630. n_comp=args.n_comp,
  1631. asr_threshold=args.asr,
  1632. )

_preprocessing.py at commit 96d3790, under MIT · at the source

Overview

Authors: Rohan Kandasamy1,2,3, Samia Elkommos4,5, Ineke A van Rossum6, David Martin-Lopez5, Akihiro Koreki2,7, Fiona Farrell8, Suzanne O’Sullivan1,8, Beate Diehl1,2,8, Fahmida A Chowdhury1,2,8, Hugo Critchley9,10, Matthew C Walker2,8, Sarah Garfinkel3, Mahinda Yogarajah2,8
  1. Department of Clinical Neurophysiology, National Hospital of Neurology and Neurosurgery, London WC1N 3BG, UK
  2. UCL Queen Square Institute of Neurology, London WC1N 3BG, UK
  3. UCL Institute of Cognitive Neuroscience, London WC1N 3AR, UK
  4. School of Neuroscience, King’s College, London WC2R 2LS, UK
  5. Atkinson Morley Regional Neurosciences Centre, St George’s University Hospitals, London SW17 0QT, UK
  6. Leiden University Medical Centre, Leiden 2333 ZA, The Netherlands
  7. Neuroscience Research Centre, St George’s University of London, London SW17 0RE, UK
  8. National Hospital of Neurology and Neurosurgery Epilepsy Group, London WC1N 3BG, UK
  9. Department of Clinical Neuroscience, Brighton and Sussex Medical School, University of Sussex, Brighton BN1 9PX, UK
  10. Sussex Partnership NHS Foundation Trust, Sussex BN13 3EP, UK
Journal: Brain communications, volume 8, issue 2, article fcag120
Dates: received 14 August 2025; accepted 1 April 2026; published online 2 April 2026
Type: Research article · Language: English
License: CC BY
Identifiers: DOI 10.1093/braincomms/fcag120 · PMID 41994620 · PMCID PMC13080701 · OpenAlex W7148454864
Open access: gold, a free copy (OpenAlex)
Status: code verified
Categories: EEG (modality), human (organism), epilepsy (population)
Methods: Connectivity, Statistics, Preprocessing, Evoked potentials, Physiology & signal measures
Keywords: interoception, dissociative seizures, non-epileptic attack
Topic: Psychosomatic Disorders and Their Treatments (Psychiatry and Mental health, Medicine), according to OpenAlex
Funding: Medical Research Council; Association of British Neurology; Patrick Berthoud Trust Clinical Research; NIH; Care Research University College London Hospitals Biomedical Research Centre
Citations: cited by 4 papers (Europe PMC); 68 references in the paper

Abstract

Functional seizures (FSs) are common, but distinguishing FS from epileptic seizures (ESs) can be challenging, and the pathophysiology is not well-understood. The heartbeat evoked potential (HEP) reflects the central processing of cardiac signals and bodily attention. Our group previously demonstrated HEP differences between FS and ES. Here, we sought to replicate these HEP findings in an independent retrospective sample and observe effects of semiology. Because we lacked symptom reporting at the time of a seizure, in the second part of the study we examined whether HEP modulation was associated with real-time bodily symptom reporting in a second retrospective sample of individuals with functional or vasovagal syncope where symptom data was available. In the first part, we identified FS (n = 57) or ES (n = 31) from video telemetry with EEG recordings of patients referred for assessment of their events. We categorized FS and ES into ‘motile’ or ‘non-motile’ according to semiology with predominantly positive motor features, or with subjective sensory or negative motor features, respectively. HEP amplitude was calculated by averaging EEG segments time-locked to ECG R-waves, correcting for pre-R wave baseline, to quantify the average voltage between 0.455 and 0.595 s after the R wave. We compared HEP amplitude at baseline, preictal and postictal periods between FS and ES of equivalent semiology. In the second part, we measured HEP amplitude in functional syncope or vasovagal syncope (30 participants per group), from EEG recorded during head-up tilt procedure. We compared the HEP amplitude around the time of symptom reporting to its baseline value. HEP amplitude distinguished FS from ES with matched semiology: In non-motile FS, HEP become more positive at the scalp from the interictal to preictal period, whereas in motile FS, the HEP became less positive at the scalp. ES were not associated with significant changes in HEP. In functional syncope, a more positive HEP amplitude was associated with reported bodily symptoms, but not for psychological or emotional symptoms. In vasovagal syncope, a less positive HEP was associated with bodily symptoms. These findings indicate that FS semiology may relate to patterns of bodily attention, as reflected by HEP amplitude change. Non-motile FS were preceded by increased HEP amplitude, and the opposite was seen in motile FS. The increased HEP amplitude associated with bodily symptom reporting in functional syncope supports a role for the HEP in tracking interoception and bodily attention. HEP may therefore help us understand interoceptive mechanisms underlying FS.

Reproduced under the paper's license (CC BY), from the paper cited above.

Repository

Its files are read in the Code ↔ Paper reader above.

ClinicalAffectiveNeuroscienceLab/HEPPy

License: MIT
State: the link answers, verified on 28 September 2026
Evidence: files inventoried
Commit: 96d379013ebb38eafd4871b6b335a9dd8333964a, 14 July 2026
Languages: Python (4)
Size: 16 files, 4 scripts
Software Heritage: not archived
Found in: “Data availability”
Holds: README, license file, environment (requirements.txt)
Not found: CITATION.cff, tests, continuous integration, documentation
Tools: NumPy (4 files), pandas (4 files), Matplotlib (3 files), MNE-Python (3 files), NeuroKit2 (3 files), ICLabel (1 file), Plotly (1 file), PyPREP (1 file), SciPy (1 file)
Availability: 1 check, the latest on 28 September 2026: the link answers
  • 28 September 2026: the link answers
6 files

The paper's code and data availability statement is in the Data section.

Tracing map

Proposed by the machine: these links were found in the paper and verified at the source, without human review. The map will receive a Zenodo DOI once one of the paper's authors has validated it with their ORCID.

What the map holds:

  • 1 repository of the authors' code, each at its verified commit, with its license and how the link was found in the paper;
  • 4 scripts, each with its path and the digest of its content;
  • no match between paragraphs and code yet;
  • neither the text of the paper nor the code itself.

Its JSON (tracing-map.json) is deposited on Zenodo with its DOI once the map is validated.

Data

No dataset and no data link were found in the paper.

Data availability

Data can be made available on request. The code used to analyse the EEG is now prepared as a repository which can be found at: https://github.com/ClinicalAffectiveNeuroscienceLab/HEPPy.

Reproduced under the paper's license (CC BY), from the paper cited above.

Versions

The history of this record: each version stored by the harvester or made by a correction of its authors or of the maintainers of its code, and what changed in its facts. The texts of the paper (its abstract, its availability statements) are not part of it; versions that changed only those are not listed.

Version 1, 28 September 2026: the first record

Recorded: type, language, journal, volume, issue, pages, dates, 13 authors, 3 keywords, 5 funders, 61 references.

Cite

This paper

Kandasamy, R., Elkommos, S., van Rossum, I. A., Martin-Lopez, D., Koreki, A., Farrell, F., O’Sullivan, S., Diehl, B., Chowdhury, F. A., Critchley, H., Walker, M. C., Garfinkel, S., & Yogarajah, M. (2026). The heartbeat evoked potential and the prediction of functional seizure semiology. Brain communications, 8(2), fcag120. https://doi.org/10.1093/braincomms/fcag120

BibTeX

@article{kandasamy2026heartbeat,
author = {Kandasamy, Rohan and Elkommos, Samia and van Rossum, Ineke A and Martin-Lopez, David and Koreki, Akihiro and Farrell, Fiona and O’Sullivan, Suzanne and Diehl, Beate and Chowdhury, Fahmida A and Critchley, Hugo and Walker, Matthew C and Garfinkel, Sarah and Yogarajah, Mahinda},
title = {{The heartbeat evoked potential and the prediction of functional seizure semiology}},
journal = {Brain communications},
year = {2026},
month = apr,
volume = {8},
number = {2},
pages = {fcag120},
publisher = {Oxford University Press},
issn = {2632-1297},
doi = {10.1093/braincomms/fcag120},
url = {https://doi.org/10.1093/braincomms/fcag120},
pmid = {41994620},
pmcid = {PMC13080701}
}

RIS

TY - JOUR
AU - Kandasamy, Rohan
AU - Elkommos, Samia
AU - van Rossum, Ineke A
AU - Martin-Lopez, David
AU - Koreki, Akihiro
AU - Farrell, Fiona
AU - O’Sullivan, Suzanne
AU - Diehl, Beate
AU - Chowdhury, Fahmida A
AU - Critchley, Hugo
AU - Walker, Matthew C
AU - Garfinkel, Sarah
AU - Yogarajah, Mahinda
TI - The heartbeat evoked potential and the prediction of functional seizure semiology
T2 - Brain communications
J2 - Brain Commun
PY - 2026
DA - 2026/04/02
VL - 8
IS - 2
SP - fcag120
SN - 2632-1297
PB - Oxford University Press
DO - 10.1093/braincomms/fcag120
UR - https://doi.org/10.1093/braincomms/fcag120
LA - en
ER -

CSL-JSON

{
"id": "10.1093/braincomms/fcag120",
"type": "article-journal",
"title": "The heartbeat evoked potential and the prediction of functional seizure semiology",
"container-title": "Brain communications",
"author": [
{
"family": "Kandasamy",
"given": "Rohan"
},
{
"family": "Elkommos",
"given": "Samia"
},
{
"family": "van Rossum",
"given": "Ineke A"
},
{
"family": "Martin-Lopez",
"given": "David"
},
{
"family": "Koreki",
"given": "Akihiro"
},
{
"family": "Farrell",
"given": "Fiona"
},
{
"family": "O’Sullivan",
"given": "Suzanne"
},
{
"family": "Diehl",
"given": "Beate"
},
{
"family": "Chowdhury",
"given": "Fahmida A"
},
{
"family": "Critchley",
"given": "Hugo"
},
{
"family": "Walker",
"given": "Matthew C"
},
{
"family": "Garfinkel",
"given": "Sarah"
},
{
"family": "Yogarajah",
"given": "Mahinda"
}
],
"container-title-short": "Brain Commun",
"volume": "8",
"issue": "2",
"page": "fcag120",
"DOI": "10.1093/braincomms/fcag120",
"PMID": "41994620",
"PMCID": "PMC13080701",
"ISSN": "2632-1297",
"publisher": "Oxford University Press",
"URL": "https://doi.org/10.1093/braincomms/fcag120",
"language": "en",
"issued": {
"date-parts": [
[
2026,
4,
2
]
]
}
}

The tracing map gets a citation of its own once an author has validated it and it has a DOI.

Similar papers

The papers with a page that share the most with this one: the tools found in their code, their categories, datasets, cited references and authors, the rarest counting most.

[1] doi:10.1111/psyp.70297 [code]
Heartbeat-Evoked Responses in M/EEG: A Systematic Review of Methods With Suggestions for Analysis and Reporting.
Journal: Psychophysiology
In common: pandas, NumPy, EEG, 13 references
[2] doi:10.1038/s41597-025-05174-7 [code]
A large-scale MEG and EEG dataset for object recognition in naturalistic scenes
Journal: n/a
In common: PyPREP, ICLabel, MNE-Python, 5 other tools, EEG
[3] doi:10.1162/imag.a.1319 [code]
When the inner clock fades: Interoceptive decline and consolidation of phase resetting in cortical rhythms by cardiac events underlie healthy lifespan aging.
Journal: Imaging neuroscience (Cambridge, Mass.)
In common: MNE-Python, SciPy, NumPy, 5 references
[4] doi:10.1093/cercor/bhag113 [code]
Long-term reliability and stability of parameterized resting state EEG: evidence from a five-year follow-up.
Journal: Cerebral cortex (New York, N.Y. : 1991)
In common: PyPREP, ICLabel, MNE-Python, 4 other tools, EEG
[5] doi:10.1097/j.pain.0000000000004044 [code]
No effect of rhythmic visual stimulation on experimental pain perception.
Journal: Pain
In common: PyPREP, ICLabel, MNE-Python, 4 other tools, EEG
[6] doi:10.1162/imag.a.1269 [code]
From early to contemporary normative modeling: Mapping individual differences in neurophysiological signals.
Journal: Imaging neuroscience (Cambridge, Mass.)
In common: ICLabel, MNE-Python, Plotly, 4 other tools, EEG
[7] doi:10.7554/elife.90013 [code]
Exposure to false cardiac feedback alters pain perception and anticipatory cardiac frequency.
Journal: eLife
In common: 5 references
[8] doi:10.1111/psyp.70301
Oscillatory Markers of Interoceptive Attention: Beta Suppression as a Neural Signature of Heartbeat Processing.
Journal: Psychophysiology
In common: 5 references
[9] doi:10.3758/s13428-026-02997-z [code]
PyLossless: A non-destructive EEG processing pipeline.
Journal: Behavior research methods
In common: ICLabel, MNE-Python, Plotly, 3 other tools, EEG
[10] doi:10.1038/s41597-026-07350-9 [code]
An open multi-center MEG-EEG dataset for studying conscious visual perception.
Journal: Scientific data
In common: PyPREP, MNE-Python, pandas, 3 other tools, EEG

Contribute

The authors of this paper can claim it, correct its record and validate its tracing map, and the maintainers of its code (its owner, or a public member of its organization) correct what it says of their repository; anyone signed in can ask for its removal. Every request goes to OSCR's own machine, which answers it; your account page follows them.

Sign in with ORCID to claim this paper as one of its authors, correct its record or validate its tracing map: when the paper's metadata lists your ORCID iD, you are recognized at once. Maintainers of its code: sign in with GitHub, then claim the repository on your account page.

Request its removal

To ask OSCR to remove this record, the copies of its authors' scripts or its tracing map, use the removal request page: signed in, you say who you are, what to remove and why, then review and confirm the request. Published rules decide every request (how).

Discussion, reproductions, activity

Discussion: questions and error reports about this paper and its code, from signed-in readers and its authors. It opens with sign-in.

Reproductions: reports from readers who ran the authors' code: what they reproduced, with which environment, commit and data. It opens with sign-in.

Activity: what happens around this paper: new versions of its record, its map's validation, discussions and reproductions. It opens with sign-in.