OSCR

Multimodal survival analysis of glioblastoma using whole-slide histopathology, gene expression, clinical variables and language-model-derived mutation features.

Code ↔ Paper

The paper beside its authors' code: matches between them have not been computed for this paper yet.

Paper

Loaded from Europe PMC by your browser, not stored by OSCR: doi.org · Europe PMC

The paper is loaded when this pane is shown.

The authors' code

Python · 237 lines · 8.1 KB · no license

  1. import torch
  2. import torch.nn as nn
  3. import torch.nn.functional as F
  5. ##IMAGE
  6. class ImgCNNWithAttention(nn.Module):
  7. def __init__(self,input_dim, num_classes):
  8. super(ImgCNNWithAttention, self).__init__()
  9. self.block1 = nn.Sequential(
  10. nn.Conv2d(input_dim, 16, kernel_size=5, stride=1),
  11. nn.MaxPool2d(kernel_size=5, stride=5, padding=0),
  12. nn.BatchNorm2d(16),
  13. nn.ReLU(),
  14. nn.Conv2d(16, 32, kernel_size=5, stride=1),
  15. nn.MaxPool2d(kernel_size=3, stride=3, padding=0),
  16. nn.BatchNorm2d(32),
  17. nn.ReLU(),
  18. nn.Conv2d(32, 64, kernel_size=5, stride=1),
  19. nn.MaxPool2d(kernel_size=3, stride=3, padding=0),
  20. nn.BatchNorm2d(64),
  21. nn.ReLU()
  22. )
  23. self.attention = nn.Sequential(
  24. nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
  25. nn.BatchNorm2d(64),
  26. nn.Sigmoid()
  27. )
  28. self.block2=nn.Sequential(
  29. nn.Conv2d(64, 128, kernel_size=5, stride=1),
  30. nn.MaxPool2d(kernel_size=3, stride=3, padding=0),
  31. nn.BatchNorm2d(128))
  32. self.block3=nn.Sequential(
  33. nn.Linear(128 * 20 * 20, 32),
  34. nn.BatchNorm1d(32),
  35. nn.ReLU(),
  36. nn.Dropout(0.3),
  37. nn.Linear(32, num_classes)
  38. )
  39. def forward(self, x):
  40. x = self.block1(x)
  41. attention = self.attention(x)
  42. x = x * attention
  43. x = self.block2(x)
  44. x1 = x.view(x.size(0), -1)
  45. x = self.block3(x1)
  46. return x,x1,attention
  49. ##SEQ
  50. class AddNormBlock1(nn.Module):
  51. def __init__(self, input_size, output_size, dropout_rate):
  52. super(AddNormBlock1, self).__init__()
  53. self.linear = nn.Linear(input_size, output_size)
  54. self.norm = nn.BatchNorm1d(output_size)
  55. self.dropout = nn.Dropout(dropout_rate)
  57. def forward(self, x):
  58. x=self.linear(x)
  59. y = F.relu(x)
  60. y = self.norm(y)
  61. y = self.dropout(y)
  62. # Add & Norm: Add input x to output y and apply layer normalization
  63. return y
  64. # return self.norm(x + y)
  66. class AddNormBlock(nn.Module):
  67. def __init__(self, input_size, output_size, dropout_rate):
  68. super(AddNormBlock, self).__init__()
  69. self.linear = nn.Linear(input_size, output_size)
  70. self.norm = nn.LayerNorm(output_size)
  71. self.dropout = nn.Dropout(dropout_rate)
  72. def forward(self, x):
  73. x=self.linear(x)
  74. y = F.relu(x)
  75. y = self.norm(y)
  76. # y = self.dropout(y)
  77. # Add & Norm: Add input x to output y and apply layer normalization
  78. return y
  79. # return self.norm(x + y)
  80. class DeepSeq(nn.Module):
  81. def __init__(self, hidden_sizes_phi=None, hidden_sizes_rho=None,num_intervals=None, dropout_rate=0.3):
  82. super(DeepSeq, self).__init__()
  83. if hidden_sizes_phi is None:
  84. hidden_sizes_phi = [768, 512, 256, 128, 64]
  85. if hidden_sizes_rho is None:
  86. hidden_sizes_rho = [64, 32, 16]
  87. if num_intervals is None:
  88. num_intervals = 1
  89. # Phi function: Transform each element of the set
  90. self.blocks1 = self.create_blocks(hidden_sizes_phi, dropout_rate=0.5)
  91. self.blocks2 = self.create_blocks(hidden_sizes_rho, dropout_rate=0.2)
  92. # Attention mechanism
  93. self.attention = nn.Sequential(
  94. nn.Linear(hidden_sizes_phi[-1], hidden_sizes_phi[-1]),
  95. nn.ReLU(),
  96. nn.Dropout(dropout_rate),
  97. nn.Linear(hidden_sizes_phi[-1], 1),
  98. )
  99. self.blocks1_ = nn.ModuleList([AddNormBlock(hidden_sizes_phi[i], hidden_sizes_phi[i+1], dropout_rate) for i in range(len(hidden_sizes_phi)-1)])
  100. self.blocks2_ = nn.ModuleList([AddNormBlock1(hidden_sizes_rho[i], hidden_sizes_rho[i+1], dropout_rate=0.1) for i in range(len(hidden_sizes_rho)-1)])
  101. self.final_layer = nn.Linear(hidden_sizes_rho[-1], num_intervals)
  102. def forward_blocks(self, blocks, x):
  103. for block in blocks:
  104. x = block(x)
  105. return x
  107. def create_blocks(self, hidden_sizes, dropout_rate):
  108. layers = []
  109. for idx, hidden_size in enumerate(hidden_sizes):
  110. if idx == 0:
  111. input_size = hidden_sizes[0]
  112. else:
  113. input_size = hidden_sizes[idx - 1]
  115. layers.extend([
  116. nn.Linear(input_size, hidden_size),
  117. nn.LayerNorm(hidden_size),
  118. nn.ReLU(),
  119. nn.Dropout(dropout_rate)
  120. ])
  122. return nn.Sequential(*layers)
  124. def attention_block(self, x, mask=None):
  125. attention_weights = self.attention(x).squeeze(-1)
  126. if mask is not None:
  127. attention_weights = attention_weights.masked_fill(mask == 0, -1e9)
  128. attention_weights = torch.softmax(attention_weights, dim=1)
  130. # Apply attention weights without changing set size
  131. x_weighted = x * attention_weights.unsqueeze(-1)
  132. return x_weighted,[x,attention_weights]
  134. def forward(self, x, mask=None):
  135. original_shape = x.shape
  136. x = x.view(-1, x.size(-1))
  137. # Phi function
  138. # x = self.blocks1(x)
  139. x = self.forward_blocks(self.blocks1_, x)
  140. x = x.view(original_shape[0], original_shape[1], -1)
  141. # Attention and aggregation
  142. x ,attention_weights= self.attention_block(x, mask)
  143. x1 = x.sum(dim=1) # Summing over the set elements
  144. # Rho function
  145. x = self.blocks2(x1)
  146. # x = self.forward_blocks(self.blocks2_, x)
  147. # Final output
  148. x = self.final_layer(x)
  149. return x,x1,attention_weights
  151. ##TABULAR
  152. class AttentionSurvivalNet(nn.Module):
  153. def __init__(self,input_dim,output_dim):
  154. super(AttentionSurvivalNet,self).__init__()
  155. self.block1 = nn.Sequential(
  156. nn.Linear(input_dim, 2000),
  157. nn.BatchNorm1d(2000),
  158. nn.ReLU(),
  159. nn.Dropout(0.3),
  160. nn.Linear(2000, 800),
  161. nn.BatchNorm1d(800),
  162. nn.ReLU(),
  163. nn.Dropout(0.3))
  164. self.block2=nn.Sequential(
  165. nn.Linear(800, 100),
  166. nn.BatchNorm1d(100),
  167. nn.ReLU(),
  168. nn.Dropout(0.1),
  169. nn.Linear(100, output_dim))
  170. self.attention= nn.Sequential(
  171. nn.Linear(input_dim, input_dim),
  172. nn.ReLU(),
  173. nn.Dropout(0.3),
  174. nn.Linear(input_dim, 1),
  175. )
  176. # self.attention=AttentionLayer(input_dim)
  177. def forward(self, x):
  178. x1 = self.block1(x)
  179. x = self.block2(x1)
  180. return x,x1
  182. ##MULTIMODAL
  183. class MultimodalNet(nn.Module):
  184. def __init__(self, num_classes, input_sizes):
  185. super(MultimodalNet, self).__init__()
  186. self.block1 = nn.Sequential(
  187. nn.Linear(input_sizes[0], 200), # size_x1
  188. nn.BatchNorm1d(200),
  189. nn.ReLU(),
  190. nn.Dropout(0.1)
  191. )
  192. if len(input_sizes) > 1:
  193. self.block2 = nn.Sequential(
  194. nn.Linear(input_sizes[1], 200), # size_x2
  195. nn.BatchNorm1d(200),
  196. nn.ReLU(),
  197. nn.Dropout(0.1)
  198. )
  199. else:
  200. self.block2 = None
  201. self.block3_input_size = 200 + (200 if self.block2 else 0) + (input_sizes[2] if len(input_sizes) > 2 else 0)
  203. self.block3 = nn.Sequential(
  204. nn.Linear(self.block3_input_size, 256),
  205. nn.BatchNorm1d(256),
  206. nn.ReLU(),
  207. nn.Dropout(0.1),
  208. nn.Linear(256, 128),
  209. nn.BatchNorm1d(128),
  210. nn.ReLU(),
  211. nn.Dropout(0.1)
  212. )
  214. self.out = nn.Sequential(
  215. nn.Linear(128, 64),
  216. nn.BatchNorm1d(64),
  217. nn.ReLU(),
  218. nn.Dropout(0.1),
  219. nn.Linear(64, num_classes)
  220. )
  222. def forward(self, x1, x2=None, x3=None):
  223. x1 = self.block1(x1)
  225. inputs = [x1] # Start with x1 as the mandatory input
  227. if self.block2 and x2 is not None:
  228. x2 = self.block2(x2)
  229. inputs.append(x2)
  231. if x3 is not None:
  232. inputs.append(x3)
  234. x = torch.cat(inputs, dim=1) # Concatenate only the provided inputs
  235. x = self.block3(x)
  236. x = self.out(x)
  237. return x

Net.py at commit 644bbb0, no license · at the source

Overview

Authors: Tongjie Wang1, Javier Alfaro2, Paul M Brennan3,4, Ajitha Rajan1
ORCID iDs: Tongjie Wang, Ajitha Rajan
  1. School of Informatics, University of Edinburgh, Edinburgh, EH8 9AB United Kingdom
  2. Cancer Research UK Scotland Centre (Edinburgh), Institute of Genetics and Cancer, University of Edinburgh, Edinburgh, EH4 2XU United Kingdom
  3. Department for Clinical Neuroscience, NHS Lothian, Royal Infirmary Edinburgh, Edinburgh, United Kingdom
  4. Translational Neurosurgery, Centre for Clinical Brain Sciences, University of Edinburgh, Edinburgh, EH16 4SB United Kingdom
Institutions: University of Edinburgh (United Kingdom); Cancer Research UK (United Kingdom); Edinburgh Cancer Research (United Kingdom); Institute of Genetics and Cancer (United Kingdom); NHS Lothian (United Kingdom); Edinburgh Royal Infirmary (United Kingdom)
Journal: Scientific reports, volume 16, issue 1, article 18546
Dates: received 25 March 2025; accepted 9 April 2026; published online 21 April 2026
Type: Research article · Language: English
License: CC BY
Identifiers: DOI 10.1038/s41598-026-48666-1 · PMID 42014603 · PMCID PMC13269476 · OpenAlex W7155058161
Open access: gold, a free copy (OpenAlex)
Status: code verified
Categories: histology / microscopy (modality), human (organism), other condition (population), clinical / translational (subfield)
Methods: Machine learning, Statistics
Keywords: Mathematics and computing, Computer science, Cancer, Cancer imaging, Cancer models, Outcomes research
MeSH: Brain Neoplasms*, Glioblastoma*, Mutation*, Gene Expression Regulation, Neoplastic, Humans, Prognosis, Survival Analysis (* major topic)
Topic: Glioma Diagnosis and Treatment (Genetics, Medicine), according to OpenAlex
Funding: Horizon 2020 (101017453)
Citations: not cited yet (Europe PMC); 22 references in the paper

Abstract

Glioblastoma (GBM) is a highly aggressive brain tumor with poor prognosis, motivating the development of more accurate survival prediction models that can integrate complementary clinical and molecular information. However, existing multimodal survival frameworks often rely on simplified genomic summaries, underuse sequence-context information from mutations, or discard global spatial structure in whole-slide histopathology. In this study, we present MUSA , a multimodal survival framework that integrates three data sources: whole-slide H&E histopathology, tabular clinical and gene expression variables, and language-model-derived mutation features. For histopathology, we use a class-map-based feature extraction pipeline that summarizes slide-level composition, spatial adjacency, and fragmentation patterns. For the molecular branch, we represent missense mutations using protein-sequence-context-based features derived from a language model and evaluate both frozen and survival-supervised representations. These modality-specific features are then fused within a survival prediction framework and evaluated under nested cross-validation. In mutation-only experiments, survival-supervised language-model features outperformed one-hot and VEP-derived baselines. In multimodal benchmark comparisons, MUSA outperformed matched reference models, and in ablation analyses the full trimodal model achieved the best overall median concordance. Together, these results show that mutation-sequence representations and spatial histopathology features provide complementary prognostic information for GBM survival modeling.

Reproduced under the paper's license (CC BY), from the paper cited above.

Repository

Its files are read in the Code ↔ Paper reader above.

tongjie-w/MUSA

License: none: the authors keep all their rights
State: the link answers, verified on 29 September 2026
Evidence: files inventoried
Commit: 644bbb073d78b460986824f357dd5e2ed3edfd2f, 14 April 2025
Languages: Python (11), Jupyter (1)
Size: 13 files, 12 scripts
Software Heritage: not archived
Found in: “Code availability”
Holds: README, 1 notebook
Not found: license file, CITATION.cff, environment file, tests, continuous integration, documentation
Tools: PyTorch (11 files), NumPy (10 files), pandas (7 files), OpenCV (3 files), scikit-learn (3 files), Keras (1 file), Matplotlib (1 file), scikit-image (1 file), TensorFlow (1 file)
Availability: 1 check, the latest on 29 September 2026: the link answers
  • 29 September 2026: the link answers
13 files
At the source: github.com/tongjie-w/MUSA

Code availability

Code is available from our MUSA Github repository: https://github.com/tongjie-w/MUSA.

Reproduced under the paper's license (CC BY), from the paper cited above.

Tracing map

Proposed by the machine: these links were found in the paper and verified at the source, without human review. The map will receive a Zenodo DOI once one of the paper's authors has validated it with their ORCID.

What the map holds:

  • 1 repository of the authors' code, each at its verified commit, with its license and how the link was found in the paper;
  • 12 scripts, each with its path and the digest of its content;
  • no match between paragraphs and code yet;
  • neither the text of the paper nor the code itself.

Its JSON (tracing-map.json) is deposited on Zenodo with its DOI once the map is validated.

Data

No dataset and no data link were found in the paper.

Data availability

The MoNuSeg dataset used for image segmentation study is available in MoNuSeg challenge repository, https://monuseg.grand-challenge.org/Data/Data used in this work on H&E slide survival analysis are from the GDC Data Portal (https://portal.gdc.cancer.gov/). The patients’ datasets used on multimodal survival analysis are not publicly available due to patients’ privacy concerns of the institutional review board policies on human tissue data.

Reproduced under the paper's license (CC BY), from the paper cited above.

Versions

The history of this record: each version stored by the harvester or made by a correction of its authors or of the maintainers of its code, and what changed in its facts. The texts of the paper (its abstract, its availability statements) are not part of it; versions that changed only those are not listed.

Version 1, 29 September 2026: the first record

Recorded: type, language, journal, volume, issue, pages, dates, 4 authors, 6 keywords, 7 MeSH terms, 1 funder, 15 references.

Cite

This paper

Wang, T., Alfaro, J., Brennan, P. M., & Rajan, A. (2026). Multimodal survival analysis of glioblastoma using whole-slide histopathology, gene expression, clinical variables and language-model-derived mutation features. Scientific reports, 16(1), 18546. https://doi.org/10.1038/s41598-026-48666-1

BibTeX

@article{wang2026multimodal,
author = {Wang, Tongjie and Alfaro, Javier and Brennan, Paul M and Rajan, Ajitha},
title = {{Multimodal survival analysis of glioblastoma using whole-slide histopathology, gene expression, clinical variables and language-model-derived mutation features}},
journal = {Scientific reports},
year = {2026},
month = apr,
volume = {16},
number = {1},
pages = {18546},
publisher = {Nature Publishing Group},
issn = {2045-2322},
doi = {10.1038/s41598-026-48666-1},
url = {https://doi.org/10.1038/s41598-026-48666-1},
pmid = {42014603},
pmcid = {PMC13269476}
}

RIS

TY - JOUR
AU - Wang, Tongjie
AU - Alfaro, Javier
AU - Brennan, Paul M
AU - Rajan, Ajitha
TI - Multimodal survival analysis of glioblastoma using whole-slide histopathology, gene expression, clinical variables and language-model-derived mutation features
T2 - Scientific reports
J2 - Sci Rep
PY - 2026
DA - 2026/04/21
VL - 16
IS - 1
SP - 18546
SN - 2045-2322
PB - Nature Publishing Group
DO - 10.1038/s41598-026-48666-1
UR - https://doi.org/10.1038/s41598-026-48666-1
LA - en
ER -

CSL-JSON

{
"id": "10.1038/s41598-026-48666-1",
"type": "article-journal",
"title": "Multimodal survival analysis of glioblastoma using whole-slide histopathology, gene expression, clinical variables and language-model-derived mutation features",
"container-title": "Scientific reports",
"author": [
{
"family": "Wang",
"given": "Tongjie"
},
{
"family": "Alfaro",
"given": "Javier"
},
{
"family": "Brennan",
"given": "Paul M"
},
{
"family": "Rajan",
"given": "Ajitha"
}
],
"container-title-short": "Sci Rep",
"volume": "16",
"issue": "1",
"page": "18546",
"DOI": "10.1038/s41598-026-48666-1",
"PMID": "42014603",
"PMCID": "PMC13269476",
"ISSN": "2045-2322",
"publisher": "Nature Publishing Group",
"URL": "https://doi.org/10.1038/s41598-026-48666-1",
"language": "en",
"issued": {
"date-parts": [
[
2026,
4,
21
]
]
}
}

The tracing map gets a citation of its own once an author has validated it and it has a DOI.

Similar papers

The papers with a page that share the most with this one: the tools found in their code, their categories, datasets, cited references and authors, the rarest counting most.

[1] doi:10.1016/j.xcrm.2026.102766 [code]
A longitudinal single-cell and spatial multiomic atlas of pediatric high-grade glioma.
Journal: Cell reports. Medicine
In common: Keras, TensorFlow, OpenCV, 5 other tools, other condition, 1 reference
[2] doi:10.1093/jnen/nlaf152 [code]
Clinical and pathologic correlations of machine learning quantification of Aβ deposits across 3 brain regions of decedents with Alzheimer disease.
Journal: Journal of neuropathology and experimental neurology
In common: Keras, TensorFlow, OpenCV, 6 other tools, clinical / translational
[3] doi:10.1038/s42003-026-10957-8 [code]
Brain defence by the extracellular matrix protein Cochlin.
Journal: Communications biology
In common: Keras, TensorFlow, OpenCV, 6 other tools
[4] doi:10.1371/journal.pcbi.1014571 [code]
SynAPSeg: A novel dataset and image analysis framework for deep learning-based synapse detection and quantification.
Journal: PLoS computational biology
In common: Keras, TensorFlow, OpenCV, 6 other tools
[5] doi:10.1126/sciadv.aed3650 [code]
Truthful visualizations for mass spectrometry imaging enable high-spatial-resolution interactive <i>m/z</i> mapping and exploration.
Journal: Science advances
In common: Keras, TensorFlow, OpenCV, 6 other tools
[6] doi:10.1364/boe.605322 [code]
Generalized plaque digitization framework for multi-dimensional mesoscopic images.
Journal: Biomedical optics express
In common: Keras, TensorFlow, OpenCV, 6 other tools
[7] doi:10.1038/s41467-026-72057-9 [code]
Sex-specific behavioral feedback modulates sensorimotor processing and drives flexible social behavior.
Journal: Nature communications
In common: Keras, TensorFlow, OpenCV, 6 other tools
[8] doi:10.1002/epi.70296 [code]
Fully automated three-dimensional deep learning-based magnetic resonance imaging segmentation of brain cavities in epilepsy surgery.
Journal: Epilepsia
In common: Keras, TensorFlow, OpenCV, 5 other tools, clinical / translational
[9] doi:10.1186/s12880-026-02481-2 [code]
Deep learning-based neuroanatomical profiling reveals population-specific brain changes in multiple sclerosis: a large-scale Middle Eastern study.
Journal: BMC medical imaging
In common: Keras, TensorFlow, OpenCV, 5 other tools, clinical / translational
[10] doi:10.1038/s41586-026-10348-3 [code]
An enteric neuron ionotropic receptor regulates salt stress resistance.
Journal: Nature
In common: Keras, OpenCV, scikit-image, 5 other tools, 1 reference

Contribute

The authors of this paper can claim it, correct its record and validate its tracing map, and the maintainers of its code (its owner, or a public member of its organization) correct what it says of their repository; anyone signed in can ask for its removal. Every request goes to OSCR's own machine, which answers it; your account page follows them.

Sign in with ORCID to claim this paper as one of its authors, correct its record or validate its tracing map: when the paper's metadata lists your ORCID iD, you are recognized at once. Maintainers of its code: sign in with GitHub, then claim the repository on your account page.

Request its removal

To ask OSCR to remove this record, the copies of its authors' scripts or its tracing map, use the removal request page: signed in, you say who you are, what to remove and why, then review and confirm the request. Published rules decide every request (how).

Discussion, reproductions, activity

Discussion: questions and error reports about this paper and its code, from signed-in readers and its authors. It opens with sign-in.

Reproductions: reports from readers who ran the authors' code: what they reproduced, with which environment, commit and data. It opens with sign-in.

Activity: what happens around this paper: new versions of its record, its map's validation, discussions and reproductions. It opens with sign-in.