OSCR

Intracellular protein GBF1 displays significant associations with amyloid pathology in Alzheimer's disease.

Code ↔ Paper

1 match between paragraphs of the paper and lines of its authors' code, computed by the harvester (lexical-v1). Click a colored paragraph or line to see its counterpart.

The 1 match
  1. [1] § METHODS › Genetic discovery ↔ bcbio/pipeline/main.py, lines 116–202 · score 0.65 · quality control, Variant calls, pruned, jointly, GEMINI, SNPEff

Paper

Loaded from Europe PMC by your browser, not stored by OSCR: doi.org · Europe PMC

The paper is loaded when this pane is shown.

The authors' code

Python · 533 lines · 27 KB · MIT · 1 match

  1. """Main entry point for distributed next-gen sequencing pipelines.
  3. Handles running the full pipeline based on instructions
  4. """
  5. from __future__ import print_function
  6. from collections import defaultdict
  7. import copy
  8. import os
  9. import sys
  10. import resource
  11. import tempfile
  13. import toolz as tz
  15. from bcbio import log, heterogeneity, hla, structural, utils
  16. from bcbio.cwl.inspect import initialize_watcher
  17. from bcbio.distributed import prun
  18. from bcbio.distributed.transaction import tx_tmpdir
  19. from bcbio.log import logger, DEFAULT_LOG_DIR
  20. from bcbio.ngsalign import alignprep
  21. from bcbio.pipeline import datadict as dd
  22. from bcbio.pipeline import (archive, config_utils, disambiguate, region,
  23. run_info, qcsummary, rnaseq)
  24. from bcbio.provenance import profile, system
  25. from bcbio.variation import (ensemble, genotype, population, validate, joint,
  26. peddy)
  27. from bcbio.chipseq import peaks, atac
  29. def run_main(workdir, config_file=None, fc_dir=None, run_info_yaml=None,
  30. parallel=None, workflow=None):
  31. """Run variant analysis, handling command line options.
  32. """
  33. # Set environment to standard to use periods for decimals and avoid localization
  34. locale_to_use = utils.get_locale()
  35. os.environ["LC_ALL"] = locale_to_use
  36. os.environ["LC"] = locale_to_use
  37. os.environ["LANG"] = locale_to_use
  38. workdir = utils.safe_makedir(os.path.abspath(workdir))
  39. os.chdir(workdir)
  40. config, config_file = config_utils.load_system_config(config_file, workdir)
  41. parallel = log.create_base_logger(config, parallel)
  42. log.setup_local_logging(config, parallel)
  43. logger.info(f"System YAML configuration: {os.path.abspath(config_file)}.")
  44. logger.info(f"Locale set to {locale_to_use}.")
  45. if config.get("log_dir", None) is None:
  46. config["log_dir"] = os.path.join(workdir, DEFAULT_LOG_DIR)
  47. if parallel["type"] in ["local", "clusterk"]:
  48. _setup_resources()
  49. _run_toplevel(config, config_file, workdir, parallel,
  50. fc_dir, run_info_yaml)
  51. elif parallel["type"] == "ipython":
  52. assert parallel["scheduler"] is not None, "IPython parallel requires a specified scheduler (-s)"
  53. if parallel["scheduler"] != "sge":
  54. assert parallel["queue"] is not None, "IPython parallel requires a specified queue (-q)"
  55. elif not parallel["queue"]:
  56. parallel["queue"] = ""
  57. _run_toplevel(config, config_file, workdir, parallel,
  58. fc_dir, run_info_yaml)
  59. else:
  60. raise ValueError("Unexpected type of parallel run: %s" % parallel["type"])
  62. def _setup_resources():
  63. """Attempt to increase resource limits up to hard limits.
  65. This allows us to avoid out of file handle limits where we can
  66. move beyond the soft limit up to the hard limit.
  67. """
  68. target_procs = 10240
  69. cur_proc, max_proc = resource.getrlimit(resource.RLIMIT_NPROC)
  70. target_proc = min(max_proc, target_procs) if max_proc > 0 else target_procs
  71. resource.setrlimit(resource.RLIMIT_NPROC, (max(cur_proc, target_proc), max_proc))
  72. cur_hdls, max_hdls = resource.getrlimit(resource.RLIMIT_NOFILE)
  73. target_hdls = min(max_hdls, target_procs) if max_hdls > 0 else target_procs
  74. resource.setrlimit(resource.RLIMIT_NOFILE, (max(cur_hdls, target_hdls), max_hdls))
  76. def _run_toplevel(config, config_file, work_dir, parallel,
  77. fc_dir=None, run_info_yaml=None):
  78. """
  79. Run toplevel analysis, processing a set of input files.
  80. config_file -- Main YAML configuration file with system parameters
  81. fc_dir -- Directory of fastq files to process
  82. run_info_yaml -- YAML configuration file specifying inputs to process
  83. """
  84. dirs = run_info.setup_directories(work_dir, fc_dir, config, config_file)
  85. config_file = os.path.join(dirs["config"], os.path.basename(config_file))
  86. pipelines, config = _pair_samples_with_pipelines(run_info_yaml, config)
  87. system.write_info(dirs, parallel, config)
  88. with tx_tmpdir(config if parallel.get("type") == "local" else None) as tmpdir:
  89. tempfile.tempdir = tmpdir
  90. for pipeline, samples in pipelines.items():
  91. for xs in pipeline(config, run_info_yaml, parallel, dirs, samples):
  92. pass
  94. # ## Generic pipeline framework
  96. def _wres(parallel, progs, fresources=None, ensure_mem=None):
  97. """Add resource information to the parallel environment on required programs and files.
  99. Enables spinning up required machines and operating in non-shared filesystem
  100. environments.
  102. progs -- Third party tools used in processing
  103. fresources -- Required file-based resources needed. These will be transferred on non-shared
  104. filesystems.
  105. ensure_mem -- Dictionary of required minimum memory for programs used. Ensures
  106. enough memory gets allocated on low-core machines.
  107. """
  108. parallel = copy.deepcopy(parallel)
  109. parallel["progs"] = progs
  110. if fresources:
  111. parallel["fresources"] = fresources
  112. if ensure_mem:
  113. parallel["ensure_mem"] = ensure_mem
  114. return parallel
  116. def variant2pipeline(config, run_info_yaml, parallel, dirs, samples):
  117. ## Alignment and preparation requiring the entire input file (multicore cluster)
  118. # Assign GATK supplied memory if required for post-process recalibration
  119. align_programs = ["aligner", "samtools", "sambamba"]
  120. if any(tz.get_in(["algorithm", "recalibrate"], utils.to_single_data(d)) in [True, "gatk"] for d in samples):
  121. align_programs.append("gatk")
  122. with prun.start(_wres(parallel, align_programs,
  123. (["reference", "fasta"], ["reference", "aligner"], ["files"])),
  124. samples, config, dirs, "multicore",
  125. multiplier=alignprep.parallel_multiplier(samples)) as run_parallel:
  126. with profile.report("organize samples", dirs):
  127. samples = run_parallel("organize_samples", [[dirs, config, run_info_yaml,
  128. [x[0]["description"] for x in samples]]])
  129. with profile.report("alignment preparation", dirs):
  130. samples = run_parallel("prep_align_inputs", samples)
  131. samples = run_parallel("disambiguate_split", [samples])
  132. with profile.report("alignment", dirs):
  133. samples = run_parallel("process_alignment", samples)
  134. samples = disambiguate.resolve(samples, run_parallel)
  135. samples = alignprep.merge_split_alignments(samples, run_parallel)
  136. with profile.report("callable regions", dirs):
  137. samples = run_parallel("prep_samples", [samples])
  138. samples = run_parallel("postprocess_alignment", samples)
  139. samples = run_parallel("combine_sample_regions", [samples])
  140. samples = run_parallel("calculate_sv_bins", [samples])
  141. samples = run_parallel("calculate_sv_coverage", samples)
  142. samples = run_parallel("normalize_sv_coverage", [samples])
  143. samples = region.clean_sample_data(samples)
  144. with profile.report("hla typing", dirs):
  145. samples = hla.run(samples, run_parallel)
  147. ## Variant calling on sub-regions of the input file (full cluster)
  148. with prun.start(_wres(parallel, ["gatk", "picard", "variantcaller"]),
  149. samples, config, dirs, "full",
  150. multiplier=region.get_max_counts(samples), max_multicore=1) as run_parallel:
  151. with profile.report("alignment post-processing", dirs):
  152. samples = region.parallel_prep_region(samples, run_parallel)
  153. with profile.report("variant calling", dirs):
  154. samples = genotype.parallel_variantcall_region(samples, run_parallel)
  155. with profile.report("joint squaring off/backfilling", dirs):
  156. samples = joint.square_off(samples, run_parallel)
  157. ## Finalize variants, BAMs and population databases (per-sample multicore cluster)
  158. with prun.start(_wres(parallel, ["gatk", "gatk-vqsr", "snpeff", "bcbio_variation",
  159. "gemini", "samtools", "fastqc", "sambamba",
  160. "bcbio-variation-recall", "qsignature",
  161. "svcaller", "kraken", "preseq"]),
  162. samples, config, dirs, "multicore2",
  163. multiplier=structural.parallel_multiplier(samples)) as run_parallel:
  164. with profile.report("variant post-processing", dirs):
  165. samples = run_parallel("postprocess_variants", samples)
  166. samples = run_parallel("split_variants_by_sample", samples)
  167. with profile.report("prepped BAM merging", dirs):
  168. samples = region.delayed_bamprep_merge(samples, run_parallel)
  169. with profile.report("validation", dirs):
  170. samples = run_parallel("compare_to_rm", samples)
  171. samples = genotype.combine_multiple_callers(samples)
  172. with profile.report("ensemble calling", dirs):
  173. samples = ensemble.combine_calls_parallel(samples, run_parallel)
  174. with profile.report("validation summary", dirs):
  175. samples = validate.summarize_grading(samples)
  176. with profile.report("structural variation", dirs):
  177. samples = structural.run(samples, run_parallel, "initial")
  178. with profile.report("structural variation", dirs):
  179. samples = structural.run(samples, run_parallel, "standard")
  180. with profile.report("structural variation ensemble", dirs):
  181. samples = structural.run(samples, run_parallel, "ensemble")
  182. with profile.report("structural variation validation", dirs):
  183. samples = run_parallel("validate_sv", samples)
  184. with profile.report("heterogeneity", dirs):
  185. samples = heterogeneity.run(samples, run_parallel)
  186. with profile.report("population database", dirs):
  187. samples = population.prep_db_parallel(samples, run_parallel)
  188. # after SV calling and SNV merging
  189. with profile.report("create CNV PON", dirs):
  190. samples = structural.create_cnv_pon(samples)
  191. with profile.report("peddy check", dirs):
  192. samples = peddy.run_peddy_parallel(samples, run_parallel)
  193. with profile.report("quality control", dirs):
  194. samples = qcsummary.generate_parallel(samples, run_parallel)
  195. with profile.report("archive", dirs):
  196. samples = archive.compress(samples, run_parallel)
  197. with profile.report("upload", dirs):
  198. samples = run_parallel("upload_samples", samples)
  199. for sample in samples:
  200. run_parallel("upload_samples_project", [sample])
  201. logger.info("Timing: finished")
  202. return samples
  204. def _debug_samples(i, samples):
  205. print("---", i, len(samples))
  206. for sample in (utils.to_single_data(x) for x in samples):
  207. print(" ", sample["description"], sample.get("region"), \
  208. utils.get_in(sample, ("config", "algorithm", "variantcaller")), \
  209. utils.get_in(sample, ("config", "algorithm", "jointcaller")), \
  210. utils.get_in(sample, ("metadata", "batch")), \
  211. [x.get("variantcaller") for x in sample.get("variants", [])], \
  212. sample.get("work_bam"), \
  213. sample.get("vrn_file"))
  215. def standardpipeline(config, run_info_yaml, parallel, dirs, samples):
  216. ## Alignment and preparation requiring the entire input file (multicore cluster)
  217. with prun.start(_wres(parallel, ["aligner", "samtools", "sambamba"]),
  218. samples, config, dirs, "multicore") as run_parallel:
  219. with profile.report("organize samples", dirs):
  220. samples = run_parallel("organize_samples", [[dirs, config, run_info_yaml,
  221. [x[0]["description"] for x in samples]]])
  222. with profile.report("alignment", dirs):
  223. samples = run_parallel("process_alignment", samples)
  224. with profile.report("callable regions", dirs):
  225. samples = run_parallel("prep_samples", [samples])
  226. samples = run_parallel("postprocess_alignment", samples)
  227. samples = run_parallel("combine_sample_regions", [samples])
  228. samples = region.clean_sample_data(samples)
  229. ## Quality control
  230. with prun.start(_wres(parallel, ["fastqc", "qsignature", "kraken", "gatk", "samtools", "preseq"]),
  231. samples, config, dirs, "multicore2") as run_parallel:
  232. with profile.report("quality control", dirs):
  233. samples = qcsummary.generate_parallel(samples, run_parallel)
  234. with profile.report("upload", dirs):
  235. samples = run_parallel("upload_samples", samples)
  236. for sample in samples:
  237. run_parallel("upload_samples_project", [sample])
  238. logger.info("Timing: finished")
  239. return samples
  241. def rnaseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  242. samples = rnaseq_prep_samples(config, run_info_yaml, parallel, dirs, samples)
  243. with prun.start(_wres(parallel, ["aligner", "picard", "samtools"],
  244. ensure_mem={"tophat": 10, "tophat2": 10, "star": 2, "hisat2": 8}),
  245. samples, config, dirs, "alignment",
  246. multiplier=alignprep.parallel_multiplier(samples)) as run_parallel:
  247. with profile.report("alignment", dirs):
  248. samples = run_parallel("disambiguate_split", [samples])
  249. samples = run_parallel("process_alignment", samples)
  250. with prun.start(_wres(parallel, ["samtools", "cufflinks"]),
  251. samples, config, dirs, "rnaseqcount") as run_parallel:
  252. with profile.report("disambiguation", dirs):
  253. samples = disambiguate.resolve(samples, run_parallel)
  254. with profile.report("transcript assembly", dirs):
  255. samples = rnaseq.assemble_transcripts(run_parallel, samples)
  256. with profile.report("estimate expression (threaded)", dirs):
  257. samples = rnaseq.quantitate_expression_parallel(samples, run_parallel)
  259. with prun.start(_wres(parallel, ["dexseq", "express"]), samples, config,
  260. dirs, "rnaseqcount-singlethread", max_multicore=1) as run_parallel:
  261. with profile.report("estimate expression (single threaded)", dirs):
  262. samples = rnaseq.quantitate_expression_noparallel(samples, run_parallel)
  264. samples = rnaseq.combine_files(samples)
  265. with prun.start(_wres(parallel, ["gatk", "vardict"]), samples, config,
  266. dirs, "rnaseq-variation") as run_parallel:
  267. with profile.report("RNA-seq variant calling", dirs):
  268. samples = rnaseq.rnaseq_variant_calling(samples, run_parallel)
  270. with prun.start(_wres(parallel, ["samtools", "fastqc", "qualimap",
  271. "kraken", "gatk", "preseq"], ensure_mem={"qualimap": 4}),
  272. samples, config, dirs, "qc") as run_parallel:
  273. with profile.report("quality control", dirs):
  274. samples = qcsummary.generate_parallel(samples, run_parallel)
  275. with profile.report("create SummarizedExperiment object", dirs):
  276. samples = rnaseq.load_summarizedexperiment(samples)
  277. with profile.report("upload", dirs):
  278. samples = run_parallel("upload_samples", samples)
  279. for sample in samples:
  280. run_parallel("upload_samples_project", [sample])
  281. with profile.report("bcbioRNAseq loading", dirs):
  282. tools_on = dd.get_in_samples(samples, dd.get_tools_on)
  283. bcbiornaseq_on = tools_on and "bcbiornaseq" in tools_on
  284. if bcbiornaseq_on:
  285. if len(samples) < 3:
  286. logger.warn("bcbioRNASeq needs at least three samples total, skipping.")
  287. elif len(samples) > 100:
  288. logger.warn("Over 100 samples, skipping bcbioRNASeq.")
  289. else:
  290. run_parallel("run_bcbiornaseqload", [sample])
  291. logger.info("Timing: finished")
  292. return samples
  294. def fastrnaseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  295. samples = rnaseq_prep_samples(config, run_info_yaml, parallel, dirs, samples)
  296. ww = initialize_watcher(samples)
  297. with prun.start(_wres(parallel, ["samtools"]), samples, config,
  298. dirs, "fastrnaseq") as run_parallel:
  299. with profile.report("fastrnaseq", dirs):
  300. samples = rnaseq.fast_rnaseq(samples, run_parallel)
  301. ww.report("fastrnaseq", samples)
  302. samples = rnaseq.combine_files(samples)
  303. with profile.report("quality control", dirs):
  304. samples = qcsummary.generate_parallel(samples, run_parallel)
  305. ww.report("qcsummary", samples)
  306. with profile.report("upload", dirs):
  307. samples = run_parallel("upload_samples", samples)
  308. for samples in samples:
  309. run_parallel("upload_samples_project", [samples])
  310. logger.info("Timing: finished")
  311. return samples
  313. def singlecellrnaseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  314. samples = rnaseq_prep_samples(config, run_info_yaml, parallel, dirs, samples)
  315. with prun.start(_wres(parallel, ["samtools", "rapmap"]), samples, config,
  316. dirs, "singlecell-rnaseq") as run_parallel:
  317. with profile.report("singlecell-rnaseq", dirs):
  318. samples = rnaseq.singlecell_rnaseq(samples, run_parallel)
  319. with profile.report("quality control", dirs):
  320. samples = qcsummary.generate_parallel(samples, run_parallel)
  321. with profile.report("upload", dirs):
  322. samples = run_parallel("upload_samples", samples)
  323. for samples in samples:
  324. run_parallel("upload_samples_project", [samples])
  325. logger.info("Timing: finished")
  326. return samples
  328. def smallrnaseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  329. # causes a circular import at the top level
  330. from bcbio.srna.group import report as srna_report
  332. samples = rnaseq_prep_samples(config, run_info_yaml, parallel, dirs, samples)
  334. with prun.start(_wres(parallel, ["aligner", "picard", "samtools"],
  335. ensure_mem={"bowtie": 8, "bowtie2": 8, "star": 2}),
  336. [samples[0]], config, dirs, "alignment") as run_parallel:
  337. with profile.report("prepare", dirs):
  338. samples = run_parallel("seqcluster_prepare", [samples])
  339. with profile.report("seqcluster alignment", dirs):
  340. samples = run_parallel("srna_alignment", [samples])
  342. with prun.start(_wres(parallel, ["aligner", "picard", "samtools"],
  343. ensure_mem={"tophat": 10, "tophat2": 10, "star": 2, "hisat2": 8}),
  344. samples, config, dirs, "alignment_samples",
  345. multiplier=alignprep.parallel_multiplier(samples)) as run_parallel:
  346. with profile.report("alignment", dirs):
  347. samples = run_parallel("process_alignment", samples)
  349. with prun.start(_wres(parallel, ["picard", "miraligner"]),
  350. samples, config, dirs, "annotation") as run_parallel:
  351. with profile.report("small RNA annotation", dirs):
  352. samples = run_parallel("srna_annotation", samples)
  354. with prun.start(_wres(parallel, ["seqcluster", "mirge"],
  355. ensure_mem={"seqcluster": 8}),
  356. [samples[0]], config, dirs, "cluster") as run_parallel:
  357. with profile.report("cluster", dirs):
  358. samples = run_parallel("seqcluster_cluster", [samples])
  360. with prun.start(_wres(parallel, ["picard", "fastqc"]),
  361. samples, config, dirs, "qc") as run_parallel:
  362. with profile.report("quality control", dirs):
  363. samples = qcsummary.generate_parallel(samples, run_parallel)
  364. with profile.report("report", dirs):
  365. srna_report(samples)
  366. with profile.report("upload", dirs):
  367. samples = run_parallel("upload_samples", samples)
  368. for sample in samples:
  369. run_parallel("upload_samples_project", [sample])
  371. return samples
  373. def chipseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  374. with prun.start(_wres(parallel, ["aligner", "picard"]),
  375. samples, config, dirs, "multicore",
  376. multiplier=alignprep.parallel_multiplier(samples)) as run_parallel:
  377. with profile.report("organize samples", dirs):
  378. samples = run_parallel("organize_samples", [[dirs, config, run_info_yaml,
  379. [x[0]["description"] for x in samples]]])
  380. with profile.report("alignment", dirs):
  381. samples = run_parallel("prepare_sample", samples)
  382. samples = run_parallel("trim_sample", samples)
  383. samples = run_parallel("disambiguate_split", [samples])
  384. samples = run_parallel("process_alignment", samples)
  386. with profile.report("disambiguation", dirs):
  387. samples = disambiguate.resolve(samples, run_parallel)
  388. samples = run_parallel("clean_chipseq_alignment", samples)
  390. with prun.start(_wres(parallel, ["peakcaller"]),
  391. samples, config, dirs, "peakcalling",
  392. multiplier = peaks._get_multiplier(samples)) as run_parallel:
  393. with profile.report("peakcalling", dirs):
  394. samples = peaks.peakcall_prepare(samples, run_parallel)
  395. samples = peaks.call_consensus(samples)
  396. samples = run_parallel("run_chipseq_count", samples)
  398. samples = peaks.create_peaktable(samples)
  400. with prun.start(_wres(parallel, ["picard", "fastqc"]),
  401. samples, config, dirs, "qc") as run_parallel:
  402. with profile.report("quality control", dirs):
  403. samples = qcsummary.generate_parallel(samples, run_parallel)
  404. samples = atac.create_ataqv_report(samples)
  405. with profile.report("upload", dirs):
  406. samples = run_parallel("upload_samples", samples)
  407. for sample in samples:
  408. run_parallel("upload_samples_project", [sample])
  409. logger.info("Timing: finished")
  410. return samples
  413. def wgbsseqpipeline(config, run_info_yaml, parallel, dirs, samples):
  414. with prun.start(_wres(parallel, ["fastqc", "picard"], ensure_mem={"fastqc" : 4}),
  415. samples, config, dirs, "trimming") as run_parallel:
  416. with profile.report("organize samples", dirs):
  417. samples = run_parallel("organize_samples", [[dirs, config, run_info_yaml,
  418. [x[0]["description"] for x in samples]]])
  419. samples = run_parallel("prepare_sample", samples)
  420. samples = run_parallel("trim_bs_sample", samples)
  422. with prun.start(_wres(parallel, ["aligner", "bismark", "picard", "samtools"]),
  423. samples, config, dirs, "multicore",
  424. multiplier=alignprep.parallel_multiplier(samples)) as run_parallel:
  425. with profile.report("alignment", dirs):
  426. samples = run_parallel("process_alignment", samples)
  428. with prun.start(_wres(parallel, ['samtools']), samples, config, dirs,
  429. 'deduplication') as run_parallel:
  430. with profile.report('deduplicate', dirs):
  431. samples = run_parallel('deduplicate_bismark', samples)
  433. with prun.start(_wres(parallel, ["caller"], ensure_mem={"caller": 5}),
  434. samples, config, dirs, "multicore2",
  435. multiplier=24) as run_parallel:
  436. with profile.report("cpg calling", dirs):
  437. samples = run_parallel("cpg_calling", samples)
  439. with prun.start(_wres(parallel, ["picard", "fastqc", "samtools"]),
  440. samples, config, dirs, "qc") as run_parallel:
  441. with profile.report("quality control", dirs):
  442. samples = qcsummary.generate_parallel(samples, run_parallel)
  443. with profile.report("upload", dirs):
  444. samples = run_parallel("upload_samples", samples)
  445. for sample in samples:
  446. run_parallel("upload_samples_project", [sample])
  447. logger.info("Timing: finished")
  448. return samples
  451. def rnaseq_prep_samples(config, run_info_yaml, parallel, dirs, samples):
  452. """
  453. organizes RNA-seq and small-RNAseq samples, converting from BAM if
  454. necessary and trimming if necessary
  455. """
  456. pipeline = dd.get_in_samples(samples, dd.get_analysis)
  457. trim_reads_set = any([tz.get_in(["algorithm", "trim_reads"], d) for d in dd.sample_data_iterator(samples)])
  458. resources = ["picard"]
  459. needs_trimming = (_is_smallrnaseq(pipeline) or trim_reads_set)
  460. if needs_trimming:
  461. resources.append("atropos")
  462. with prun.start(_wres(parallel, resources),
  463. samples, config, dirs, "trimming",
  464. max_multicore=1 if not needs_trimming else None) as run_parallel:
  465. with profile.report("organize samples", dirs):
  466. samples = run_parallel("organize_samples", [[dirs, config, run_info_yaml,
  467. [x[0]["description"] for x in samples]]])
  468. samples = run_parallel("prepare_sample", samples)
  469. if needs_trimming:
  470. with profile.report("adapter trimming", dirs):
  471. if _is_smallrnaseq(pipeline):
  472. samples = run_parallel("trim_srna_sample", samples)
  473. else:
  474. samples = run_parallel("trim_sample", samples)
  475. return samples
  477. def _get_pipeline(item):
  478. from bcbio.log import logger
  479. analysis_type = item.get("analysis", "").lower()
  480. if analysis_type not in SUPPORTED_PIPELINES:
  481. logger.error("Cannot determine which type of analysis to run, "
  482. "set in the run_info under details.")
  483. sys.exit(1)
  484. else:
  485. return SUPPORTED_PIPELINES[analysis_type]
  487. def _pair_samples_with_pipelines(run_info_yaml, config):
  488. """Map samples defined in input file to pipelines to run.
  489. """
  490. samples = config_utils.load_config(run_info_yaml)
  491. if isinstance(samples, dict):
  492. resources = samples.pop("resources")
  493. samples = samples["details"]
  494. else:
  495. resources = {}
  496. ready_samples = []
  497. for sample in samples:
  498. if "files" in sample:
  499. del sample["files"]
  500. # add any resources to this item to recalculate global configuration
  501. usample = copy.deepcopy(sample)
  502. usample.pop("algorithm", None)
  503. if "resources" not in usample:
  504. usample["resources"] = {}
  505. for prog, pkvs in resources.items():
  506. if prog not in usample["resources"]:
  507. usample["resources"][prog] = {}
  508. if pkvs is not None:
  509. for key, val in pkvs.items():
  510. usample["resources"][prog][key] = val
  511. config = config_utils.update_w_custom(config, usample)
  512. sample["resources"] = {}
  513. ready_samples.append(sample)
  514. paired = [(x, _get_pipeline(x)) for x in ready_samples]
  515. d = defaultdict(list)
  516. for x in paired:
  517. d[x[1]].append([x[0]])
  518. return d, config
  520. SUPPORTED_PIPELINES = {"variant2": variant2pipeline,
  521. "snp calling": variant2pipeline,
  522. "variant": variant2pipeline,
  523. "standard": standardpipeline,
  524. "minimal": standardpipeline,
  525. "rna-seq": rnaseqpipeline,
  526. "smallrna-seq": smallrnaseqpipeline,
  527. "chip-seq": chipseqpipeline,
  528. "wgbs-seq": wgbsseqpipeline,
  529. "fastrna-seq": fastrnaseqpipeline,
  530. "scrna-seq": singlecellrnaseqpipeline}
  532. def _is_smallrnaseq(pipeline):
  533. return pipeline.lower() == "smallrna-seq"

main.py at commit 594339f, under MIT · at the source

Overview

Authors: Sean J. Miller1,2, Dmitry Prokopenko1, Ping Bai3, Prasenjit Mondal1, Abigael Scott1, Wei Zhang1, Ashley Gomm1, Siyi Zhang1, Daniel D. Child1, Nolan Shen1, Joseph Ward1, Scott Schulte1, Dan Lei1, Brian P. Hafler2,4,5, Changning Wang3, Rudolph E. Tanzi1, Can Zhang1
ORCID iDs: Sean J. Miller, Prasenjit Mondal, Daniel D. Child
  1. Genetics and Aging Research Unit McCance Center for Brain Health MassGeneral Institute for Neurodegenerative Disease Department of Neurology Massachusetts General Hospital and Harvard Medical School Charlestown Massachusetts USA
  2. Department of Ophthalmology and Visual Science Yale School of Medicine New Haven Connecticut USA
  3. Athinoula A. Martinos Center for Biomedical Imaging Department of Radiology Massachusetts General Hospital and Harvard Medical School Charlestown Massachusetts USA
  4. Department of Pathology Yale School of Medicine New Haven Connecticut USA
  5. The Broad Institute MIT and Harvard Cambridge Massachusetts USA
Institutions: Harvard University (United States); Yale University (United States); Massachusetts General Hospital (United States); Athinoula A. Martinos Center for Biomedical Imaging (United States); Broad Institute (United States); Massachusetts Institute of Technology (United States)
Journal: Alzheimer's & dementia : the journal of the Alzheimer's Association, volume 22, issue 4, article e71271
Dates: received 25 August 2025; accepted 30 December 2025; published online 16 April 2026; in print April 2026
Type: Research article · Language: English
License: CC BY-NC-ND
Identifiers: DOI 10.1002/alz.71271 · PMID 41988936 · PMCID PMC13084705 · OpenAlex W7154624661
Open access: hybrid, a free copy (OpenAlex)
Status: code verified
Categories: histology / microscopy (modality), human (organism), mouse (organism), Alzheimer's / dementia (population), cellular / molecular (subfield)
Methods: Statistics, Smoothing, state filtering, decompositions, Machine learning, Evoked potentials
Keywords: Alzheimer's disease, amyloid beta, amyloid precursor protein, Golgi brefeldin A‐resistant guanine nucleotide exchange factor‐1, golgicide A, neuropathology
MeSH: Alzheimer Disease*, Brain*, Guanine Nucleotide Exchange Factors*, Plaque, Amyloid*, Amyloid beta-Peptides, Amyloid beta-Protein Precursor, Animals, Disease Models, Animal, Female, Humans, Male, Mice, Mice, Transgenic (* major topic)
Topic: Alzheimer's disease research and treatments (Physiology, Medicine), according to OpenAlex
Funding: Cure Alzheimer’s Fund; MADRC (1P30AG062421‐01); U.S. Department of Health & Human Services | NIH | National Eye Institute (NEI) (R01‐EY034234); H. Eric Cushing Foundation; Nancy Lurie Marks Family Foundation; C.J.L. charitable Foundation; NIH (R01AG077016)
Citations: not cited yet (Europe PMC); 80 references in the paper

Abstract

The abstract is not reproduced here: the paper's license (CC BY-NC-ND) does not allow it. Read it in the paper, at the publisher or on Europe PMC.

Repository

Its files are read in the Code ↔ Paper reader above, with 1 match between paragraphs and lines of code.

chapmanb/bcbio-nextgen

License: MIT
State: the link answers, verified on 28 September 2026
Evidence: files inventoried
Commit: 594339fd1a9694c8f8ed61084e2bd5740cfd59b0, 24 August 2024
Languages: Python (297), Shell (17), R (2)
Size: 581 files, 316 scripts
Software Heritage: archived
Found in: the text, “Genetic discovery”
Holds: README, license file, environment (requirements-conda.txt, requirements-dev.txt, requirements.txt, setup.cfg, setup.py, docs/requirements-local.txt, docs/requirements.txt), tests, continuous integration, documentation, 1 notebook
Not found: CITATION.cff
Tools: pysam (31 files), BEDTools (27 files), pandas (27 files), NumPy (19 files), Biopython (5 files), SciPy (3 files), DESeq2 (2 files), SAMtools (2 files), statsmodels (2 files), tidyverse (2 files), BCFtools (1 file), ggplot2 (1 file), h5py (1 file), Matplotlib (1 file)
Availability: 1 check, the latest on 28 September 2026: the link answers
  • 28 September 2026: the link answers
318 files
At the source: github.com/chapmanb/bcbio-nextgen

Tracing map

Proposed by the machine: these links were found in the paper and verified at the source, without human review. The map will receive a Zenodo DOI once one of the paper's authors has validated it with their ORCID.

What the map holds:

  • 1 repository of the authors' code, each at its verified commit, with its license and how the link was found in the paper;
  • 316 scripts, each with its path and the digest of its content;
  • 1 match between paragraphs of the paper and lines of the code (method lexical-v1);
  • neither the text of the paper nor the code itself.

Its JSON (tracing-map.json) is deposited on Zenodo with its DOI once the map is validated.

Data

No dataset and no data link were found in the paper.

Versions

The history of this record: each version stored by the harvester or made by a correction of its authors or of the maintainers of its code, and what changed in its facts. The texts of the paper (its abstract, its availability statements) are not part of it; versions that changed only those are not listed.

Version 1, 28 September 2026: the first record

Recorded: type, language, journal, volume, issue, pages, dates, 17 authors, 6 keywords, 13 MeSH terms, 7 funders, 80 references.

Cite

This paper

Miller, S. J., Prokopenko, D., Bai, P., Mondal, P., Scott, A., Zhang, W., Gomm, A., Zhang, S., Child, D. D., Shen, N., Ward, J., Schulte, S., Lei, D., Hafler, B. P., Wang, C., Tanzi, R. E., & Zhang, C. (2026). Intracellular protein GBF1 displays significant associations with amyloid pathology in Alzheimer's disease. Alzheimer's & dementia : the journal of the Alzheimer's Association, 22(4), e71271. https://doi.org/10.1002/alz.71271

BibTeX

@article{miller2026intracellular,
author = {Miller, Sean J. and Prokopenko, Dmitry and Bai, Ping and Mondal, Prasenjit and Scott, Abigael and Zhang, Wei and Gomm, Ashley and Zhang, Siyi and Child, Daniel D. and Shen, Nolan and Ward, Joseph and Schulte, Scott and Lei, Dan and Hafler, Brian P. and Wang, Changning and Tanzi, Rudolph E. and Zhang, Can},
title = {{Intracellular protein GBF1 displays significant associations with amyloid pathology in Alzheimer's disease}},
journal = {Alzheimer's \& dementia : the journal of the Alzheimer's Association},
year = {2026},
month = apr,
volume = {22},
number = {4},
pages = {e71271},
publisher = {Wiley},
issn = {1552-5260},
doi = {10.1002/alz.71271},
url = {https://doi.org/10.1002/alz.71271},
pmid = {41988936},
pmcid = {PMC13084705}
}

RIS

TY - JOUR
AU - Miller, Sean J.
AU - Prokopenko, Dmitry
AU - Bai, Ping
AU - Mondal, Prasenjit
AU - Scott, Abigael
AU - Zhang, Wei
AU - Gomm, Ashley
AU - Zhang, Siyi
AU - Child, Daniel D.
AU - Shen, Nolan
AU - Ward, Joseph
AU - Schulte, Scott
AU - Lei, Dan
AU - Hafler, Brian P.
AU - Wang, Changning
AU - Tanzi, Rudolph E.
AU - Zhang, Can
TI - Intracellular protein GBF1 displays significant associations with amyloid pathology in Alzheimer's disease
T2 - Alzheimer's & dementia : the journal of the Alzheimer's Association
J2 - Alzheimers Dement
PY - 2026
DA - 2026/04/01
VL - 22
IS - 4
SP - e71271
SN - 1552-5260
PB - Wiley
DO - 10.1002/alz.71271
UR - https://doi.org/10.1002/alz.71271
LA - en
ER -

CSL-JSON

{
"id": "10.1002/alz.71271",
"type": "article-journal",
"title": "Intracellular protein GBF1 displays significant associations with amyloid pathology in Alzheimer's disease",
"container-title": "Alzheimer's & dementia : the journal of the Alzheimer's Association",
"author": [
{
"family": "Miller",
"given": "Sean J."
},
{
"family": "Prokopenko",
"given": "Dmitry"
},
{
"family": "Bai",
"given": "Ping"
},
{
"family": "Mondal",
"given": "Prasenjit"
},
{
"family": "Scott",
"given": "Abigael"
},
{
"family": "Zhang",
"given": "Wei"
},
{
"family": "Gomm",
"given": "Ashley"
},
{
"family": "Zhang",
"given": "Siyi"
},
{
"family": "Child",
"given": "Daniel D."
},
{
"family": "Shen",
"given": "Nolan"
},
{
"family": "Ward",
"given": "Joseph"
},
{
"family": "Schulte",
"given": "Scott"
},
{
"family": "Lei",
"given": "Dan"
},
{
"family": "Hafler",
"given": "Brian P."
},
{
"family": "Wang",
"given": "Changning"
},
{
"family": "Tanzi",
"given": "Rudolph E."
},
{
"family": "Zhang",
"given": "Can"
}
],
"container-title-short": "Alzheimers Dement",
"volume": "22",
"issue": "4",
"page": "e71271",
"DOI": "10.1002/alz.71271",
"PMID": "41988936",
"PMCID": "PMC13084705",
"ISSN": "1552-5260",
"publisher": "Wiley",
"URL": "https://doi.org/10.1002/alz.71271",
"language": "en",
"issued": {
"date-parts": [
[
2026,
4,
1
]
]
}
}

The tracing map gets a citation of its own once an author has validated it and it has a DOI.

Similar papers

The papers with a page that share the most with this one: the tools found in their code, their categories, datasets, cited references and authors, the rarest counting most.

[1] doi:10.1038/s41467-026-76675-1 [code]
Long-read proteogenomic atlas of human neuronal differentiation reveals isoform diversity informing neurodevelopmental risk mechanisms.
Journal: Nature communications
In common: pysam, Biopython, BEDTools, 9 other tools
[2] doi:10.1038/s41592-026-03211-w [code]
Spatial isoform sequencing at single-cell resolution reveals cell-type-specific spatial isoform variability in multiple brain cell types.
Journal: Nature methods
In common: pysam, Biopython, BEDTools, 9 other tools, mouse
[3] doi:10.1038/s41467-026-71790-5 [code]
Recurrent DNA break clusters drive replication-stress-induced copy number variants and genome diversification.
Journal: Nature communications
In common: BCFtools, pysam, BEDTools, 8 other tools, mouse, cellular / molecular
[4] doi:10.1038/s41586-026-10512-9 [code]
Astrocyte glucocorticoid receptor signalling restricts neuronal plasticity.
Journal: Nature
In common: pysam, Biopython, BEDTools, 8 other tools, mouse, cellular / molecular
[5] doi:10.21203/rs.3.rs-9927928/v1 [code]
Genome-wide and allele-resolved maps of the radial architecture of the mouse genome
Journal: Research Square (preprint)
In common: BCFtools, pysam, BEDTools, 8 other tools, mouse
[6] doi:10.1126/sciadv.aed2952 [code]
Activation of transposable elements is linked to a region- and cell type-specific interferon response in Parkinson's disease.
Journal: Science advances
In common: pysam, Biopython, BEDTools, 8 other tools, cellular / molecular
[7] doi:10.1007/s00262-026-04390-3 [code]
Identification and prioritisation of tumour antigen candidates from 79 glioblastoma transcriptomes.
Journal: Cancer immunology, immunotherapy : CII
In common: BCFtools, pysam, BEDTools, 8 other tools, cellular / molecular
[8] doi:10.1038/s42003-026-10957-8 [code]
Brain defence by the extracellular matrix protein Cochlin.
Journal: Communications biology
In common: BCFtools, Biopython, SAMtools, 8 other tools, mouse, cellular / molecular
[9] doi:10.1038/s44318-026-00818-9 [code]
FAM134B-mediated ER-phagy degrades APP and suppresses Alzheimer's disease pathology.
Journal: The EMBO journal
In common: Biopython, BEDTools, DESeq2, 7 other tools, Alzheimer's / dementia, mouse, cellular / molecular, 1 reference
[10] doi:10.1093/molbev/msag035 [code]
Mammalian mitochondrial DNA accumulates insertions and deletions with age in energetically demanding tissues.
Journal: Molecular biology and evolution
In common: BCFtools, Biopython, BEDTools, 7 other tools, mouse, cellular / molecular

Contribute

The authors of this paper can claim it, correct its record and validate its tracing map, and the maintainers of its code (its owner, or a public member of its organization) correct what it says of their repository; anyone signed in can ask for its removal. Every request goes to OSCR's own machine, which answers it; your account page follows them.

Sign in with ORCID to claim this paper as one of its authors, correct its record or validate its tracing map: when the paper's metadata lists your ORCID iD, you are recognized at once. Maintainers of its code: sign in with GitHub, then claim the repository on your account page.

Request its removal

To ask OSCR to remove this record, the copies of its authors' scripts or its tracing map, use the removal request page: signed in, you say who you are, what to remove and why, then review and confirm the request. Published rules decide every request (how).

Discussion, reproductions, activity

Discussion: questions and error reports about this paper and its code, from signed-in readers and its authors. It opens with sign-in.

Reproductions: reports from readers who ran the authors' code: what they reproduced, with which environment, commit and data. It opens with sign-in.

Activity: what happens around this paper: new versions of its record, its map's validation, discussions and reproductions. It opens with sign-in.