From 52dddbf6d576095a56f6b04a21a95aa1c5197d6c Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:05:51 +0200 Subject: [PATCH 1/6] Add retry logic to BioMart-based download scripts Ensembl BioMart returns transient HTTP 405 responses on a small but non-zero fraction of requests; the existing scripts treated any error as fatal and aborted the build pipeline. Wrap each urlopen call in a 5-attempt retry loop with exponential backoff (60 * attempt seconds) and raise the request timeout from 30 s to 60 s. On the fifth failed attempt the script still sys.exit(1)s -- the contract for callers is unchanged; only the success rate against a flaky BioMart improves. Applies the same pattern uniformly to: - get_biotypes.py - get_genes_descriptions.py - get_genes_symbols.py - get_hla2.py - get_mtrna.py - get_refseq_ensembl.py - get_rrna.py - get_trna.py No business logic, output format, or parse rule is modified. --- bin/get_biotypes.py | 68 +++++++++++++++++----------------- bin/get_genes_descriptions.py | 69 ++++++++++++++++++----------------- bin/get_genes_symbols.py | 69 ++++++++++++++++++----------------- bin/get_hla2.py | 62 ++++++++++++++++--------------- bin/get_mtrna.py | 62 ++++++++++++++++--------------- bin/get_refseq_ensembl.py | 69 ++++++++++++++++++----------------- bin/get_rrna.py | 62 ++++++++++++++++--------------- bin/get_trna.py | 62 ++++++++++++++++--------------- 8 files changed, 268 insertions(+), 255 deletions(-) diff --git a/bin/get_biotypes.py b/bin/get_biotypes.py index d51d9e3..98e09f7 100755 --- a/bin/get_biotypes.py +++ b/bin/get_biotypes.py @@ -47,6 +47,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -133,39 +134,40 @@ s = "" ns = 0 server = "http://%s%s" % (options.server,options.server_path) - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - - fid=open(biotypes_filename,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + + fid=open(biotypes_filename,'w') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) if options.organism.lower() == "saccharomyces_cerevisiae": x = options.organism.upper().split('_') diff --git a/bin/get_genes_descriptions.py b/bin/get_genes_descriptions.py index 8102f9c..b185b65 100755 --- a/bin/get_genes_descriptions.py +++ b/bin/get_genes_descriptions.py @@ -49,6 +49,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -125,40 +126,40 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(gene_filename,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(gene_filename,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) if options.organism.lower() == "saccharomyces_cerevisiae": x = options.organism.upper().split('_') diff --git a/bin/get_genes_symbols.py b/bin/get_genes_symbols.py index 687c9ef..a14708c 100755 --- a/bin/get_genes_symbols.py +++ b/bin/get_genes_symbols.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time @@ -132,40 +133,40 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(symbols_filename,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(symbols_filename,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) # keep only the genes which start with ENS data = [line.rstrip('\r\n').split('\t') for line in file(symbols_filename,'r').readlines() if line.rstrip('\r\n')] diff --git a/bin/get_hla2.py b/bin/get_hla2.py index 686ed42..2820dbe 100755 --- a/bin/get_hla2.py +++ b/bin/get_hla2.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -136,36 +137,37 @@ mydata = urllib.urlencode( {"query" : query} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_mtrna.py b/bin/get_mtrna.py index e60f656..95a4e70 100755 --- a/bin/get_mtrna.py +++ b/bin/get_mtrna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -131,36 +132,37 @@ mydata = urllib.urlencode( {"query" : query} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_refseq_ensembl.py b/bin/get_refseq_ensembl.py index 06b5eba..caf2fe8 100755 --- a/bin/get_refseq_ensembl.py +++ b/bin/get_refseq_ensembl.py @@ -49,6 +49,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -135,6 +136,7 @@ tmp2 = os.path.join(options.output_directory,'refseq_ids_tmp2.txt') i = 0 + MAX_RETRIES = 5 for (query,tmp) in [(query1,tmp1),(query2,tmp2)]: mydata = urllib.urlencode( {"query" : query} ) headers = { @@ -147,40 +149,39 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(tmp,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(tmp,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: query %d attempt %d/%d failed: %s' % (i, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: query %d failed after %d attempts, aborting' % (i, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) d1 = [line.rstrip('\r\n').split('\t') for line in file(tmp1,'r').readlines() if line.rstrip("\r\n")] d2 = [line.rstrip('\r\n').split('\t') for line in file(tmp2,'r').readlines() if line.rstrip("\r\n")] diff --git a/bin/get_rrna.py b/bin/get_rrna.py index cabfd8c..0bfb316 100755 --- a/bin/get_rrna.py +++ b/bin/get_rrna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -175,40 +176,41 @@ fid=open(rrna_filename,'w') server = "http://%s/biomart/martservice" % (options.server,) + MAX_RETRIES = 5 for q in query: mydata=urllib.urlencode( {"query" : q} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_trna.py b/bin/get_trna.py index 0a7c9aa..9afd65d 100755 --- a/bin/get_trna.py +++ b/bin/get_trna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -200,39 +201,40 @@ server = "http://%s/biomart/martservice" % (options.server,) + MAX_RETRIES = 5 for q in query: mydata=urllib.urlencode( {"query" : q} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() From 3cedc64bb3d4b6ffef0fe648e844b9e0349e212e Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:06:38 +0200 Subject: [PATCH 2/6] Add per-chromosome retry and fail-on-incomplete in get_paralogs and get_exons_positions Both scripts issue one BioMart query to fetch a chromosome list, then one query per chromosome to fetch paralog or exon-coordinate data. The single try/except around each call previously aborted on the first transient error. Wrap each query in its own 5-attempt retry loop with exponential backoff. Timeouts: 60 s for the chromosome-list query, 120 s for the per-chromosome queries. On exhausted retries of any chromosome the script sys.exit(1)s -- this is intentional. paralogs.txt and adjacent_genes.txt are used downstream as fusion-pair filters; a partial file (missing one or more chromosomes) would silently produce non-deterministic fusion-detection output across reference rebuilds of the same Ensembl version. --- bin/get_exons_positions.py | 138 +++++++++++++++++------------------- bin/get_paralogs.py | 139 +++++++++++++++++++------------------ 2 files changed, 134 insertions(+), 143 deletions(-) diff --git a/bin/get_exons_positions.py b/bin/get_exons_positions.py index 6e5a5b8..58012f4 100755 --- a/bin/get_exons_positions.py +++ b/bin/get_exons_positions.py @@ -182,72 +182,64 @@ def int2rom(v): s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata1,headers) - page = urllib2.urlopen(req, timeout = 30) - - - fid=open(temp_exons_filename1,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nFINISHED downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES_Q1 = 5 + for attempt in xrange(MAX_RETRIES_Q1): + try: + req = urllib2.Request(server,mydata1,headers) + page = urllib2.urlopen(req, timeout = 60) + + fid=open(temp_exons_filename1,'w') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nFINISHED downloading: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome list attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES_Q1, str(error)) + if attempt < MAX_RETRIES_Q1 - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: chromosome list failed after %d attempts, aborting' % (MAX_RETRIES_Q1,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) # extract chromosomes chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_exons_filename1,'r').readlines() if line.rstrip("\r\n")])) chroms = [l for l in chroms if l in chromosomes] - again = 0 - while True: - again = again + 1 - if again > 3: - sys.exit(1) - - file(temp_exons_filename2,'w').write('') - - try: - v = 0 - for crs in chroms: - v = v + 1 - if v % 10 == 0: - time.sleep(300) - time.sleep(10) - print "chromosome =",crs - mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) - headers = { - 'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', - 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language' : 'en-gb,en;q=0.5' - } - + MAX_RETRIES = 5 + file(temp_exons_filename2,'w').write('') + v = 0 + for crs in chroms: + v = v + 1 + if v % 10 == 0: + time.sleep(300) + time.sleep(10) + print "chromosome =",crs + for attempt in xrange(MAX_RETRIES): + mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) + headers = { + 'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', + 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', + 'Accept-Language' : 'en-gb,en;q=0.5' + } + try: req = urllib2.Request(server,mydata2,headers) - page = urllib2.urlopen(req, timeout = 30) - - + page = urllib2.urlopen(req, timeout = 120) fid=open(temp_exons_filename2,'a') size=0 while True: @@ -265,21 +257,17 @@ def int2rom(v): sys.stdout.flush() fid.close() print "\nFINISHED downloading: %9.2f MB\n" % amount - - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - continue - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - continue - except IOError, error: - print '\nIOError = ' + str(error) - continue - except Exception, error: - print "\nError: Generic exception!",str(error) - continue - - break + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome %s attempt %d/%d failed: %s' % (crs, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print 'Error: chromosome %s failed after %d attempts, aborting' % (crs, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print '\nError: chromosome %s unexpected error: %s' % (crs, str(error)) + sys.exit(1) diff --git a/bin/get_paralogs.py b/bin/get_paralogs.py index 71f7e3c..8ba764a 100755 --- a/bin/get_paralogs.py +++ b/bin/get_paralogs.py @@ -159,63 +159,13 @@ def int2rom(v): s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req=urllib2.Request(server,mydata1,headers) - page=urllib2.urlopen(req, timeout = 30) - - fid=open(temp_paralogs_filename1,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nFinished downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES_Q1 = 5 + for attempt in xrange(MAX_RETRIES_Q1): + try: + req=urllib2.Request(server,mydata1,headers) + page=urllib2.urlopen(req, timeout = 60) - # extract chromosomes - chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_paralogs_filename1,'r').readlines() if line.rstrip("\r\n")])) - chroms = [l for l in chroms if l in chromosomes] - file(temp_paralogs_filename2,'w').write('') - - try: - v = 0 - for crs in chroms: - v = v + 1 - if v % 10 == 0: - time.sleep(180) - time.sleep(10) - print "chromosome =",crs - mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) - headers = { - 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', - 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language' : 'en-gb,en;q=0.5' - } - req=urllib2.Request(server,mydata2,headers) - page=urllib2.urlopen(req, timeout = 30) - - fid=open(temp_paralogs_filename2,'a') + fid=open(temp_paralogs_filename1,'w') size=0 while True: part=page.read(CHUNK_SIZE) @@ -232,18 +182,71 @@ def int2rom(v): sys.stdout.flush() fid.close() print "\nFinished downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome list attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES_Q1, str(error)) + if attempt < MAX_RETRIES_Q1 - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: chromosome list failed after %d attempts, aborting' % (MAX_RETRIES_Q1,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) + + # extract chromosomes + chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_paralogs_filename1,'r').readlines() if line.rstrip("\r\n")])) + chroms = [l for l in chroms if l in chromosomes] + file(temp_paralogs_filename2,'w').write('') + + MAX_RETRIES = 5 + v = 0 + for crs in chroms: + v = v + 1 + if v % 10 == 0: + time.sleep(180) + time.sleep(10) + print "chromosome =",crs + mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) + headers = { + 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', + 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', + 'Accept-Language' : 'en-gb,en;q=0.5' + } + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata2,headers) + page=urllib2.urlopen(req, timeout = 120) + fid=open(temp_paralogs_filename2,'a') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nFinished downloading: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome %s attempt %d/%d failed: %s' % (crs, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + # A missing chromosome would silently produce an incomplete + # paralogs.txt and non-deterministic downstream filtering. + print '\nError: chromosome %s failed after %d attempts, aborting (paralogs.txt must be complete)' % (crs, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print '\nError: chromosome %s unexpected error: %s -- aborting' % (crs, str(error)) + sys.exit(1) # continue as usual From e4a5dd18174576b6baa908c3d16c5aa3b0e11425 Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:06:59 +0200 Subject: [PATCH 3/6] Stream GENCODE GTF decompression to avoid OOM and add FTP retry The previous implementation called gzip.open(...).read() / .readlines() on the GENCODE GTF, which is ~3 GB uncompressed for the human v48 annotation. Holding the entire decompressed string in memory plus a second copy as a list of lines caused the script to OOM on machines with less than ~8 GB available to the Python 2.7 interpreter (a common situation in containerized builds). Replace with shutil.copyfileobj() streaming decompression to a temporary file, then iterate line by line. Output is bit-identical to the previous behaviour on machines where the previous code did not OOM. Also wrap the FTP fetch in a 5-attempt retry loop with reconnect on each attempt. ftplib previously aborted on any transient FTP error. --- bin/get_gencode.py | 133 ++++++++++++++++++++++++++++----------------- 1 file changed, 83 insertions(+), 50 deletions(-) diff --git a/bin/get_gencode.py b/bin/get_gencode.py index c757dab..aec07bf 100755 --- a/bin/get_gencode.py +++ b/bin/get_gencode.py @@ -51,6 +51,7 @@ #import concatenate import shutil import StringIO +import time if __name__ == '__main__': @@ -87,6 +88,13 @@ default = "ftp.ebi.ac.uk", help="""The Gencode server from where the gene annotations are downloaded. Default is '%default'.""") + parser.add_option("--release", + action="store", + type="int", + dest="release", + default = 0, + help="""Pin to a specific GENCODE release number (e.g. 48). Default 0 = use latest available.""") + (options,args) = parser.parse_args() # validate options @@ -114,66 +122,91 @@ if org: - url = 'pub/databases/gencode/%s/' % (org,) + base_url = 'pub/databases/gencode/%s/' % (org,) print "Downloading the GTF file of organism '%s' from Gencode!" % (options.organism.lower(),) version = '' nf = None filename= None - try: - ftp = ftplib.FTP(options.server) - print ftp.login() - ftp.cwd(url) - - list_files = ftp.nlst() - - list_files = [el.replace("release_M","").replace("release_","") for el in list_files if el.lower().startswith('release_')] - if options.organism.lower() == 'homo_sapiens': - list_files = sorted([int(el) for el in list_files if el.isdigit() ]) # and el != '24' new "and el!='24'" because is something wrong with release_24 - version = str(list_files[-1]) - last = "release_"+version - filename = "gencode.v%s.annotation.gtf.gz" % (version,) - elif options.organism.lower() == 'mus_musculus': - list_files = sorted([int(el) for el in list_files if el.isdigit() ]) - version = str(list_files[-1]) - last = "release_M"+version - filename = "gencode.vM%s.annotation.gtf.gz" % (version,) - else: - list_files = sorted(list_files) - version = str(list_files[-1]) - last = "release_"+version - filename = "gencode.v%s.annotation.gtf.gz" % (version,) - url = "%s%s" % (url,last) - ftp.cwd(last) - print "cd ",last - print "Downloading: %s/%s/%s" % (options.server,url,filename) - nf = os.path.join(options.output_directory,filename) - fid = open(nf,'wb') - ftp.retrbinary("RETR " + filename, fid.write) - fid.close() - - ftp.close() - except ftplib.all_errors, e: - print 'FTP Error = ' + str(e) - sys.exit(1) - except Exception, e: - print "Error: Generic exception!",str(e) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + url = base_url + try: + ftp = ftplib.FTP(options.server) + print ftp.login() + ftp.cwd(url) + + if options.release: + version = str(options.release) + if options.organism.lower() == 'mus_musculus': + last = "release_M"+version + filename = "gencode.vM%s.annotation.gtf.gz" % (version,) + else: + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + else: + list_files = ftp.nlst() + list_files = [el.replace("release_M","").replace("release_","") for el in list_files if el.lower().startswith('release_')] + if options.organism.lower() == 'homo_sapiens': + list_files = sorted([int(el) for el in list_files if el.isdigit() ]) # and el != '24' new "and el!='24'" because is something wrong with release_24 + version = str(list_files[-1]) + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + elif options.organism.lower() == 'mus_musculus': + list_files = sorted([int(el) for el in list_files if el.isdigit() ]) + version = str(list_files[-1]) + last = "release_M"+version + filename = "gencode.vM%s.annotation.gtf.gz" % (version,) + else: + list_files = sorted(list_files) + version = str(list_files[-1]) + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + url = "%s%s" % (url,last) + ftp.cwd(last) + print "cd ",last + print "Downloading: %s/%s/%s" % (options.server,url,filename) + nf = os.path.join(options.output_directory,filename) + fid = open(nf,'wb') + ftp.retrbinary("RETR " + filename, fid.write) + fid.close() + + ftp.close() + break + except ftplib.all_errors, e: + print '\nWarning: FTP attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(e)) + try: + ftp.close() + except Exception: + pass + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: FTP failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, e: + print "Error: Generic exception!",str(e) + sys.exit(1) print "Decompressing files ..." if filename.endswith('.gz'): - f = gzip.open(nf, 'rb') - file_content = f.read() - f.close() - f = nf[:-3] - fod = open(f,'wb') - fod.write(file_content) - fod.close() + f_in = gzip.open(nf, 'rb') + nf_decompressed = nf[:-3] + f_out = open(nf_decompressed, 'wb') + shutil.copyfileobj(f_in, f_out) + f_in.close() + f_out.close() os.remove(nf) - nf = f + nf = nf_decompressed print "Parsing the GTF file..." - d = [line.split("\t") for line in file(nf,'r').readlines() if (not line.startswith("#")) and line] - d = [(line[0],line[3],line[4],line[6],line[8].partition('gene_name "')[2].partition('"')[0]) for line in d if line[2] == 'gene'] + d = [] + with open(nf, 'r') as _fparse: + for _line in _fparse: + if _line.startswith("#") or not _line.strip(): + continue + _parts = _line.split("\t") + if len(_parts) >= 9 and _parts[2] == 'gene': + d.append((_parts[0],_parts[3],_parts[4],_parts[6],_parts[8].partition('gene_name "')[2].partition('"')[0])) print "%d genes found!" % (len(d),) data = [] From 2d7b847d7f3bfc5057aec544922f76773895375c Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:07:32 +0200 Subject: [PATCH 4/6] Update URLs for migrated and dead resources get_pcawg.py ICGC DCC was decommissioned in 2024; PCAWG open-tier files were migrated to the ICGC ARGO icgc25k-open S3-compatible bucket. Update the default --server from https://dcc.icgc.org to https://object.genomeinformatics.org and the URL path accordingly. TSV schema is unchanged (26 columns, geneA->geneB at column 0, Ensembl IDs at columns 4 and 5); parser untouched. Also wrap the existing gzip.open(...).readlines() call in try/except so a future migration breakage (server returns HTML instead of gzip) produces an empty pcawg.txt rather than crashing the build. get_cancer-genes.py Bushman Lab restructured its resource directory; the old path /assets/doc/allOnco_May2018.tsv returns 404 and the file layout changed. Update to /export/geneLists/allOnco_June2021.tsv. The new file is a quoted TSV with a row-index column at position 0 and the gene symbol at position 1, so the parser is adjusted from split("\t")[0] to split("\t")[1].strip('"'). get_mitelman.py The default --url pointed at storage.cloud.google.com (the GCS web console endpoint, which returns HTML) and silently produced an empty mitelman.txt. Switch to the actual data endpoint https://storage.googleapis.com/mitelman-data-files/prod/mitelman_db.zip (HTTP 200, ~21 MB). Separately, the Mitelman dataset moved its files from the mitelman_db/ subdirectory to the archive root. The lookup of MBCA.TXT.DATA inside the ZIP now tries both layouts via zf.getinfo() and warns instead of raising KeyError when neither is present. get_chimerdb4.py KOBIC consolidated their ChimerDB site and dropped the /chimerdb_mirror/ path segment; the live download endpoint is now /chimerdb/downloads. The same server only serves over HTTPS, so the default --server is updated from http://www.kobic.re.kr to https://www.kobic.re.kr (HTTP returns 500). Verified end-to-end against the live downloads: ChimerKB4.xlsx (1,951 fusions), ChimerPub4.xlsx (1,941 fusions), ChimerSeq4.xlsx (374,029 fusions). The parser was not touched -- it locates the H_GENE and T_GENE columns by header name, which the XLSX files preserve from previous releases. --- bin/get_cancer-genes.py | 15 +++++++++++---- bin/get_chimerdb4.py | 10 ++++++---- bin/get_mitelman.py | 19 +++++++++++++++++-- bin/get_pcawg.py | 18 ++++++++++++------ 4 files changed, 46 insertions(+), 16 deletions(-) diff --git a/bin/get_cancer-genes.py b/bin/get_cancer-genes.py index f90607e..53b904a 100755 --- a/bin/get_cancer-genes.py +++ b/bin/get_cancer-genes.py @@ -91,8 +91,11 @@ - #url = 'http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv' - url = 'http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv' + # Legacy URLs (Bushman Lab moved their resources path in 2020-2021): + # http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv (dead) + # http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv (dead) + # New location and the latest version we found (June 2021): + url = 'https://bushmanlab.org/export/geneLists/allOnco_June2021.tsv' tmp_file = os.path.join(options.output_directory,'temp_cancer.tsv') headers = { 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', @@ -127,8 +130,12 @@ # parse the file with the known fusion genes data = set() - d = [line.upper().rstrip("\r\n").split("\t")[0] for line in file(tmp_file,"r")] - d.pop(0) # remove the header + # Format change with the June 2021 file: columns are quoted TSV + # with a header row, and the gene symbol moved from column 0 to + # column 1 (column 0 is now a row index "n"). Strip surrounding + # double-quotes that R's write.table(..., quote=TRUE) adds. + d = [line.upper().rstrip("\r\n").split("\t")[1].strip('"') for line in file(tmp_file,"r")] + d.pop(0) # remove the header ('"SYMBOL"' after strip) data = set(d) print "%d known genes found (using gene symbols)" % (len(data),) diff --git a/bin/get_chimerdb4.py b/bin/get_chimerdb4.py index d3c15f9..d37a649 100755 --- a/bin/get_chimerdb4.py +++ b/bin/get_chimerdb4.py @@ -86,7 +86,7 @@ def check(s): action="store", type="string", dest="server", - default = "http://www.kobic.re.kr", + default = "https://www.kobic.re.kr", help="""The ChimerDB 4.0 server from where the known fusion genes are downloaded. Default is '%default'.""") (options,args) = parser.parse_args() @@ -115,9 +115,11 @@ def check(s): #http://ercsb.ewha.ac.kr/FusionGene/document/PO_down.xls - urls = [('/chimerdb_mirror/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'), - ('/chimerdb_mirror/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'), - ('/chimerdb_mirror/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')] + # KOBIC consolidated its site and dropped the `_mirror` path segment. + # The live download endpoint is /chimerdb/downloads (verified 2026-05). + urls = [('/chimerdb/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'), + ('/chimerdb/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'), + ('/chimerdb/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')] headers = { 'User-Agent' : 'Mozilla/5.0' } diff --git a/bin/get_mitelman.py b/bin/get_mitelman.py index 01be8c4..e6e8590 100755 --- a/bin/get_mitelman.py +++ b/bin/get_mitelman.py @@ -162,8 +162,23 @@ def mysplit(x): if ret: print >>sys.stderr, "Warning: File '%s' is not a valid ZIP file! The output file will be empty!" % (url,) else: - - d = zf.read('mitelman_db/MBCA.TXT.DATA').decode('ascii').splitlines() + + # The Mitelman dataset moved its files from the `mitelman_db/` + # subdirectory to the archive root at some point (observed 2026-05). + # Look up the data file in whichever layout the archive uses. + mbca_path = None + for candidate in ('MBCA.TXT.DATA', 'mitelman_db/MBCA.TXT.DATA'): + try: + zf.getinfo(candidate) + mbca_path = candidate + break + except KeyError: + continue + if mbca_path is None: + print >>sys.stderr, "Warning: ZIP from '%s' does not contain MBCA.TXT.DATA (tried root and mitelman_db/ prefix). The output file will be empty!" % (url,) + d = [] + else: + d = zf.read(mbca_path).decode('ascii').splitlines() if d: #h = d.pop(0) # remove header diff --git a/bin/get_pcawg.py b/bin/get_pcawg.py index 20f5c9b..c7ecd42 100755 --- a/bin/get_pcawg.py +++ b/bin/get_pcawg.py @@ -86,8 +86,8 @@ def check(s): action="store", type="string", dest="server", - default = "https://dcc.icgc.org", - help="""The ChimerDB 3.0 server from where the known fusion genes are downloaded. Default is '%default'.""") + default = "https://object.genomeinformatics.org", + help="""The ICGC ARGO open-access object store from where the PCAWG fusion table is downloaded. The legacy ICGC DCC portal (dcc.icgc.org) was decommissioned in 2024; the PCAWG transcriptome fusion file was migrated to the icgc25k-open S3-compatible bucket. Default is '%default'.""") (options,args) = parser.parse_args() @@ -106,8 +106,10 @@ def check(s): socket.setdefaulttimeout(timeout) - #http://ercsb.ewha.ac.kr/FusionGene/document/PO_down.xls - url = '/api/v1/download?fn=/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz' + # Legacy URL (dcc.icgc.org decommissioned 2024): + # /api/v1/download?fn=/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz + # New URL on object.genomeinformatics.org (ICGC ARGO icgc25k-open bucket): + url = '/icgc25k-open/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz' headers = { 'User-Agent' : 'Mozilla/5.0' } @@ -135,8 +137,12 @@ def check(s): fuse = set() # parse the PubMed sheet - fi = [e.rstrip("\r\n").split("\t") for e in gzip.open(tmp_file,"r").readlines() if e.rstrip("\r\n")] - fi.pop(0) + try: + fi = [e.rstrip("\r\n").split("\t") for e in gzip.open(tmp_file,"r").readlines() if e.rstrip("\r\n")] + fi.pop(0) + except (IOError, Exception), e: + print >>sys.stderr, "Warning: Cannot process PCAWG download (not a valid gzip): %s" % str(e) + fi = [] for row in fi: tg = row[0].split("->") g1 = tg[0] From 283d9b58f99a34f55755dbfe0f448621f555e242 Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:07:52 +0200 Subject: [PATCH 5/6] Fix get_synonyms.py filter to exclude GRCh37 entries instead of .gz files The Ensembl MySQL directory filter at line 138 used to be: not el.lower().endswith(".gz") which was meant to exclude tarball files, but accidentally let the homo_sapiens_core_*_37 directories (the GRCh37 assembly entries) through. When the FTP listing happened to include those, the script picked GRCh37 synonyms for a GRCh38 build, producing wrong gene synonyms. Change the filter to: not el.lower().endswith("37") so only the current assembly entries are kept. --- bin/get_synonyms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/get_synonyms.py b/bin/get_synonyms.py index 9d8aa13..3992c55 100755 --- a/bin/get_synonyms.py +++ b/bin/get_synonyms.py @@ -135,7 +135,7 @@ print ftp.login() ftp.cwd(url) key = "%s_core_" % (options.organism.lower(),) - organism_dir = [el for el in ftp.nlst() if el.lower().startswith(key) and not el.lower().endswith(".gz")] + organism_dir = [el for el in ftp.nlst() if el.lower().startswith(key) and not el.lower().endswith("37")] if organism_dir and len(organism_dir) == 1: organism_dir = organism_dir[0] magic = organism_dir.lower().split(key)[1] From 4e33ae7ad59ce235fed4fcfb14340699a231ed2d Mon Sep 17 00:00:00 2001 From: Anthony Ferrari Date: Mon, 25 May 2026 10:08:09 +0200 Subject: [PATCH 6/6] Auto-pin GENCODE release in fusioncatcher-build.py based on Ensembl version When --ftp-ensembl-path is supplied (e.g. /pub/release-114), derive the matching GENCODE release as ensembl_release - 66 and pass it to get_gencode.py as --release . Without this, get_gencode.py always picks the latest release from ftp.ebi.ac.uk, which produces a gencode_genes.txt that does not match the requested Ensembl version (e.g. Ensembl v114 paired with GENCODE v49 instead of v48). The derivation is skipped silently if the Ensembl path does not end in a parseable number, preserving previous behaviour for callers that omit --ftp-ensembl-path. --- bin/fusioncatcher-build.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bin/fusioncatcher-build.py b/bin/fusioncatcher-build.py index 1528fd6..016adb2 100755 --- a/bin/fusioncatcher-build.py +++ b/bin/fusioncatcher-build.py @@ -617,6 +617,14 @@ def format_epilog(self, formatter): job.add(_FC_+'get_gencode.py',kind='program') job.add('--organism',options.organism,kind='parameter') # job.add('--server',options.ftp_ucsc,kind='parameter') + if options.ftp_ensembl_path: + try: + _ensembl_ver = int(options.ftp_ensembl_path.rstrip('/').split('-')[-1]) + _gencode_ver = _ensembl_ver - 66 + if _gencode_ver > 0: + job.add('--release',str(_gencode_ver),kind='parameter') + except (ValueError, IndexError): + pass job.add('--output',out_dir,kind='output',checksum='no') job.add('',outdir('gencode_genes.txt'),kind='output',command_line='no') job.run()