diff --git a/bin/fusioncatcher-build.py b/bin/fusioncatcher-build.py index 1528fd6..016adb2 100755 --- a/bin/fusioncatcher-build.py +++ b/bin/fusioncatcher-build.py @@ -617,6 +617,14 @@ def format_epilog(self, formatter): job.add(_FC_+'get_gencode.py',kind='program') job.add('--organism',options.organism,kind='parameter') # job.add('--server',options.ftp_ucsc,kind='parameter') + if options.ftp_ensembl_path: + try: + _ensembl_ver = int(options.ftp_ensembl_path.rstrip('/').split('-')[-1]) + _gencode_ver = _ensembl_ver - 66 + if _gencode_ver > 0: + job.add('--release',str(_gencode_ver),kind='parameter') + except (ValueError, IndexError): + pass job.add('--output',out_dir,kind='output',checksum='no') job.add('',outdir('gencode_genes.txt'),kind='output',command_line='no') job.run() diff --git a/bin/get_biotypes.py b/bin/get_biotypes.py index d51d9e3..98e09f7 100755 --- a/bin/get_biotypes.py +++ b/bin/get_biotypes.py @@ -47,6 +47,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -133,39 +134,40 @@ s = "" ns = 0 server = "http://%s%s" % (options.server,options.server_path) - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - - fid=open(biotypes_filename,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + + fid=open(biotypes_filename,'w') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) if options.organism.lower() == "saccharomyces_cerevisiae": x = options.organism.upper().split('_') diff --git a/bin/get_cancer-genes.py b/bin/get_cancer-genes.py index f90607e..53b904a 100755 --- a/bin/get_cancer-genes.py +++ b/bin/get_cancer-genes.py @@ -91,8 +91,11 @@ - #url = 'http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv' - url = 'http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv' + # Legacy URLs (Bushman Lab moved their resources path in 2020-2021): + # http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv (dead) + # http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv (dead) + # New location and the latest version we found (June 2021): + url = 'https://bushmanlab.org/export/geneLists/allOnco_June2021.tsv' tmp_file = os.path.join(options.output_directory,'temp_cancer.tsv') headers = { 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', @@ -127,8 +130,12 @@ # parse the file with the known fusion genes data = set() - d = [line.upper().rstrip("\r\n").split("\t")[0] for line in file(tmp_file,"r")] - d.pop(0) # remove the header + # Format change with the June 2021 file: columns are quoted TSV + # with a header row, and the gene symbol moved from column 0 to + # column 1 (column 0 is now a row index "n"). Strip surrounding + # double-quotes that R's write.table(..., quote=TRUE) adds. + d = [line.upper().rstrip("\r\n").split("\t")[1].strip('"') for line in file(tmp_file,"r")] + d.pop(0) # remove the header ('"SYMBOL"' after strip) data = set(d) print "%d known genes found (using gene symbols)" % (len(data),) diff --git a/bin/get_chimerdb4.py b/bin/get_chimerdb4.py index d3c15f9..d37a649 100755 --- a/bin/get_chimerdb4.py +++ b/bin/get_chimerdb4.py @@ -86,7 +86,7 @@ def check(s): action="store", type="string", dest="server", - default = "http://www.kobic.re.kr", + default = "https://www.kobic.re.kr", help="""The ChimerDB 4.0 server from where the known fusion genes are downloaded. Default is '%default'.""") (options,args) = parser.parse_args() @@ -115,9 +115,11 @@ def check(s): #http://ercsb.ewha.ac.kr/FusionGene/document/PO_down.xls - urls = [('/chimerdb_mirror/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'), - ('/chimerdb_mirror/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'), - ('/chimerdb_mirror/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')] + # KOBIC consolidated its site and dropped the `_mirror` path segment. + # The live download endpoint is /chimerdb/downloads (verified 2026-05). + urls = [('/chimerdb/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'), + ('/chimerdb/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'), + ('/chimerdb/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')] headers = { 'User-Agent' : 'Mozilla/5.0' } diff --git a/bin/get_exons_positions.py b/bin/get_exons_positions.py index 6e5a5b8..58012f4 100755 --- a/bin/get_exons_positions.py +++ b/bin/get_exons_positions.py @@ -182,72 +182,64 @@ def int2rom(v): s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata1,headers) - page = urllib2.urlopen(req, timeout = 30) - - - fid=open(temp_exons_filename1,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nFINISHED downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES_Q1 = 5 + for attempt in xrange(MAX_RETRIES_Q1): + try: + req = urllib2.Request(server,mydata1,headers) + page = urllib2.urlopen(req, timeout = 60) + + fid=open(temp_exons_filename1,'w') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nFINISHED downloading: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome list attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES_Q1, str(error)) + if attempt < MAX_RETRIES_Q1 - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: chromosome list failed after %d attempts, aborting' % (MAX_RETRIES_Q1,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) # extract chromosomes chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_exons_filename1,'r').readlines() if line.rstrip("\r\n")])) chroms = [l for l in chroms if l in chromosomes] - again = 0 - while True: - again = again + 1 - if again > 3: - sys.exit(1) - - file(temp_exons_filename2,'w').write('') - - try: - v = 0 - for crs in chroms: - v = v + 1 - if v % 10 == 0: - time.sleep(300) - time.sleep(10) - print "chromosome =",crs - mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) - headers = { - 'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', - 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language' : 'en-gb,en;q=0.5' - } - + MAX_RETRIES = 5 + file(temp_exons_filename2,'w').write('') + v = 0 + for crs in chroms: + v = v + 1 + if v % 10 == 0: + time.sleep(300) + time.sleep(10) + print "chromosome =",crs + for attempt in xrange(MAX_RETRIES): + mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) + headers = { + 'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36', + 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', + 'Accept-Language' : 'en-gb,en;q=0.5' + } + try: req = urllib2.Request(server,mydata2,headers) - page = urllib2.urlopen(req, timeout = 30) - - + page = urllib2.urlopen(req, timeout = 120) fid=open(temp_exons_filename2,'a') size=0 while True: @@ -265,21 +257,17 @@ def int2rom(v): sys.stdout.flush() fid.close() print "\nFINISHED downloading: %9.2f MB\n" % amount - - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - continue - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - continue - except IOError, error: - print '\nIOError = ' + str(error) - continue - except Exception, error: - print "\nError: Generic exception!",str(error) - continue - - break + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome %s attempt %d/%d failed: %s' % (crs, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print 'Error: chromosome %s failed after %d attempts, aborting' % (crs, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print '\nError: chromosome %s unexpected error: %s' % (crs, str(error)) + sys.exit(1) diff --git a/bin/get_gencode.py b/bin/get_gencode.py index c757dab..aec07bf 100755 --- a/bin/get_gencode.py +++ b/bin/get_gencode.py @@ -51,6 +51,7 @@ #import concatenate import shutil import StringIO +import time if __name__ == '__main__': @@ -87,6 +88,13 @@ default = "ftp.ebi.ac.uk", help="""The Gencode server from where the gene annotations are downloaded. Default is '%default'.""") + parser.add_option("--release", + action="store", + type="int", + dest="release", + default = 0, + help="""Pin to a specific GENCODE release number (e.g. 48). Default 0 = use latest available.""") + (options,args) = parser.parse_args() # validate options @@ -114,66 +122,91 @@ if org: - url = 'pub/databases/gencode/%s/' % (org,) + base_url = 'pub/databases/gencode/%s/' % (org,) print "Downloading the GTF file of organism '%s' from Gencode!" % (options.organism.lower(),) version = '' nf = None filename= None - try: - ftp = ftplib.FTP(options.server) - print ftp.login() - ftp.cwd(url) - - list_files = ftp.nlst() - - list_files = [el.replace("release_M","").replace("release_","") for el in list_files if el.lower().startswith('release_')] - if options.organism.lower() == 'homo_sapiens': - list_files = sorted([int(el) for el in list_files if el.isdigit() ]) # and el != '24' new "and el!='24'" because is something wrong with release_24 - version = str(list_files[-1]) - last = "release_"+version - filename = "gencode.v%s.annotation.gtf.gz" % (version,) - elif options.organism.lower() == 'mus_musculus': - list_files = sorted([int(el) for el in list_files if el.isdigit() ]) - version = str(list_files[-1]) - last = "release_M"+version - filename = "gencode.vM%s.annotation.gtf.gz" % (version,) - else: - list_files = sorted(list_files) - version = str(list_files[-1]) - last = "release_"+version - filename = "gencode.v%s.annotation.gtf.gz" % (version,) - url = "%s%s" % (url,last) - ftp.cwd(last) - print "cd ",last - print "Downloading: %s/%s/%s" % (options.server,url,filename) - nf = os.path.join(options.output_directory,filename) - fid = open(nf,'wb') - ftp.retrbinary("RETR " + filename, fid.write) - fid.close() - - ftp.close() - except ftplib.all_errors, e: - print 'FTP Error = ' + str(e) - sys.exit(1) - except Exception, e: - print "Error: Generic exception!",str(e) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + url = base_url + try: + ftp = ftplib.FTP(options.server) + print ftp.login() + ftp.cwd(url) + + if options.release: + version = str(options.release) + if options.organism.lower() == 'mus_musculus': + last = "release_M"+version + filename = "gencode.vM%s.annotation.gtf.gz" % (version,) + else: + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + else: + list_files = ftp.nlst() + list_files = [el.replace("release_M","").replace("release_","") for el in list_files if el.lower().startswith('release_')] + if options.organism.lower() == 'homo_sapiens': + list_files = sorted([int(el) for el in list_files if el.isdigit() ]) # and el != '24' new "and el!='24'" because is something wrong with release_24 + version = str(list_files[-1]) + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + elif options.organism.lower() == 'mus_musculus': + list_files = sorted([int(el) for el in list_files if el.isdigit() ]) + version = str(list_files[-1]) + last = "release_M"+version + filename = "gencode.vM%s.annotation.gtf.gz" % (version,) + else: + list_files = sorted(list_files) + version = str(list_files[-1]) + last = "release_"+version + filename = "gencode.v%s.annotation.gtf.gz" % (version,) + url = "%s%s" % (url,last) + ftp.cwd(last) + print "cd ",last + print "Downloading: %s/%s/%s" % (options.server,url,filename) + nf = os.path.join(options.output_directory,filename) + fid = open(nf,'wb') + ftp.retrbinary("RETR " + filename, fid.write) + fid.close() + + ftp.close() + break + except ftplib.all_errors, e: + print '\nWarning: FTP attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(e)) + try: + ftp.close() + except Exception: + pass + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: FTP failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, e: + print "Error: Generic exception!",str(e) + sys.exit(1) print "Decompressing files ..." if filename.endswith('.gz'): - f = gzip.open(nf, 'rb') - file_content = f.read() - f.close() - f = nf[:-3] - fod = open(f,'wb') - fod.write(file_content) - fod.close() + f_in = gzip.open(nf, 'rb') + nf_decompressed = nf[:-3] + f_out = open(nf_decompressed, 'wb') + shutil.copyfileobj(f_in, f_out) + f_in.close() + f_out.close() os.remove(nf) - nf = f + nf = nf_decompressed print "Parsing the GTF file..." - d = [line.split("\t") for line in file(nf,'r').readlines() if (not line.startswith("#")) and line] - d = [(line[0],line[3],line[4],line[6],line[8].partition('gene_name "')[2].partition('"')[0]) for line in d if line[2] == 'gene'] + d = [] + with open(nf, 'r') as _fparse: + for _line in _fparse: + if _line.startswith("#") or not _line.strip(): + continue + _parts = _line.split("\t") + if len(_parts) >= 9 and _parts[2] == 'gene': + d.append((_parts[0],_parts[3],_parts[4],_parts[6],_parts[8].partition('gene_name "')[2].partition('"')[0])) print "%d genes found!" % (len(d),) data = [] diff --git a/bin/get_genes_descriptions.py b/bin/get_genes_descriptions.py index 8102f9c..b185b65 100755 --- a/bin/get_genes_descriptions.py +++ b/bin/get_genes_descriptions.py @@ -49,6 +49,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -125,40 +126,40 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(gene_filename,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(gene_filename,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) if options.organism.lower() == "saccharomyces_cerevisiae": x = options.organism.upper().split('_') diff --git a/bin/get_genes_symbols.py b/bin/get_genes_symbols.py index 687c9ef..a14708c 100755 --- a/bin/get_genes_symbols.py +++ b/bin/get_genes_symbols.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time @@ -132,40 +133,40 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(symbols_filename,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(symbols_filename,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) # keep only the genes which start with ENS data = [line.rstrip('\r\n').split('\t') for line in file(symbols_filename,'r').readlines() if line.rstrip('\r\n')] diff --git a/bin/get_hla2.py b/bin/get_hla2.py index 686ed42..2820dbe 100755 --- a/bin/get_hla2.py +++ b/bin/get_hla2.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -136,36 +137,37 @@ mydata = urllib.urlencode( {"query" : query} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_mitelman.py b/bin/get_mitelman.py index 01be8c4..e6e8590 100755 --- a/bin/get_mitelman.py +++ b/bin/get_mitelman.py @@ -162,8 +162,23 @@ def mysplit(x): if ret: print >>sys.stderr, "Warning: File '%s' is not a valid ZIP file! The output file will be empty!" % (url,) else: - - d = zf.read('mitelman_db/MBCA.TXT.DATA').decode('ascii').splitlines() + + # The Mitelman dataset moved its files from the `mitelman_db/` + # subdirectory to the archive root at some point (observed 2026-05). + # Look up the data file in whichever layout the archive uses. + mbca_path = None + for candidate in ('MBCA.TXT.DATA', 'mitelman_db/MBCA.TXT.DATA'): + try: + zf.getinfo(candidate) + mbca_path = candidate + break + except KeyError: + continue + if mbca_path is None: + print >>sys.stderr, "Warning: ZIP from '%s' does not contain MBCA.TXT.DATA (tried root and mitelman_db/ prefix). The output file will be empty!" % (url,) + d = [] + else: + d = zf.read(mbca_path).decode('ascii').splitlines() if d: #h = d.pop(0) # remove header diff --git a/bin/get_mtrna.py b/bin/get_mtrna.py index e60f656..95a4e70 100755 --- a/bin/get_mtrna.py +++ b/bin/get_mtrna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -131,36 +132,37 @@ mydata = urllib.urlencode( {"query" : query} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES = 5 + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_paralogs.py b/bin/get_paralogs.py index 71f7e3c..8ba764a 100755 --- a/bin/get_paralogs.py +++ b/bin/get_paralogs.py @@ -159,63 +159,13 @@ def int2rom(v): s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req=urllib2.Request(server,mydata1,headers) - page=urllib2.urlopen(req, timeout = 30) - - fid=open(temp_paralogs_filename1,'w') - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nFinished downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + MAX_RETRIES_Q1 = 5 + for attempt in xrange(MAX_RETRIES_Q1): + try: + req=urllib2.Request(server,mydata1,headers) + page=urllib2.urlopen(req, timeout = 60) - # extract chromosomes - chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_paralogs_filename1,'r').readlines() if line.rstrip("\r\n")])) - chroms = [l for l in chroms if l in chromosomes] - file(temp_paralogs_filename2,'w').write('') - - try: - v = 0 - for crs in chroms: - v = v + 1 - if v % 10 == 0: - time.sleep(180) - time.sleep(10) - print "chromosome =",crs - mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) - headers = { - 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', - 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', - 'Accept-Language' : 'en-gb,en;q=0.5' - } - req=urllib2.Request(server,mydata2,headers) - page=urllib2.urlopen(req, timeout = 30) - - fid=open(temp_paralogs_filename2,'a') + fid=open(temp_paralogs_filename1,'w') size=0 while True: part=page.read(CHUNK_SIZE) @@ -232,18 +182,71 @@ def int2rom(v): sys.stdout.flush() fid.close() print "\nFinished downloading: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome list attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES_Q1, str(error)) + if attempt < MAX_RETRIES_Q1 - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: chromosome list failed after %d attempts, aborting' % (MAX_RETRIES_Q1,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) + + # extract chromosomes + chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_paralogs_filename1,'r').readlines() if line.rstrip("\r\n")])) + chroms = [l for l in chroms if l in chromosomes] + file(temp_paralogs_filename2,'w').write('') + + MAX_RETRIES = 5 + v = 0 + for crs in chroms: + v = v + 1 + if v % 10 == 0: + time.sleep(180) + time.sleep(10) + print "chromosome =",crs + mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} ) + headers = { + 'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3', + 'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', + 'Accept-Language' : 'en-gb,en;q=0.5' + } + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata2,headers) + page=urllib2.urlopen(req, timeout = 120) + fid=open(temp_paralogs_filename2,'a') + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nFinished downloading: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: chromosome %s attempt %d/%d failed: %s' % (crs, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + # A missing chromosome would silently produce an incomplete + # paralogs.txt and non-deterministic downstream filtering. + print '\nError: chromosome %s failed after %d attempts, aborting (paralogs.txt must be complete)' % (crs, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print '\nError: chromosome %s unexpected error: %s -- aborting' % (crs, str(error)) + sys.exit(1) # continue as usual diff --git a/bin/get_pcawg.py b/bin/get_pcawg.py index 20f5c9b..c7ecd42 100755 --- a/bin/get_pcawg.py +++ b/bin/get_pcawg.py @@ -86,8 +86,8 @@ def check(s): action="store", type="string", dest="server", - default = "https://dcc.icgc.org", - help="""The ChimerDB 3.0 server from where the known fusion genes are downloaded. Default is '%default'.""") + default = "https://object.genomeinformatics.org", + help="""The ICGC ARGO open-access object store from where the PCAWG fusion table is downloaded. The legacy ICGC DCC portal (dcc.icgc.org) was decommissioned in 2024; the PCAWG transcriptome fusion file was migrated to the icgc25k-open S3-compatible bucket. Default is '%default'.""") (options,args) = parser.parse_args() @@ -106,8 +106,10 @@ def check(s): socket.setdefaulttimeout(timeout) - #http://ercsb.ewha.ac.kr/FusionGene/document/PO_down.xls - url = '/api/v1/download?fn=/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz' + # Legacy URL (dcc.icgc.org decommissioned 2024): + # /api/v1/download?fn=/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz + # New URL on object.genomeinformatics.org (ICGC ARGO icgc25k-open bucket): + url = '/icgc25k-open/PCAWG/transcriptome/fusion/gene.fusions.V1.tsv.gz' headers = { 'User-Agent' : 'Mozilla/5.0' } @@ -135,8 +137,12 @@ def check(s): fuse = set() # parse the PubMed sheet - fi = [e.rstrip("\r\n").split("\t") for e in gzip.open(tmp_file,"r").readlines() if e.rstrip("\r\n")] - fi.pop(0) + try: + fi = [e.rstrip("\r\n").split("\t") for e in gzip.open(tmp_file,"r").readlines() if e.rstrip("\r\n")] + fi.pop(0) + except (IOError, Exception), e: + print >>sys.stderr, "Warning: Cannot process PCAWG download (not a valid gzip): %s" % str(e) + fi = [] for row in fi: tg = row[0].split("->") g1 = tg[0] diff --git a/bin/get_refseq_ensembl.py b/bin/get_refseq_ensembl.py index 06b5eba..caf2fe8 100755 --- a/bin/get_refseq_ensembl.py +++ b/bin/get_refseq_ensembl.py @@ -49,6 +49,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -135,6 +136,7 @@ tmp2 = os.path.join(options.output_directory,'refseq_ids_tmp2.txt') i = 0 + MAX_RETRIES = 5 for (query,tmp) in [(query1,tmp1),(query2,tmp2)]: mydata = urllib.urlencode( {"query" : query} ) headers = { @@ -147,40 +149,39 @@ s = "" ns = 0 server = "http://%s/biomart/martservice" % (options.server,) - try: - req = urllib2.Request(server,mydata,headers) - page = urllib2.urlopen(req) - - - fid = open(tmp,'w') - size = 0 - while True: - part = page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - fid.close() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req = urllib2.Request(server,mydata,headers) + page = urllib2.urlopen(req, timeout=60) + + fid = open(tmp,'w') + size = 0 + while True: + part = page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + fid.close() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: query %d attempt %d/%d failed: %s' % (i, attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: query %d failed after %d attempts, aborting' % (i, MAX_RETRIES) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) d1 = [line.rstrip('\r\n').split('\t') for line in file(tmp1,'r').readlines() if line.rstrip("\r\n")] d2 = [line.rstrip('\r\n').split('\t') for line in file(tmp2,'r').readlines() if line.rstrip("\r\n")] diff --git a/bin/get_rrna.py b/bin/get_rrna.py index cabfd8c..0bfb316 100755 --- a/bin/get_rrna.py +++ b/bin/get_rrna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -175,40 +176,41 @@ fid=open(rrna_filename,'w') server = "http://%s/biomart/martservice" % (options.server,) + MAX_RETRIES = 5 for q in query: mydata=urllib.urlencode( {"query" : q} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close() diff --git a/bin/get_synonyms.py b/bin/get_synonyms.py index 9d8aa13..3992c55 100755 --- a/bin/get_synonyms.py +++ b/bin/get_synonyms.py @@ -135,7 +135,7 @@ print ftp.login() ftp.cwd(url) key = "%s_core_" % (options.organism.lower(),) - organism_dir = [el for el in ftp.nlst() if el.lower().startswith(key) and not el.lower().endswith(".gz")] + organism_dir = [el for el in ftp.nlst() if el.lower().startswith(key) and not el.lower().endswith("37")] if organism_dir and len(organism_dir) == 1: organism_dir = organism_dir[0] magic = organism_dir.lower().split(key)[1] diff --git a/bin/get_trna.py b/bin/get_trna.py index 0a7c9aa..9afd65d 100755 --- a/bin/get_trna.py +++ b/bin/get_trna.py @@ -48,6 +48,7 @@ import urllib import urllib2 import optparse +import time if __name__ == '__main__': @@ -200,39 +201,40 @@ server = "http://%s/biomart/martservice" % (options.server,) + MAX_RETRIES = 5 for q in query: mydata=urllib.urlencode( {"query" : q} ) s = "" ns = 0 - try: - req=urllib2.Request(server,mydata,headers) - page=urllib2.urlopen(req) - size=0 - while True: - part=page.read(CHUNK_SIZE) - if not part: - break - size=size+len(part) - fid.write(part) - amount=size/float(1024*1024) - sys.stdout.write("\b"*ns) - sys.stdout.flush() - s = "Downloaded: %9.2f MB" % amount - ns = len(s) - sys.stdout.write(s) - sys.stdout.flush() - print "\nDownloaded: %9.2f MB\n" % amount - except urllib2.HTTPError, error: - print '\nHTTPError = ' + str(error) - sys.exit(1) - except urllib2.URLError, error: - print '\nURLError = ' + str(error) - sys.exit(1) - except IOError, error: - print '\nIOError = ' + str(error) - sys.exit(1) - except Exception, error: - print "\nError: Generic exception!",str(error) - sys.exit(1) + for attempt in xrange(MAX_RETRIES): + try: + req=urllib2.Request(server,mydata,headers) + page=urllib2.urlopen(req, timeout=60) + size=0 + while True: + part=page.read(CHUNK_SIZE) + if not part: + break + size=size+len(part) + fid.write(part) + amount=size/float(1024*1024) + sys.stdout.write("\b"*ns) + sys.stdout.flush() + s = "Downloaded: %9.2f MB" % amount + ns = len(s) + sys.stdout.write(s) + sys.stdout.flush() + print "\nDownloaded: %9.2f MB\n" % amount + break + except (urllib2.HTTPError, urllib2.URLError, IOError), error: + print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error)) + if attempt < MAX_RETRIES - 1: + time.sleep(60 * (attempt + 1)) + else: + print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,) + sys.exit(1) + except Exception, error: + print "\nError: Generic exception!",str(error) + sys.exit(1) fid.close()