Build-pipeline resilience fixes: BioMart retries, GENCODE streaming, URL updates

bin/fusioncatcher-build.py

@@ -617,6 +617,14 @@ def format_epilog(self, formatter):
     job.add(_FC_+'get_gencode.py',kind='program')
     job.add('--organism',options.organism,kind='parameter')
 #    job.add('--server',options.ftp_ucsc,kind='parameter')
+    if options.ftp_ensembl_path:
+        try:
+            _ensembl_ver = int(options.ftp_ensembl_path.rstrip('/').split('-')[-1])
+            _gencode_ver = _ensembl_ver - 66
+            if _gencode_ver > 0:
+                job.add('--release',str(_gencode_ver),kind='parameter')
+        except (ValueError, IndexError):
+            pass
     job.add('--output',out_dir,kind='output',checksum='no')
     job.add('',outdir('gencode_genes.txt'),kind='output',command_line='no')
     job.run()

bin/get_biotypes.py

@@ -47,6 +47,7 @@
 import urllib
 import urllib2
 import optparse
+import time
 
 if __name__ == '__main__':
 
@@ -133,39 +134,40 @@
     s = ""
     ns = 0
     server = "http://%s%s" % (options.server,options.server_path)
-    try:
-        req=urllib2.Request(server,mydata,headers)
-        page=urllib2.urlopen(req)
-
-        fid=open(biotypes_filename,'w')
-        size=0
-        while True:
-            part=page.read(CHUNK_SIZE)
-            if not part:
-                break
-            size=size+len(part)
-            fid.write(part)
-            amount=size/float(1024*1024)
-            sys.stdout.write("\b"*ns)
-            sys.stdout.flush()
-            s = "Downloaded: %9.2f MB" % amount
-            ns = len(s)
-            sys.stdout.write(s)
-            sys.stdout.flush()
-        fid.close()
-        print "\nDownloaded: %9.2f MB\n" % amount
-    except urllib2.HTTPError, error:
-        print '\nHTTPError = ' + str(error)
-        sys.exit(1)
-    except urllib2.URLError, error:
-        print '\nURLError = ' + str(error)
-        sys.exit(1)
-    except IOError, error:
-        print '\nIOError = ' + str(error)
-        sys.exit(1)
-    except Exception, error:
-        print "\nError: Generic exception!",str(error)
-        sys.exit(1)
+    MAX_RETRIES = 5
+    for attempt in xrange(MAX_RETRIES):
+        try:
+            req=urllib2.Request(server,mydata,headers)
+            page=urllib2.urlopen(req, timeout=60)
+
+            fid=open(biotypes_filename,'w')
+            size=0
+            while True:
+                part=page.read(CHUNK_SIZE)
+                if not part:
+                    break
+                size=size+len(part)
+                fid.write(part)
+                amount=size/float(1024*1024)
+                sys.stdout.write("\b"*ns)
+                sys.stdout.flush()
+                s = "Downloaded: %9.2f MB" % amount
+                ns = len(s)
+                sys.stdout.write(s)
+                sys.stdout.flush()
+            fid.close()
+            print "\nDownloaded: %9.2f MB\n" % amount
+            break
+        except (urllib2.HTTPError, urllib2.URLError, IOError), error:
+            print '\nWarning: attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES, str(error))
+            if attempt < MAX_RETRIES - 1:
+                time.sleep(60 * (attempt + 1))
+            else:
+                print '\nError: failed after %d attempts, aborting' % (MAX_RETRIES,)
+                sys.exit(1)
+        except Exception, error:
+            print "\nError: Generic exception!",str(error)
+            sys.exit(1)
 
     if options.organism.lower() == "saccharomyces_cerevisiae":
         x = options.organism.upper().split('_')

bin/get_cancer-genes.py

@@ -91,8 +91,11 @@
 
 
 
-    #url = 'http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv'
-    url = 'http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv'
+    # Legacy URLs (Bushman Lab moved their resources path in 2020-2021):
+    #   http://www.bushmanlab.org/assets/doc/allOnco_Feb2017.tsv  (dead)
+    #   http://www.bushmanlab.org/assets/doc/allOnco_May2018.tsv  (dead)
+    # New location and the latest version we found (June 2021):
+    url = 'https://bushmanlab.org/export/geneLists/allOnco_June2021.tsv'
     tmp_file = os.path.join(options.output_directory,'temp_cancer.tsv')
 
     headers = {     'User-agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3',
@@ -127,8 +130,12 @@
             # parse the file with the known fusion genes
             data = set()
 
-            d = [line.upper().rstrip("\r\n").split("\t")[0] for line in file(tmp_file,"r")]
-            d.pop(0) # remove the header
+            # Format change with the June 2021 file: columns are quoted TSV
+            # with a header row, and the gene symbol moved from column 0 to
+            # column 1 (column 0 is now a row index "n"). Strip surrounding
+            # double-quotes that R's write.table(..., quote=TRUE) adds.
+            d = [line.upper().rstrip("\r\n").split("\t")[1].strip('"') for line in file(tmp_file,"r")]
+            d.pop(0) # remove the header ('"SYMBOL"' after strip)
             data = set(d)
 
             print "%d known genes found (using gene symbols)" % (len(data),)

bin/get_chimerdb4.py

@@ -86,7 +86,7 @@ def check(s):
                       action="store",
                       type="string",
                       dest="server",
-                      default = "http://www.kobic.re.kr",
+                      default = "https://www.kobic.re.kr",
                       help="""The ChimerDB 4.0 server from where the known fusion genes are downloaded. Default is '%default'.""")
 
     (options,args) = parser.parse_args()
@@ -115,9 +115,11 @@ def check(s):
 
 
     #http://ercsb.ewha.ac.kr/FusionGene/document/PO_down.xls
-    urls = [('/chimerdb_mirror/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'),
-           ('/chimerdb_mirror/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'),
-           ('/chimerdb_mirror/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')]
+    # KOBIC consolidated its site and dropped the `_mirror` path segment.
+    # The live download endpoint is /chimerdb/downloads (verified 2026-05).
+    urls = [('/chimerdb/downloads?name=ChimerKB4.xlsx', 'chimerdb4kb.txt'),
+           ('/chimerdb/downloads?name=ChimerPub4.xlsx', 'chimerdb4pub.txt'),
+           ('/chimerdb/downloads?name=ChimerSeq4.xlsx','chimerdb4seq.txt')]
 
     headers = { 'User-Agent' : 'Mozilla/5.0' }
 

bin/get_exons_positions.py

@@ -182,72 +182,64 @@ def int2rom(v):
     s = ""
     ns = 0
     server = "http://%s/biomart/martservice" % (options.server,)
-    try:
-        req = urllib2.Request(server,mydata1,headers)
-        page = urllib2.urlopen(req, timeout = 30)
-
-
-        fid=open(temp_exons_filename1,'w')
-        size=0
-        while True:
-            part=page.read(CHUNK_SIZE)
-            if not part:
-                break
-            size=size+len(part)
-            fid.write(part)
-            amount=size/float(1024*1024)
-            sys.stdout.write("\b"*ns)
-            sys.stdout.flush()
-            s = "Downloaded: %9.2f MB" % amount
-            ns = len(s)
-            sys.stdout.write(s)
-            sys.stdout.flush()
-        fid.close()
-        print "\nFINISHED downloading: %9.2f MB\n" % amount
-    except urllib2.HTTPError, error:
-        print '\nHTTPError = ' + str(error)
-        sys.exit(1)
-    except urllib2.URLError, error:
-        print '\nURLError = ' + str(error)
-        sys.exit(1)
-    except IOError, error:
-        print '\nIOError = ' + str(error)
-        sys.exit(1)
-    except Exception, error:
-        print "\nError: Generic exception!",str(error)
-        sys.exit(1)
+    MAX_RETRIES_Q1 = 5
+    for attempt in xrange(MAX_RETRIES_Q1):
+        try:
+            req = urllib2.Request(server,mydata1,headers)
+            page = urllib2.urlopen(req, timeout = 60)
+
+            fid=open(temp_exons_filename1,'w')
+            size=0
+            while True:
+                part=page.read(CHUNK_SIZE)
+                if not part:
+                    break
+                size=size+len(part)
+                fid.write(part)
+                amount=size/float(1024*1024)
+                sys.stdout.write("\b"*ns)
+                sys.stdout.flush()
+                s = "Downloaded: %9.2f MB" % amount
+                ns = len(s)
+                sys.stdout.write(s)
+                sys.stdout.flush()
+            fid.close()
+            print "\nFINISHED downloading: %9.2f MB\n" % amount
+            break
+        except (urllib2.HTTPError, urllib2.URLError, IOError), error:
+            print '\nWarning: chromosome list attempt %d/%d failed: %s' % (attempt+1, MAX_RETRIES_Q1, str(error))
+            if attempt < MAX_RETRIES_Q1 - 1:
+                time.sleep(60 * (attempt + 1))
+            else:
+                print '\nError: chromosome list failed after %d attempts, aborting' % (MAX_RETRIES_Q1,)
+                sys.exit(1)
+        except Exception, error:
+            print "\nError: Generic exception!",str(error)
+            sys.exit(1)
 
     # extract chromosomes
     chroms = sorted(set([line.rstrip('\r\n').split('\t')[-1] for line in file(temp_exons_filename1,'r').readlines() if line.rstrip("\r\n")]))
     chroms = [l for l in chroms if l in chromosomes]
 
-    again = 0
-    while True:
-        again = again + 1
-        if again > 3:
-            sys.exit(1)
-
-        file(temp_exons_filename2,'w').write('')
-
-        try:
-            v = 0
-            for crs in chroms:
-                v = v + 1
-                if v % 10 == 0:
-                    time.sleep(300)
-                time.sleep(10)
-                print "chromosome =",crs
-                mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} )
-                headers = {
-                'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
-                'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
-                'Accept-Language' : 'en-gb,en;q=0.5'
-                }
-
+    MAX_RETRIES = 5
+    file(temp_exons_filename2,'w').write('')
+    v = 0
+    for crs in chroms:
+        v = v + 1
+        if v % 10 == 0:
+            time.sleep(300)
+        time.sleep(10)
+        print "chromosome =",crs
+        for attempt in xrange(MAX_RETRIES):
+            mydata2=urllib.urlencode( {"query" : query2.replace("%%%chromosome%%%",crs)} )
+            headers = {
+            'User-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36',
+            'Accept' : 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
+            'Accept-Language' : 'en-gb,en;q=0.5'
+            }
+            try:
                 req = urllib2.Request(server,mydata2,headers)
-                page = urllib2.urlopen(req, timeout = 30)
-
-
+                page = urllib2.urlopen(req, timeout = 120)
                 fid=open(temp_exons_filename2,'a')
                 size=0
                 while True:
@@ -265,21 +257,17 @@ def int2rom(v):
                     sys.stdout.flush()
                 fid.close()
                 print "\nFINISHED downloading: %9.2f MB\n" % amount
-
-        except urllib2.HTTPError, error:
-            print '\nHTTPError = ' + str(error)
-            continue
-        except urllib2.URLError, error:
-            print '\nURLError = ' + str(error)
-            continue
-        except IOError, error:
-            print '\nIOError = ' + str(error)
-            continue
-        except Exception, error:
-            print "\nError: Generic exception!",str(error)
-            continue
-
-        break
+                break
+            except (urllib2.HTTPError, urllib2.URLError, IOError), error:
+                print '\nWarning: chromosome %s attempt %d/%d failed: %s' % (crs, attempt+1, MAX_RETRIES, str(error))
+                if attempt < MAX_RETRIES - 1:
+                    time.sleep(60 * (attempt + 1))
+                else:
+                    print 'Error: chromosome %s failed after %d attempts, aborting' % (crs, MAX_RETRIES)
+                    sys.exit(1)
+            except Exception, error:
+                print '\nError: chromosome %s unexpected error: %s' % (crs, str(error))
+                sys.exit(1)