From 8c2d093a700b213446a05e4265da8be866cd698e Mon Sep 17 00:00:00 2001 From: froggleston Date: Thu, 31 Aug 2017 13:55:41 +0100 Subject: [PATCH 01/18] First stab at status reporting for aspera --- python/utils.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/python/utils.py b/python/utils.py index 7d85652..012c6db 100644 --- a/python/utils.py +++ b/python/utils.py @@ -26,6 +26,8 @@ import sys import urllib import urllib2 +import pexpect +from datetime import datetime import xml.etree.ElementTree as ElementTree from ConfigParser import SafeConfigParser @@ -400,13 +402,102 @@ def asperaretrieve(url, dest_dir, dest_file): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - print aspera_line - subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True + #print aspera_line + #subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True + _do_aspera_transfer(aspera_line) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False +def _do_aspera_transfer(cmd): + try: + sub_id = base64.b64encode(cmd) + thread = pexpect.spawn(cmd, timeout=None) + #thread.expect(["assword:", pexpect.EOF]) + #thread.sendline(password) + + cpl = thread.compile_pattern_list([pexpect.EOF, '(.+)']) + + while True: + i = thread.expect_list(cpl, timeout=None) + if i == 0: # EOF! Possible error point if encountered before transfer completion + print("Process termination - check exit status!") + break + elif i == 1: + pexp_match = thread.match.group(1) + prev_file = '' + tokens_to_match = ["Mb/s"] + units_to_match = ["KB", "MB"] + time_units = ['d', 'h', 'm', 's'] + end_of_transfer = False + + if all(tm in pexp_match.decode("utf-8") for tm in tokens_to_match): + fields = { + "transfer_status": "transferring", + "current_time": datetime.now().strftime("%d-%m-%Y %H:%M:%S") + } + + tokens = pexp_match.decode("utf-8").split(" ") + + #lg.log(tokens, level=Loglvl.INFO, type=Logtype.FILE) + + for token in tokens: + if not token == '': + if "file" in token: + fields['file_path'] = token.split('=')[-1] + if prev_file != fields['file_path']: + k = k + 1 + prev_file == fields['file_path'] + elif '%' in token: + pct = float((token.rstrip("%"))) + # pct = (1/len(file_path) * pct) + (k * 1/len(file_path) * 100) + fields['pct_completed'] = pct + # flag end of transfer + print(str(pct) + '% transfered') + if token.rstrip("%") == 100: + end_of_transfer = True + elif any(um in token for um in units_to_match): + fields['amt_transferred'] = token + elif "Mb/s" in token or "Mbps" in token: + t = token[:-4] + if '=' in t: + fields['transfer_rate'] = t[t.find('=') + 1:] + else: + fields['transfer_rate'] = t + elif "status" in token: + fields['transfer_status'] = token.split('=')[-1] + elif "rate" in token: + fields['transfer_rate'] = token.split('=')[-1] + elif "elapsed" in token: + fields['elapsed_time'] = token.split('=')[-1] + elif "loss" in token: + fields['bytes_lost'] = token.split('=')[-1] + elif "size" in token: + fields['file_size_bytes'] = token.split('=')[-1] + + elif "ETA" in token: + eta = tokens[-2] + estimated_completion = "" + eta_split = eta.split(":") + t_u = time_units[-len(eta_split):] + for indx, eta_token in enumerate(eta.split(":")): + if eta_token == "00": + continue + estimated_completion += eta_token + t_u[indx] + " " + fields['estimated_completion'] = estimated_completion + + kwargs = dict(target_id=sub_id, completed_on=datetime.now()) + #Submission().save_record(dict(), **kwargs) + # close thread + thread.close() + print 'Aspera Transfer completed' + + except OSError: + print "Error" + finally: + pass + def get_wgs_file_ext(output_format): if output_format == EMBL_FORMAT: return WGS_EMBL_EXT From 8524adc657658166a94669e445378b40bdf576a8 Mon Sep 17 00:00:00 2001 From: froggleston Date: Fri, 6 Oct 2017 17:27:52 +0100 Subject: [PATCH 02/18] Add support for an aspera stdout redirect handler --- python/utils.py | 105 +++++++----------------------------------------- 1 file changed, 14 insertions(+), 91 deletions(-) diff --git a/python/utils.py b/python/utils.py index 012c6db..1058799 100644 --- a/python/utils.py +++ b/python/utils.py @@ -22,13 +22,13 @@ import hashlib import re import os -import subprocess import sys import urllib import urllib2 import pexpect from datetime import datetime import xml.etree.ElementTree as ElementTree +from subprocess import call, Popen, PIPE from ConfigParser import SafeConfigParser @@ -386,7 +386,7 @@ def set_aspera(aspera_filepath): aspera = False return aspera -def asperaretrieve(url, dest_dir, dest_file): +def asperaretrieve(url, dest_dir, dest_file, handler=None): try: if not os.path.exists(ASPERA_BIN) or not os.path.exists(ASPERA_PRIVATE_KEY): raise FileNotFoundError('Aspera not available. Check your ascp binary path and your private key file is specified correctly') @@ -402,101 +402,24 @@ def asperaretrieve(url, dest_dir, dest_file): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - #print aspera_line + #sys.stdout.write(aspera_line) #subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True - _do_aspera_transfer(aspera_line) + _do_aspera_transfer(aspera_line, handler) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False -def _do_aspera_transfer(cmd): - try: - sub_id = base64.b64encode(cmd) - thread = pexpect.spawn(cmd, timeout=None) - #thread.expect(["assword:", pexpect.EOF]) - #thread.sendline(password) - - cpl = thread.compile_pattern_list([pexpect.EOF, '(.+)']) - - while True: - i = thread.expect_list(cpl, timeout=None) - if i == 0: # EOF! Possible error point if encountered before transfer completion - print("Process termination - check exit status!") - break - elif i == 1: - pexp_match = thread.match.group(1) - prev_file = '' - tokens_to_match = ["Mb/s"] - units_to_match = ["KB", "MB"] - time_units = ['d', 'h', 'm', 's'] - end_of_transfer = False - - if all(tm in pexp_match.decode("utf-8") for tm in tokens_to_match): - fields = { - "transfer_status": "transferring", - "current_time": datetime.now().strftime("%d-%m-%Y %H:%M:%S") - } - - tokens = pexp_match.decode("utf-8").split(" ") - - #lg.log(tokens, level=Loglvl.INFO, type=Logtype.FILE) - - for token in tokens: - if not token == '': - if "file" in token: - fields['file_path'] = token.split('=')[-1] - if prev_file != fields['file_path']: - k = k + 1 - prev_file == fields['file_path'] - elif '%' in token: - pct = float((token.rstrip("%"))) - # pct = (1/len(file_path) * pct) + (k * 1/len(file_path) * 100) - fields['pct_completed'] = pct - # flag end of transfer - print(str(pct) + '% transfered') - if token.rstrip("%") == 100: - end_of_transfer = True - elif any(um in token for um in units_to_match): - fields['amt_transferred'] = token - elif "Mb/s" in token or "Mbps" in token: - t = token[:-4] - if '=' in t: - fields['transfer_rate'] = t[t.find('=') + 1:] - else: - fields['transfer_rate'] = t - elif "status" in token: - fields['transfer_status'] = token.split('=')[-1] - elif "rate" in token: - fields['transfer_rate'] = token.split('=')[-1] - elif "elapsed" in token: - fields['elapsed_time'] = token.split('=')[-1] - elif "loss" in token: - fields['bytes_lost'] = token.split('=')[-1] - elif "size" in token: - fields['file_size_bytes'] = token.split('=')[-1] - - elif "ETA" in token: - eta = tokens[-2] - estimated_completion = "" - eta_split = eta.split(":") - t_u = time_units[-len(eta_split):] - for indx, eta_token in enumerate(eta.split(":")): - if eta_token == "00": - continue - estimated_completion += eta_token + t_u[indx] + " " - fields['estimated_completion'] = estimated_completion - - kwargs = dict(target_id=sub_id, completed_on=datetime.now()) - #Submission().save_record(dict(), **kwargs) - # close thread - thread.close() - print 'Aspera Transfer completed' - - except OSError: - print "Error" - finally: - pass +def _do_aspera_transfer(cmd, handler): + p = Popen(cmd, stdout=PIPE, bufsize=1) + with p.stdout: + for line in iter(p.stdout.readline, b''): + if handler and callable(handler): + handler(line) + else: + print line, + p.wait() + pass def get_wgs_file_ext(output_format): if output_format == EMBL_FORMAT: From d6a5d2d29b98d176dafb08fc896d05663c89a0cb Mon Sep 17 00:00:00 2001 From: froggleston Date: Fri, 6 Oct 2017 17:28:37 +0100 Subject: [PATCH 03/18] Add init.py for module loading support --- python/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/__init__.py diff --git a/python/__init__.py b/python/__init__.py new file mode 100644 index 0000000..e69de29 From 6bc7923b3a6689d1f73044fead67aca80996f266 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sun, 19 Nov 2017 02:10:08 +0000 Subject: [PATCH 04/18] Add handler callback support for aspera transfers --- python/enaDataGet.py | 6 +++++- python/readGet.py | 16 ++++++++-------- python/utils.py | 8 ++++---- python3/enaDataGet.py | 8 ++++++-- python3/readGet.py | 16 ++++++++-------- python3/utils.py | 22 +++++++++++++++++----- 6 files changed, 48 insertions(+), 28 deletions(-) diff --git a/python/enaDataGet.py b/python/enaDataGet.py index 2e04681..6776494 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -50,6 +50,9 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") + parser.add_argument('-r', '--redirect-handler', default=None, + choices=['stdout', 'file'], + help="""File download progress handler. Specify an output handler to process the download progress. Default is no handler (output is swallowed). 'stdout' redirects all output to standard out. 'file' redirects to a file handle (default is [current_file_download.log]).""") parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.4.1') return parser @@ -66,6 +69,7 @@ def set_parser(): fetch_index = args.index aspera = args.aspera aspera_settings = args.aspera_settings + handler = args.redirect_handler if aspera or aspera_settings is not None: aspera = utils.set_aspera(aspera_settings) @@ -85,7 +89,7 @@ def set_parser(): elif utils.is_analysis(accession): if output_format is not None: readGet.check_read_format(output_format) - readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera) + readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler) elif utils.is_run(accession) or utils.is_experiment(accession): if output_format is not None: readGet.check_read_format(output_format) diff --git a/python/readGet.py b/python/readGet.py index 8e505d7..8d67ba3 100644 --- a/python/readGet.py +++ b/python/readGet.py @@ -39,11 +39,11 @@ def check_analysis_format(output_format): ) sys.exit(1) -def attempt_file_download(file_url, dest_dir, md5, aspera): +def attempt_file_download(file_url, dest_dir, md5, aspera, handler=None): if md5 is not None: print 'Downloading file with md5 check:' + file_url if aspera: - return utils.get_aspera_file_with_md5_check(file_url, dest_dir, md5) + return utils.get_aspera_file_with_md5_check(file_url, dest_dir, md5, handler) else: return utils.get_ftp_file_with_md5_check('ftp://' + file_url, dest_dir, md5) print 'Downloading file:' + file_url @@ -51,12 +51,12 @@ def attempt_file_download(file_url, dest_dir, md5, aspera): return utils.get_aspera_file(file_url, dest_dir) return utils.get_ftp_file('ftp://' + file_url, dest_dir) -def download_file(file_url, dest_dir, md5, aspera): +def download_file(file_url, dest_dir, md5, aspera, handler=None): if utils.file_exists(file_url, dest_dir, md5): return - success = attempt_file_download(file_url, dest_dir, md5, aspera) + success = attempt_file_download(file_url, dest_dir, md5, aspera, handler) if not success: - success = attempt_file_download(file_url, dest_dir, md5, aspera) + success = attempt_file_download(file_url, dest_dir, md5, aspera, handler) if not success: print 'Failed to download file after two attempts' @@ -75,7 +75,7 @@ def download_experiment_meta(run_accession, dest_dir): break download_meta(experiment_accession, dest_dir) -def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera): +def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler=None): accession_dir = os.path.join(dest_dir, accession) utils.create_dir(accession_dir) # download experiment xml @@ -116,11 +116,11 @@ def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, file_url = filelist[i] md5 = md5list[i] if file_url != '': - download_file(file_url, target_dir, md5, aspera) + download_file(file_url, target_dir, md5, aspera, handler) if fetch_index: for index_file in indexlist: if index_file != '': - download_file(index_file, target_dir, None, aspera) + download_file(index_file, target_dir, None, aspera, handler) if utils.is_empty_dir(target_dir): print 'Deleting directory ' + os.path.basename(target_dir) os.rmdir(target_dir) diff --git a/python/utils.py b/python/utils.py index 1058799..f8d788e 100644 --- a/python/utils.py +++ b/python/utils.py @@ -289,11 +289,11 @@ def get_ftp_file(ftp_url, dest_dir): except Exception: return False -def get_aspera_file(aspera_url, dest_dir): +def get_aspera_file(aspera_url, dest_dir, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - asperaretrieve(aspera_url, dest_dir, dest_file) + asperaretrieve(aspera_url, dest_dir, dest_file, handler) return True except Exception: return False @@ -335,11 +335,11 @@ def get_ftp_file_with_md5_check(ftp_url, dest_dir, md5): sys.stderr.write("Error with FTP transfer: {0}\n".format(e)) return False -def get_aspera_file_with_md5_check(aspera_url, dest_dir, md5): +def get_aspera_file_with_md5_check(aspera_url, dest_dir, md5, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - success = asperaretrieve(aspera_url, dest_dir, dest_file) + success = asperaretrieve(aspera_url, dest_dir, dest_file, handler) if success: return check_md5(dest_file, md5) return success diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index fe2b66f..a7020ca 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -50,6 +50,9 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") + parser.add_argument('-r', '--redirect-handler', default=None, + choices=['stdout', 'file'], + help="""File download progress handler. Specify an output handler to process the download progress. Default is no handler (output is swallowed). 'stdout' redirects all output to standard out. 'file' redirects to a file handle (default is [current_file_download.log]).""") parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.4.1') return parser @@ -66,6 +69,7 @@ def set_parser(): fetch_index = args.index aspera = args.aspera aspera_settings = args.aspera_settings + handler = args.redirect_handler if aspera or aspera_settings is not None: aspera = utils.set_aspera(aspera_settings) @@ -85,11 +89,11 @@ def set_parser(): elif utils.is_analysis(accession): if output_format is not None: readGet.check_read_format(output_format) - readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera) + readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler) elif utils.is_run(accession) or utils.is_experiment(accession): if output_format is not None: readGet.check_read_format(output_format) - readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera) + readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler) elif utils.is_assembly(accession): if output_format is not None: assemblyGet.check_format(output_format) diff --git a/python3/readGet.py b/python3/readGet.py index dd62c30..a3f8597 100644 --- a/python3/readGet.py +++ b/python3/readGet.py @@ -38,11 +38,11 @@ def check_analysis_format(output_format): ) sys.exit(1) -def attempt_file_download(file_url, dest_dir, md5, aspera): +def attempt_file_download(file_url, dest_dir, md5, aspera, handler=None): if md5 is not None: print('Downloading file with md5 check:' + file_url) if aspera: - return utils.get_aspera_file_with_md5_check(file_url, dest_dir, md5) + return utils.get_aspera_file_with_md5_check(file_url, dest_dir, md5, handler) else: return utils.get_ftp_file_with_md5_check('ftp://' + file_url, dest_dir, md5) print('Downloading file:' + file_url) @@ -50,12 +50,12 @@ def attempt_file_download(file_url, dest_dir, md5, aspera): return utils.get_aspera_file(file_url, dest_dir) return utils.get_ftp_file('ftp://' + file_url, dest_dir) -def download_file(file_url, dest_dir, md5, aspera): +def download_file(file_url, dest_dir, md5, aspera, handler=None): if utils.file_exists(file_url, dest_dir, md5): return - success = attempt_file_download(file_url, dest_dir, md5, aspera) + success = attempt_file_download(file_url, dest_dir, md5, aspera, handler) if not success: - success = attempt_file_download(file_url, dest_dir, md5, aspera) + success = attempt_file_download(file_url, dest_dir, md5, aspera, handler) if not success: print('Failed to download file after two attempts') @@ -74,7 +74,7 @@ def download_experiment_meta(run_accession, dest_dir): break download_meta(experiment_accession, dest_dir) -def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera): +def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler=None): accession_dir = os.path.join(dest_dir, accession) utils.create_dir(accession_dir) # download experiment xml @@ -114,10 +114,10 @@ def download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, file_url = filelist[i] md5 = md5list[i] if file_url != '': - download_file(file_url, target_dir, md5, aspera) + download_file(file_url, target_dir, md5, aspera, handler) for index_file in indexlist: if index_file != '': - download_file(index_file, target_dir, None, aspera) + download_file(index_file, target_dir, None, aspera, handler) if utils.is_empty_dir(target_dir): print('Deleting directory ' + os.path.basename(target_dir)) os.rmdir(target_dir) diff --git a/python3/utils.py b/python3/utils.py index 92eba10..e502bf1 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -289,11 +289,11 @@ def get_ftp_file(ftp_url, dest_dir): print ("Error with FTP transfer: {0}".format(e)) return False -def get_aspera_file(aspera_url, dest_dir): +def get_aspera_file(aspera_url, dest_dir, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - asperaretrieve(aspera_url, dest_dir, dest_file) + asperaretrieve(aspera_url, dest_dir, dest_file, handler) return True except Exception: print ("Error with FTP transfer: {0}".format(e)) @@ -387,7 +387,7 @@ def set_aspera(aspera_filepath): aspera = False return aspera -def asperaretrieve(url, dest_dir, dest_file): +def asperaretrieve(url, dest_dir, dest_file, handler=None): try: if not os.path.exists(ASPERA_BIN) or not os.path.exists(ASPERA_PRIVATE_KEY): raise FileNotFoundError('Aspera not available. Check your ascp binary path and your private key file is specified correctly') @@ -404,13 +404,25 @@ def asperaretrieve(url, dest_dir, dest_file): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - print(aspera_line) - subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True + #print(aspera_line) + #subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True + _do_aspera_transfer(aspera_line, handler) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False +def _do_aspera_transfer(cmd, handler): + p = Popen(cmd, stdout=PIPE, bufsize=1) + with p.stdout: + for line in iter(p.stdout.readline, b''): + if handler and callable(handler): + handler(line) + else: + print line, + p.wait() + pass + def get_wgs_file_ext(output_format): if output_format == EMBL_FORMAT: return WGS_EMBL_EXT From 88314c1f8bb19d0f9c0efb392bc7e59be15b89a3 Mon Sep 17 00:00:00 2001 From: froggleston Date: Mon, 11 Dec 2017 11:06:56 +0000 Subject: [PATCH 05/18] Intermediate commit for download output handler --- python/enaDataGet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/enaDataGet.py b/python/enaDataGet.py index 6776494..74a059d 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -51,8 +51,8 @@ def set_parser(): help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") parser.add_argument('-r', '--redirect-handler', default=None, - choices=['stdout', 'file'], - help="""File download progress handler. Specify an output handler to process the download progress. Default is no handler (output is swallowed). 'stdout' redirects all output to standard out. 'file' redirects to a file handle (default is [current_file_download.log]).""") + choices=['queue', 'file'], + help="""File download progress handler. Specify an output handler to process the download progress. Default is no handler (output is printed to stdout). 'queue' redirects all output to a queue handler, such as RabbitMQ. 'file' redirects to a file handle (default is [current_file_download.log]).""") parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.4.1') return parser From 3ae64a20bb6b996737b8a795bdbe3280fdd6f3f6 Mon Sep 17 00:00:00 2001 From: "Robert Davey (EI)" Date: Mon, 11 Dec 2017 15:09:03 +0000 Subject: [PATCH 06/18] Complete stdout handler code --- python/assemblyGet.py | 6 +-- python/enaDataGet.py | 13 +++++-- python/sequenceGet.py | 22 +++++------ python/utils.py | 87 ++++++++++++++++++++++++++++++++++--------- 4 files changed, 93 insertions(+), 35 deletions(-) diff --git a/python/assemblyGet.py b/python/assemblyGet.py index 2fb8f1a..626454d 100644 --- a/python/assemblyGet.py +++ b/python/assemblyGet.py @@ -93,7 +93,7 @@ def download_sequences(sequence_report, assembly_dir, output_format, quiet): download_sequence_set(unplaced_list, UNPLACED, assembly_dir, output_format, quiet) download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) -def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False): +def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False, handler=None): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -106,12 +106,12 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False has_sequence_report = False # download sequence report if sequence_report is not None: - has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir) + has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir, handler) # download wgs set if needed if wgs_set is not None and fetch_wgs: if not quiet: print 'fetching wgs set' - sequenceGet.download_wgs(assembly_dir, wgs_set, output_format) + sequenceGet.download_wgs(assembly_dir, wgs_set, output_format, handler) # parse sequence report and download sequences if has_sequence_report: download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) diff --git a/python/enaDataGet.py b/python/enaDataGet.py index 74a059d..c962fbc 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -76,28 +76,33 @@ def set_parser(): try: if utils.is_wgs_set(accession): + print("Downloading WGS set") if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_wgs(dest_dir, accession, output_format) + sequenceGet.download_wgs(dest_dir, accession, output_format, handler) elif not utils.is_available(accession): sys.stderr.write('ERROR: Record does not exist or is not available for accession provided\n') sys.exit(1) elif utils.is_sequence(accession): + print("Downloading sequence(s)") if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_sequence(dest_dir, accession, output_format) + sequenceGet.download_sequence(dest_dir, accession, output_format, handler) elif utils.is_analysis(accession): + print("Downloading analysis") if output_format is not None: readGet.check_read_format(output_format) readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler) elif utils.is_run(accession) or utils.is_experiment(accession): + print("Downloading reads") if output_format is not None: readGet.check_read_format(output_format) - readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera) + readGet.download_files(accession, output_format, dest_dir, fetch_index, fetch_meta, aspera, handler) elif utils.is_assembly(accession): + print("Downloading assembly") if output_format is not None: assemblyGet.check_format(output_format) - assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs) + assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, handler) else: sys.stderr.write('ERROR: Invalid accession provided\n') sys.exit(1) diff --git a/python/sequenceGet.py b/python/sequenceGet.py index 37df7a5..79925fb 100644 --- a/python/sequenceGet.py +++ b/python/sequenceGet.py @@ -26,44 +26,44 @@ def append_record(dest_file, accession, output_format): url = utils.get_record_url(accession, output_format) return utils.append_record(url, dest_file) -def download_sequence(dest_dir, accession, output_format): +def download_sequence(dest_dir, accession, output_format, handler=None): if output_format is None: output_format = utils.EMBL_FORMAT - success = utils.download_record(dest_dir, accession, output_format) + success = utils.download_record(dest_dir, accession, output_format, handler) if not success: print 'Unable to fetch file for {0}, format {1}'.format(accession, output_format) return success -def download_wgs(dest_dir, accession, output_format): +def download_wgs(dest_dir, accession, output_format, handler=None): if utils.is_unversioned_wgs_set(accession): - return download_unversioned_wgs(dest_dir, accession, output_format) + return download_unversioned_wgs(dest_dir, accession, output_format, handler) else: - return download_versioned_wgs(dest_dir, accession, output_format) + return download_versioned_wgs(dest_dir, accession, output_format, handler) -def download_versioned_wgs(dest_dir, accession, output_format): +def download_versioned_wgs(dest_dir, accession, output_format, handler=None): prefix = accession[:6] if output_format is None: output_format = utils.EMBL_FORMAT public_set_url = utils.get_wgs_ftp_url(prefix, utils.PUBLIC, output_format) supp_set_url = utils.get_wgs_ftp_url(prefix, utils.SUPPRESSED, output_format) - success = utils.get_ftp_file(public_set_url, dest_dir) + success = utils.get_ftp_file(public_set_url, dest_dir, handler) if not success: - success = utils.get_ftp_file(supp_set_url, dest_dir) + success = utils.get_ftp_file(supp_set_url, dest_dir, handler) if not success: print 'No WGS set file available for {0}, format {1}'.format(accession, output_format) print 'Please contact ENA (datasubs@ebi.ac.uk) if you feel this set should be available' -def download_unversioned_wgs(dest_dir, accession, output_format): +def download_unversioned_wgs(dest_dir, accession, output_format, handler=None): prefix = accession[:4] if output_format is None: output_format = utils.EMBL_FORMAT public_set_url = utils.get_nonversioned_wgs_ftp_url(prefix, utils.PUBLIC, output_format) if public_set_url is not None: - utils.get_ftp_file(public_set_url, dest_dir) + utils.get_ftp_file(public_set_url, dest_dir, handler) else: supp_set_url = utils.get_nonversioned_wgs_ftp_url(prefix, utils.SUPPRESSED, output_format) if supp_set_url is not None: - utils.get_ftp_file(supp_set_url, dest_dir) + utils.get_ftp_file(supp_set_url, dest_dir, handler) else: print 'No WGS set file available for {0}, format {1}'.format(accession, output_format) print 'Please contact ENA (datasubs@ebi.ac.uk) if you feel this set should be available' diff --git a/python/utils.py b/python/utils.py index f8d788e..bf4665b 100644 --- a/python/utils.py +++ b/python/utils.py @@ -26,6 +26,7 @@ import urllib import urllib2 import pexpect +import time from datetime import datetime import xml.etree.ElementTree as ElementTree from subprocess import call, Popen, PIPE @@ -256,10 +257,10 @@ def get_destination_file(dest_dir, accession, output_format): return os.path.join(dest_dir, filename) return None -def download_single_record(url, dest_file): +def download_single_record(url, dest_file, handler=None): urllib.urlretrieve(url, dest_file) -def download_record(dest_dir, accession, output_format): +def download_record(dest_dir, accession, output_format, handler=None): try: dest_file = get_destination_file(dest_dir, accession, output_format) url = get_record_url(accession, output_format) @@ -280,11 +281,12 @@ def append_record(url, dest_file): except Exception: return False -def get_ftp_file(ftp_url, dest_dir): +def get_ftp_file(ftp_url, dest_dir, handler=None): try: filename = ftp_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - urllib.urlretrieve(ftp_url, dest_file) + response = urllib.urlopen(ftp_url) + chunk_read(response, file_handle=dest_file, handler=handler) return True except Exception: return False @@ -293,7 +295,7 @@ def get_aspera_file(aspera_url, dest_dir, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - asperaretrieve(aspera_url, dest_dir, dest_file, handler) + asperaretrieve(aspera_url, dest_dir, dest_file, handler=handler) return True except Exception: return False @@ -305,12 +307,14 @@ def get_md5(filepath): hash_md5.update(chunk) return hash_md5.hexdigest() -def check_md5(filepath, expected_md5): +def check_md5(filepath, expected_md5, handler): generated_md5 = get_md5(filepath) if expected_md5 != generated_md5: - print 'MD5 mismatch for downloaded file ' + filepath + '. Deleting file' - print ('generated md5', generated_md5) - print ('expected md5', expected_md5) + if not handler: + handler=sys.stdout.write + handler('MD5 mismatch for downloaded file ' + filepath + '. Deleting file') + handler('Generated md5: %s' % generated_md5) + handler('Expected md5: %s' % expected_md5) os.remove(filepath) return False return True @@ -325,12 +329,13 @@ def file_exists(file_url, dest_dir, md5): return True return False -def get_ftp_file_with_md5_check(ftp_url, dest_dir, md5): +def get_ftp_file_with_md5_check(ftp_url, dest_dir, md5, handler=None): try: filename = ftp_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - urllib.urlretrieve(ftp_url, dest_file) - return check_md5(dest_file, md5) + response = urllib.urlopen(ftp_url) + chunk_read(response, file_handle=dest_file, handler=handler) + return check_md5(dest_file, md5, handler=handler) except Exception as e: sys.stderr.write("Error with FTP transfer: {0}\n".format(e)) return False @@ -339,9 +344,9 @@ def get_aspera_file_with_md5_check(aspera_url, dest_dir, md5, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - success = asperaretrieve(aspera_url, dest_dir, dest_file, handler) + success = asperaretrieve(aspera_url, dest_dir, dest_file, handler=handler) if success: - return check_md5(dest_file, md5) + return check_md5(dest_file, md5, handler=handler) return success except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) @@ -402,8 +407,6 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - #sys.stdout.write(aspera_line) - #subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True _do_aspera_transfer(aspera_line, handler) return True except Exception as e: @@ -417,7 +420,7 @@ def _do_aspera_transfer(cmd, handler): if handler and callable(handler): handler(line) else: - print line, + sys.stdout.write(line), p.wait() pass @@ -595,3 +598,53 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + +def chunk_report(f, bytes_so_far, chunk_size, total_size, handler=None): + global start_time + if bytes_so_far == chunk_size: + start_time = time.time() + return + duration = time.time() - start_time + progress_size = int(bytes_so_far) + speed = int(progress_size / ((1024 * 1024) * duration)) + percent = int(progress_size * 100 /total_size) + line = "\r%s: %0.f%%, %d MB, %d MB/s" % (f, percent, progress_size / (1024 * 1024), speed) + + if handler and callable(handler): + handler(line) + else: + sys.stdout.write(line) + + if bytes_so_far >= total_size: + if handler and callable(handler): + handler('\n') + else: + sys.stdout.write('\n') + +def chunk_read(response, file_handle, chunk_size=104857, tick_rate=10, report_hook=chunk_report, handler=None): + total_size = response.info().getheader('Content-Length').strip() + bytes_so_far = 0 + + with open(file_handle, 'wb') as f: + base = os.path.basename(file_handle) + if total_size is None: # cannot chunk - no content length + f.write(response.content) + else: + tick = 0 + total_size = int(total_size) + while 1: + chunk = response.read(chunk_size) + bytes_so_far += len(chunk) + + if not chunk: + break + + f.write(chunk) + + if report_hook: + if tick == 0 or tick % int(tick_rate) == 0: + tick = 0 + report_hook(base, bytes_so_far, chunk_size, total_size, handler) + tick += 1 + + return bytes_so_far From 096468e61056864832bdf5167823660cd7268070 Mon Sep 17 00:00:00 2001 From: "Robert Davey (EI)" Date: Wed, 13 Dec 2017 15:00:56 +0000 Subject: [PATCH 07/18] Fix aspera code to take shlex'ed command instead of single string --- python/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/utils.py b/python/utils.py index bf4665b..f827a95 100644 --- a/python/utils.py +++ b/python/utils.py @@ -27,6 +27,7 @@ import urllib2 import pexpect import time +import shlex from datetime import datetime import xml.etree.ElementTree as ElementTree from subprocess import call, Popen, PIPE @@ -414,7 +415,7 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): return False def _do_aspera_transfer(cmd, handler): - p = Popen(cmd, stdout=PIPE, bufsize=1) + p = Popen(shlex.split(cmd), stdout=PIPE, bufsize=1) with p.stdout: for line in iter(p.stdout.readline, b''): if handler and callable(handler): From 8b1f2226a33d14165378a4e81f6779a8a6ef048a Mon Sep 17 00:00:00 2001 From: "Robert Davey (EI)" Date: Tue, 2 Jan 2018 16:41:57 +0000 Subject: [PATCH 08/18] Add pexpect support for aspera progress reporting --- python/utils.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/python/utils.py b/python/utils.py index f827a95..cfb4030 100644 --- a/python/utils.py +++ b/python/utils.py @@ -28,9 +28,10 @@ import pexpect import time import shlex +import json from datetime import datetime import xml.etree.ElementTree as ElementTree -from subprocess import call, Popen, PIPE +from subprocess import call, Popen, PIPE, STDOUT from ConfigParser import SafeConfigParser @@ -408,14 +409,62 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - _do_aspera_transfer(aspera_line, handler) +# _do_aspera_transfer(aspera_line, handler) + _do_aspera_pexpect(aspera_line, handler) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False +def _do_aspera_pexpect(cmd, handler): + thread = pexpect.spawn(cmd, timeout=None) + cpl = thread.compile_pattern_list([pexpect.EOF, '(.+)']) + started = 0 + + while True: + i = thread.expect_list(cpl, timeout=None) + if i == 0: + if started == 0: + if handler and callable(handler): + handler("Error initiating transfer") + else: + sys.stderr.write("Error initiating transfer") + break + elif i == 1: + started = 1 + output = dict() + pexp_match = thread.match.group(1) + prev_file = '' + tokens_to_match = ["Mb/s"] + units_to_match = ["KB", "MB", "GB"] + rates_to_match = ["Kb/s", "kb/s", "Mb/s", "mb/s", "Gb/s", "gb/s"] + end_of_transfer = False + if any(tm in pexp_match.decode("utf-8") for tm in tokens_to_match): + tokens = pexp_match.decode("utf-8").split(" ") + if 'ETA' in tokens: + pct_completed = [x for x in tokens if len(x) > 1 and x[-1] == '%'] + if pct_completed: + output["pct_completed"] = pct_completed[0][:-1] + + bytes_transferred = [x for x in tokens if len(x) > 2 and x[-2:] in units_to_match] + if bytes_transferred: + output["bytes_transferred"] = bytes_transferred[0] + + transfer_rate = [x for x in tokens if len(x) > 4 and x[-4:] in rates_to_match] + if transfer_rate: + output["transfer_rate"] = transfer_rate[0] + + if handler and callable(handler): + handler(json.dumps(output)) + else: + sys.stdout.write("%s%% transferred, %s, %s\n" % (output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])), + thread.close() + def _do_aspera_transfer(cmd, handler): - p = Popen(shlex.split(cmd), stdout=PIPE, bufsize=1) + p = Popen(shlex.split(cmd), + stdout=PIPE, + stderr=STDOUT, + bufsize=1) with p.stdout: for line in iter(p.stdout.readline, b''): if handler and callable(handler): From 8ef6519e22534693818ba9ed7fcb294128ca168d Mon Sep 17 00:00:00 2001 From: nicsilvester Date: Tue, 8 May 2018 12:31:30 +0100 Subject: [PATCH 09/18] v1.5 release Improved checks for Aspera files and binary Handle known Python 3 Mac installation problem impacting HTTPS URLs Don't re-download run/analysis file if exists (with MD5 check) Replace (rather than append to) sequence files if they exist Download WGS set if WGS-only assembly or assembly contains WGS scaffolds (even without WGS flag) Added flag to extract WGS scaffolds from WGS file into wgs_scaffolds file Added some progress information to enaDataGet assembly fetch --- README.md | 20 ++++++-- python/assemblyGet.py | 78 ++++++++++++++++++++++++++----- python/assemblyGet.pyc | Bin 0 -> 6569 bytes python/enaDataGet.py | 7 ++- python/enaGroupGet.py | 101 ++++++++++++++++++++--------------------- python/readGet.pyc | Bin 0 -> 4258 bytes python/sequenceGet.py | 4 +- python/sequenceGet.pyc | Bin 0 -> 2844 bytes python/utils.py | 54 ++++++++++++++++++---- python/utils.pyc | Bin 0 -> 24461 bytes python3/assemblyGet.py | 74 ++++++++++++++++++++++++++---- python3/enaDataGet.py | 7 ++- python3/enaGroupGet.py | 100 +++++++++++++++++++--------------------- python3/sequenceGet.py | 4 +- python3/utils.py | 75 +++++++++++++++++++++++------- 15 files changed, 358 insertions(+), 166 deletions(-) create mode 100644 python/assemblyGet.pyc create mode 100644 python/readGet.pyc create mode 100644 python/sequenceGet.pyc create mode 100644 python/utils.pyc diff --git a/README.md b/README.md index 7c263cc..28764b8 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ Both Python 2 and Python 3 scripts are available. The Python 2 scripts can be f To run these scripts you will need to have Python installed. You can download either Python 2 or Python 3 from [here](https://www.python.org/downloads/). If you already have Python installed, you can find out which version when you start the python interpreter. If using Python 2, we suggest you upgrade to the latest version if you don't already have it: 2.7. +Note that EBI now uses HTTPS servers. This can create a problem when using Python 3 on a Mac due to an oft-missed +installation step. Please run the "Install Certificates.command" command to ensure your Python 3 installation on +the Mac can correctly authenticate against the servers. To do this, run the following from a terminal window, updating +the Python version with the correct version of Python 3 that you have installed: +open "/Applications/Python 3.6/Install Certificates.command" + ## Installing and running the scripts Download the [latest release](https://github.com/enasequence/enaBrowserTools/releases/latest) and extract it to the preferred location on your computer. You will now have the enaBrowserTools folder, containing both the python 2 and 3 option scripts. If you are using a Unix/Linux or Mac computer, we suggest you add the following aliases to your .bashrc or .bash_profile file. Where INSTALLATION_DIR is the location where you have saved the enaBrowserTools to and PYTHON_CHOICE will depend on whether you are using the Python 2 or Python 3 scripts. @@ -107,12 +113,14 @@ optional arguments: File format required. Format requested must be permitted for data type selected. sequence, assembly and wgs accessions: embl(default) and fasta formats. - read group: submitted, fastq and sra - formats. analysis group: submitted only. + read group: submitted, fastq and sra formats. analysis + group: submitted only. -d DEST, --dest DEST Destination directory (default is current running directory) -w, --wgs Download WGS set for each assembly if available (default is false) + -e, --extract-wgs Extract WGS scaffolds for each assembly if available + (default is false) -m, --meta Download read or analysis XML in addition to data files (default is false) -i, --index Download CRAM index files with submitted CRAM files, @@ -141,7 +149,7 @@ usage: enaGroupGet [-h] [-g {sequence,wgs,assembly,read,analysis}] [-i] [-a] [-as ASPERA_SETTINGS] [-t] [-v] accession -Download data for a given study or sample +Download data for a given study or sample, or (for sequence and assembly) taxon positional arguments: accession Study or sample accession or NCBI tax ID to fetch data @@ -162,6 +170,8 @@ optional arguments: directory) -w, --wgs Download WGS set for each assembly if available (default is false) + -e, --extract-wgs Extract WGS scaffolds for each assembly if available + (default is false) -m, --meta Download read or analysis XML in addition to data files (default is false) -i, --index Download CRAM index files with submitted CRAM files, @@ -180,10 +190,10 @@ optional arguments: # Tips -From version 1.4, when downloading read data if you use the default format (that is, don't use the format option), the scripts will look for available files in the following priority: submitted, sra, fastq. +From version 1.4, when downloading read data if you use the default format (that is, don't use the format option), the scripts will look for available files in the following priority: submitted, sra, fastq. A word of advice for read formats: -- submitted: only read data submitted to ENA have files available as submitted by the user. +- submitted: only read data submitted to ENA have files available as submitted by the user. - sra: this is the NCBI SRA format, and is the format in which all NCBI/DDBJ data is mirrored to ENA. - fastq: not all submitted format files can be converted to FASTQ diff --git a/python/assemblyGet.py b/python/assemblyGet.py index 2fb8f1a..2b890fb 100644 --- a/python/assemblyGet.py +++ b/python/assemblyGet.py @@ -20,6 +20,7 @@ import os import sys import argparse +import gzip import xml.etree.ElementTree as ElementTree import utils @@ -31,7 +32,7 @@ PATCH = 'patch' def check_format(output_format): - if format not in [utils.EMBL_FORMAT, utils.FASTA_FORMAT]: + if output_format not in [utils.EMBL_FORMAT, utils.FASTA_FORMAT]: sys.stderr.write( 'ERROR: Invalid format. Please select a valid format for this accession: {0}\n'.format([utils.EMBL_FORMAT, utils.FASTA_FORMAT]) ) @@ -69,31 +70,72 @@ def parse_sequence_report(local_sequence_report): patch_list = [l.split('\t')[0] for l in lines[1:] if PATCH in l.split('\t')[3]] return (replicon_list, unlocalised_list, unplaced_list, patch_list) +def extract_wgs_sequences(accession_list): + wgs_sequences = [a for a in accession_list if utils.is_wgs_sequence(a)] + other_sequences = [a for a in accession_list if not utils.is_wgs_sequence(a)] + return wgs_sequences, other_sequences + def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, quiet): failed_accessions = [] - if len(accession_list) > 0: + count = 0 + sequence_cnt = len(accession_list) + divisor = utils.get_divisor(sequence_cnt) + if sequence_cnt > 0: if not quiet: print 'fetching sequences: ' + mol_type - target_file = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) + target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) + target_file = open(target_file_path, 'w') for accession in accession_list: - success = sequenceGet.append_record(target_file, accession, output_format) + success = sequenceGet.write_record(target_file, accession, output_format) if not success: failed_accessions.append(accession) + else: + count += 1 + if count % divisor == 0 and not quiet: + print 'downloaded {0} of {1} sequences'.format(count, sequence_cnt) + if not quiet: + print 'downloaded {0} of {1} sequences'.format(count, sequence_cnt) + target_file.close() elif not quiet: print 'no sequences: ' + mol_type if len(failed_accessions) > 0: - print 'Failed to fetch following ' + mol_type + ', format ' + output_format - print failed_accessions.join(',') + print 'Failed to fetch following {0}, format {1}'.format(mol_type, output_format) + print ','.join(failed_accessions) def download_sequences(sequence_report, assembly_dir, output_format, quiet): local_sequence_report = os.path.join(assembly_dir, sequence_report) replicon_list, unlocalised_list, unplaced_list, patch_list = parse_sequence_report(local_sequence_report) + wgs_scaffolds, other_unlocalised = _sequences(unlocalised_list) + wgs_unplaced, other_unplaced = extract_wgs_sequences(unplaced_list) download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, quiet) - download_sequence_set(unlocalised_list, UNLOCALISED, assembly_dir, output_format, quiet) - download_sequence_set(unplaced_list, UNPLACED, assembly_dir, output_format, quiet) + download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, quiet) + download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, quiet) download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) + wgs_scaffolds.extend(wgs_unplaced) + return wgs_scaffolds -def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False): +def extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, quiet): + if not quiet: + print 'extracting {0} WGS scaffolds from WGS set file'.format(len(wgs_scaffolds)) + accs = [a.split('.')[0] for a in wgs_scaffolds] + wgs_file_path = os.path.join(assembly_dir, wgs_set + utils.get_wgs_file_ext(output_format)) + target_file_path = os.path.join(assembly_dir, utils.get_filename('wgs_scaffolds', output_format)) + write_line = False + target_file = open(target_file_path, 'w') + with gzip.open(wgs_file_path, 'rb') as f: + for line in f: + if utils.record_start_line(line, output_format): + if utils.extract_acc_from_line(line, output_format) in accs: + write_line = True + else: + write_line = False + target_file.flush() + if write_line: + target_file.write(line) + target_file.flush() + target_file.close() + +def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, quiet=False): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -107,11 +149,23 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False # download sequence report if sequence_report is not None: has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir) + # parse sequence report and download sequences + wgs_scaffolds = [] + wgs_scaffold_cnt = 0 + if has_sequence_report: + wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + wgs_scaffold_cnt = len(wgs_scaffolds) + if wgs_scaffold_cnt > 0: + if not quiet: + print 'Assembly contains {} WGS scaffolds, will fetch WGS set'.format(wgs_scaffold_cnt) + fetch_wgs = True + else: + fetch_wgs = True # download wgs set if needed if wgs_set is not None and fetch_wgs: if not quiet: print 'fetching wgs set' sequenceGet.download_wgs(assembly_dir, wgs_set, output_format) - # parse sequence report and download sequences - if has_sequence_report: - download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + # extract wgs scaffolds from WGS file + if wgs_scaffold_cnt > 0 and extract_wgs: + extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, quiet) diff --git a/python/assemblyGet.pyc b/python/assemblyGet.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5e60dc593ed1787a018b404519d5f056b86eedc GIT binary patch literal 6569 zcmcIpO>-Mb8Sa*4*|Oz0alTm~A}m`9B=NE=yAYNoj^iXQ93?w)@Z#F6YN8q0Badg~ z>DJmI)+u}9!f)UQaN5IGapO1eH#ks5)n4Iw-ky;hZzwo8k=5$y>5sSHkLP{5Gx4vv znZND*W3!|3eSz;zh!)v@im{II8@3i z)zeaz)K*FDOsVXlvJRsL6BA};F^M{`$>QC#GuRCiYZkX6yBH?^oh;o)hxIg# z&0_z#-AX!hhxR6#C}?Lp?Cj(VT*b5D=9IpqtNOboF&3rgcNt$%=)nr@)f2ae z{_j40Uk$Z7goWVrVufBw4NK|}em|jhN-8^Dye-p4c-z`Ltu_!itp**VX}RbVZ>_zy z+Wf8PW^L)t%9@!+b1U2%gh?l?dtpD#EQ~E(8&sBP3`)$T(v6ZpDmFZ@Qfa11D4rl% z&caTbxgK$p?8rK?|16A8T(K7++g51XCYQaPiR|q`7GJ|FGlN3u3wlA<^f@hB=5|I} z2AHUW@R*j@|BIHL-Y0U+4ks8ihw>PdMf+{)3CID}qx~X!AV^>wKD0K(3iQmTokF>B zztwK7tc$`Pdn@-=Kb7X+t$3}~M@{kn2;EeJL%dTgXeEz`A-0L(gxrX+nMW)TLwQ6I zY2M#*anR4gZge1Xe*rf}gq^q*xF)TDC1DBMc1ylX*(kR&zygeL7S2900U*g`s#|-_IZr)eZTwG_~G& z)V=ScaDL5rIM|Kb-6#&7?x^Hr?j#fBn|mY56EWKbXtW{FFDV6uQ~Er7x8~Hw2F9_d z?PGu!)aG;UOUBCYFPR96tK$iz3fy}_RXuU`9{7`Lv)7^ zO{&9+I-FA7a)F5wYKEDh8cm>dQAbrnHOLHoI8+stEc64wS!l$wI&limqA^TAj;sKy zxAJ&xb@@?KT=7YB?a}hm+G=a%u0!4C*<@F7PuLP%C=n6kVKN`PzPFvFNub{x0}?1T2U<9(8FM z>XFG|%OyXD3Pi0DA#vK{0AG~vfWUP(GDm8+!)#4y-NVLGiYP$s|=-jr1hKS-}V%px(2rS3n#ApC7mWTn6L7NTR_ zUQH1TlLgLVu4wT3@y^ggQg*)kmIm4962<=RKLF{+5yJHnZAdC7CNu>sgrj#uknSki z0@2N_bE_^PyzcWWekg0I3mh10HNkrI$vD&{TG0 zGEDs4(0d0x1<;aBwO(u}@{!$AeKbxz(b=rg)Q zq@;DJazV_$qT;v{V;L?B2894D3wM%3z-MKKKLQE~5Md5$BX|&PpGnvvRH1zb0&JOO zXo8c%*%{H8P~h|BBu;;5MQbrv^J2$2R#!VR7C{g7`Su%M55t1ND^c#p-;SiH~T z=P1U&UYxGQ6CJQ0$*s=$L34}LYENNw(amMs6Za$1>u>pyiO?#*Lxy+>F$^Qz8nzHjV?N~>^D^Cqu#7|Hh$J%WzAXu4ln2lqsi49e z0b)851sxA*i8fAZzoTb#hgx_AabGZy2&Sdf4t)8~GChJqhPzmTAP8o}4+rLw3D)K_}Rv)EUoCs(p)oSn7=-KIJ&Pqhew8SfHP#2_!f6D?&9G`ocSq= zF$J@I_v-h2CBPsMK^XFU7I%y8qeUUR2g5t=CA5nRQ8@20M=)fC{>W|#Cc^CsK;Uo* zXhyp9bH`HLMtE;?JQJ!ti(=d7J&$9#`Hh=ko18>3O}oHDGPDy9Y*NJ0o(nGdM>HCy zb9cx@D3>niOQi)}Db@6O1T;TaKw zWzljrNuS#A0I#c%&^@x5CS+m4&WqWm_ua$e-WF5elZ@2>Bg8&jTyk0?vHhL9+WycX5!wcUvER;$gfF} zf3L?80hspsfl|=f5O#wpFSJF&krQPy%9MB@Zr&l z?2x5*nCq_Cyc(5jdk=N*%y{psXtWVmUBz)Dc}*(Q_6>|%<#~~pNtzXhje{(=jq@xV zHBQrCG}O9HPj4QC-e#-inM>bYzuDvhP3d-7SPe+{0IWkBI*IRDmvkt%L%AKu?Wo-F z?sy4Azjc23KSk*Mqu9wFH6t*{dP3jj`jT27I%(ztQyC}f)f=gD<0+7YU{8Bci6*9F zU;YT^`ZWw<>Y4)U(S7}1F`5d@(kX2I2`?keydH0NkJ!C~3peE=HD1lg5V4&RfZ$uF zkPe*PXF@%FHo;Anjkc#&PZH>vHbU8$=sF5|au6)g@w{w|4e|=MqiEj3Mzs%^<%5rz z`c1uQOo*6<#_Jq5Fa%ya-&cseQEw9_CrYckf60TEFo?NrX7E(aJ9xUZmJf4WFB{Qr z>|woTr1er6x>)g{ea&2E-t-`6yQND5vGNbwpO1mQN%4V@GA$+Qh5wzsr#2+SW6zUi zWFsBT3|AW+h-nr9k%yz=i}X=I9##am0l9OrDafO(7eABXEK|ij=H11_*+;_UL3y5) zS8$LPX$S&v+#6QqIp+5e+d-9>1PmGZj_0>M@wcXJL6etplr@tJZZCiV8B12j+r)K` zI%j?BF2fzH8p)>EGE6yNYpw)MYK+j%d^4;=I=Fir_ydTuJ7QGX?k5djUj%7IZrLb zEaSgw7V+@^o|!dWs(cp@P1c|vAsa?PXSGnMKx7at#2Q5!u|fb1Ul42I%McMn(6?oP zhHP8l1;Pu~A)q7da87q)oo?Xk{F{p?ucjRAxZZZ2l+0KDjt%7c`P=MvVhWMTEzi%(>@PT7NXT&xWlz$Q|+ zWw62an~m1D2C0Zws51Z`Zk`cy_ANhqY!H4NUHn`ID-hO{;T;*=nQVNm8^Kwlu9J=>2ciCw2 z_Mq~iWA=J)dr3G_;<(I;5%3VJyJ&U;3No4i)(tdgS2s;@L+?M>K#ll^l*yu|y#)*T z8$d-Mn_k%ef>vb@_{M_3CYxB|6{os+Iffc%Ps^7!ypJWl#Cbbv7{=!Sl=^w-CoiZ*{W(pk4(`zZ+G##?+!Js1>t@xm8fN6v7v{sdrel#+rJ~=JZer|ug X0a|Mk4XsD@s5X!PdaYKg)aw5RnZ(1d literal 0 HcmV?d00001 diff --git a/python/sequenceGet.py b/python/sequenceGet.py index 37df7a5..8a1bd21 100644 --- a/python/sequenceGet.py +++ b/python/sequenceGet.py @@ -22,9 +22,9 @@ import utils -def append_record(dest_file, accession, output_format): +def write_record(dest_file, accession, output_format): url = utils.get_record_url(accession, output_format) - return utils.append_record(url, dest_file) + return utils.write_record(url, dest_file) def download_sequence(dest_dir, accession, output_format): if output_format is None: diff --git a/python/sequenceGet.pyc b/python/sequenceGet.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f4c17610c3eaa7e4bd3fb8574cef1341340b80a GIT binary patch literal 2844 zcmd5;-EQ1O6h7nq%Z3eUNn270gsFtI2*`pE5<&O>;t{yz0pL5wYllSTrrHaV_4v%0@%((}JKs$0ucgIb|M+t^pxK{; z_kBF(dlVsliejSO%2u?iO20$94uuZ=gbp8%3y%9q&i6mYN>~+L$O!>=N42+)i_}rrEJ? zEzg3nx_FTkW*t^<$LP%Tk|;=H-Fuh@`LRx{=^aG2rxSl8OHa{ZJ56KLo1WRjH0c@r zL#~rR_qAP{o_P%UT|9=@MCvtFp}`J8<0}{dAtX9lplp?_q6q+TD8VjZ3m`L~Y77)0 zGJQsQQojVS{}|oTGV1b_1H+rxN&LN7yEb+AwG9qkM#9C0-De+Oyw3)9?mqkIqRa4F zUeF*-w1+p@S5RzjZ9Ex2`r6xC-?q!Bhv{h&r+zrY**4)TMkyfYmhga6yAscsd(bC@gOKSrgZ;d4dVTVz2%z*w*nV1gG{ z>IhQFOW7aKVeg=b%s5X@Ko-zThvU-&Gd9}VRWw!0yVgOzs~WF-E2Q^01eY+*9XbZ6 znjQ;&2X(_CCAF+>I6_noU4_?gr~n!e&t(=?l)d*N$|9c&f+D@a+ImUl!rzdzhrc2I zCY^)IlJ*y{n`C}0{peZxnc!&pg=J~dDx>63|1soA_%Gq%^$sH5i1~oc)cv+UauJP! z|Ne;|#gZ2<$OJRQ7sFWlM!Nxo!Vj#wIaqhQp>KVY@0t60FIw}1wfyK^H`;g4(j2*^ zW7i%=M&>YwX}o_4E=L<>EJQc}y~G?9E_9b=~K+o=c{uX(t$@#M<~b`j=iXE^jW zN2AS$B5~ZCdvX?g+}@QTud+mPa+>M==&4BMG~bJ(fVo&nrOBt$(%~@)=FKec1_pf( zkKxQ9wJ2G*s#YO?P07ZaMKk%FDA0{6X-K+LV1K%mHmmMyX`$D*3>$f|Tfl zgRJ=M=kl|Xa4Na;A4J0!BN4no#H;^5v+ND5k9m`%*TiW?Id26WyxUyd;zDS}VYwzg z9!Ju9oeP$*b%4({)^IsEzkyNcH44F(h79=bi62K{d2F(H zNYYogiqH61U5t2{UA}z$E7>{A9<7hI*UQtCFCbfRYaluVmbHcd4mz@2@Z8|G@GH%3uOr-By>mJRmEnRX_no7 zb};obQ(PnD{^Rj(JwB;L89ZLK_cn@48i_TZE0-+d^W!wkWBmn_+OQnq9^iDGcD=pO KZnl@&%l`n+a)ZbK literal 0 HcmV?d00001 diff --git a/python/utils.py b/python/utils.py index 7d85652..ec79b8a 100644 --- a/python/utils.py +++ b/python/utils.py @@ -106,7 +106,7 @@ sequence_pattern_1 = re.compile('^[A-Z]{1}[0-9]{5}(\.[0-9]+)?$') sequence_pattern_2 = re.compile('^[A-Z]{2}[0-9]{6}(\.[0-9]+)?$') -sequence_pattern_3 = re.compile('^[A-Z]{4}[0-9]{8,9}(\.[0-9]+)?$') +wgs_sequence_pattern = re.compile('^[A-Z]{4}[0-9]{8,9}(\.[0-9]+)?$') coding_pattern = re.compile('^[A-Z]{3}[0-9]{5}(\.[0-9]+)?$') wgs_prefix_pattern = re.compile('^[A-Z]{4}[0-9]{2}$') wgs_master_pattern = re.compile('^[A-Z]{4}[0-9]{2}[0]{6}$') @@ -134,8 +134,10 @@ def is_available(accession): return (not 'entry is not found' in record.text) and (len(record.getchildren()) > 0) def is_sequence(accession): - return sequence_pattern_1.match(accession) or sequence_pattern_2.match(accession) \ - or sequence_pattern_3.match(accession) + return sequence_pattern_1.match(accession) or sequence_pattern_2.match(accession) + +def is_wgs_sequence(accession): + return wgs_sequence_pattern.match(accession) def is_coding(accession): return coding_pattern.match(accession) @@ -266,14 +268,16 @@ def download_record(dest_dir, accession, output_format): except Exception: return False -def append_record(url, dest_file): +def write_record(url, dest_file): try: response = urllib2.urlopen(url) - f = open(dest_file, 'a') + linenum = 1 for line in response: - f.write(line) - f.flush() - f.close() + if linenum == 1 and line.startswith('Entry:'): + return False + dest_file.write(line) + linenum += 1 + dest_file.flush() return True except Exception: return False @@ -349,13 +353,22 @@ def set_aspera_variables(filepath): try: parser = SafeConfigParser() parser.read(filepath) + # set and check binary location global ASPERA_BIN ASPERA_BIN = parser.get('aspera', 'ASPERA_BIN') + if not os.path.exists(ASPERA_BIN): + print 'Aspera binary ({0}) does not exist. Defaulting to FTP transfer'.format(ASPERA_BIN) + return False + if not os.access(ASPERA_BIN, os.X_OK): + print 'You do not have permissions to execute the aspera binary ({0}). Defaulting to FTP transfer'.format(ASPERA_BIN) + return False + # set and check private key location global ASPERA_PRIVATE_KEY ASPERA_PRIVATE_KEY = parser.get('aspera', 'ASPERA_PRIVATE_KEY') if not os.path.exists(ASPERA_PRIVATE_KEY): print 'Private key file ({0}) does not exist. Defaulting to FTP transfer'.format(ASPERA_PRIVATE_KEY) return False + # set non-file variables global ASPERA_SPEED ASPERA_SPEED = parser.get('aspera', 'ASPERA_SPEED') global ASPERA_OPTIONS @@ -386,8 +399,6 @@ def set_aspera(aspera_filepath): def asperaretrieve(url, dest_dir, dest_file): try: - if not os.path.exists(ASPERA_BIN) or not os.path.exists(ASPERA_PRIVATE_KEY): - raise FileNotFoundError('Aspera not available. Check your ascp binary path and your private key file is specified correctly') logdir=os.path.abspath(os.path.join(dest_dir, "logs")) create_dir(logdir) aspera_line="{bin} -QT -L {logs} -l {speed} -P33001 {aspera} -i {key} era-fasp@{file} {outdir}" @@ -581,3 +592,26 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + +def get_divisor(record_cnt): + if record_cnt <= 10000: + return 1000 + elif record_cnt <= 50000: + return 5000 + return 10000 + +def record_start_line(line, output_format): + if output_format == FASTA_FORMAT: + return line.startswith(b'>') + elif output_format == EMBL_FORMAT: + return line.startswith(b'ID ') + else: + return False + +def extract_acc_from_line(line, output_format): + if output_format == FASTA_FORMAT: + return line.split(b'|')[1] + elif output_format == EMBL_FORMAT: + return line.split()[1][:-1] + else: + return '' diff --git a/python/utils.pyc b/python/utils.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3d1deaed4f8649dda6395104949da8ac4e1513f GIT binary patch literal 24461 zcmd6PdvILWdEd7H2oQWfBvQ0qD_N!?i3BK$qMnop5=#jb2|{;4ky@HsE%#gi3odrS zy$ga61z8ehS+Y~vNt8#EG<7CPJ<~erxN+Q>HZ!fKlYjb{wn?3|{ip38?R3U%CvDuO zPVJ_@-|yUecNc=_$^tqSwFloh_uO;O`QG1m&b{4#x_QH&*ZxzXZ1I2H_#@>SbL*_M&0MdQwwuFPJ5=s8hp}!k*KdVe&26y4 zE^`=bx4BJLxXl~}+GB2u6+UHds}*iHx6KN7nA>iJJI(E|!d>QeTH$VUw^$)-?p7<@ zV{Vrf?lrgD3ZFK2n-%Uex5o;fG50Ae>@|106+Ub34l4|pyVDByo4d;j516~#3J;pg zT4A5Ld#teE+`U#9H1}yM44J#n3J1)6#tILa+iQh`8sTAcpS8jvjq`}P0V^CfcfS=L zHTQrO9y9l#6&^RY&k9eN+i!&@%?(=NDRV|VRMJ9 zaLn8zRyc0%uoXtkJ!*xVxyP(M>RGN%gp@6G?SM)w4-8r|OGI^@OVDlIpXn zo=>XJsrph<9aXiER8OjUA*nvE>gST`DOE2f)zhlJoK(kDEn3)PGq2c8sSUWqOT}&zX9diddr`n;bLRw9&7Bu8YwjfhHFE_4 zb#oU4%$fU~fGg%M3TT*nSwLj2D4=QX6#?_+N&>E$yCh)2Tv@<1b56jbxlq8X<|YMP zH#a5VHFMJfmdsTIyl!qrz~{|f7Vw6-s(?4m%?fzSTus0Yb9Di4o0}8x1#?#fyko8* z;EU!W0q>e?3V6@lynrv6yDH$z<`x9}l(}mH{*t*x0q>i8RlrxwT^I0GbFT^bX>&^g ze#YGE0)E!q=LLLV?hOGyXYNe_KX2|W0l#4GhJar*_qKq)Z0-vJe#zWB0)E-t7X|!^ zxpxKp6?5+i_*HXX67W|kR{4Q1L!DOs0{@K-WB@j|p~#me!{hbZWMyi+)QG}Hvj>f( zs5}?l4q#}m)SMn_)`v>bT-YcTT~r#Z&xN%pnr`BL0Nt4CU(XHsc@|6d1y}5hnOalA9Ubs9t(x=6Z5+^(`u5Ryqrf-ezMC;2JhW8gP zUVmum!h!vdU%dXv(!l2i1wMHHGxxRkI2iXh+}Y#q))2EdjrcZARx3bj zZa5jFbEf&E87nAL6H}*AJ08a``aB9K*Vl|@z42LV!p+UtOjq0j&2*#7mNHh`%1h{$ zwm|dDth3kT4=i;V)Y?p+z1GEXX4cycv!!lx7za*Nzg_OL`s)UD&FV&8MK^f1mwR9- zIH56V&5@l=kW_GeZUxOvae*@%Ysl!TCgqo|kCWT`vY!(|~ zx!!Qa`9@V*cCy}>h4v-M&!0Y3JaHyCJv%JphEG*U0mb4tu8cPsgi@mh8e9DACk=hvdhyecQ~jaf~Ah@4>u(vndmI z?HHeXyo0M)g*Hfe4ppd>q?(Y2KABjGn1Y`-S-IA+#`KzZw~}j3vh;lIDkKRKC^UE8 zyh4Z05!Qt2R7EM(n?H?Ov>t^)F8S{hNoGJ%7Q+T3fYxOGbaXKP6tSU;Ik@=?xFRo^ zYx(dxZYq5gW4MqB?bc4{=)7b4c-Aj-x<8SskY!4J^tGQeF^4&~!h&D@iKk9iA#rWj ze&VrmP9FGcpKRK(tNs6O+VKN_<7U(TICfJp5kt0Q3B`xR=ssBLxUe?Hl8Oe5q_FYp zH(S+%xY(h$AL!KEsJ7|tL{Za=^HGI@(B-Qn@A>dFy>f7+?hmht-inQupTJ+|R5)GB z-1RY4RWOdb)=obWNg&Z2;{Q@((Ua=8(e@K*s^H9;&XszQ>;#v)=F-`I^JcUC7~&EH z$>f7PpfzC%##8Os7TVk8L!D?87+3AThsm9D@>OzliYd=^5PE4@@opX zG6>=F=W08w2j2bV4c6FenM?-4oImf7oYTC=d;mjNzk(DL#blx}A1a7?u~eN8bBzY_ zDSXC>QZ)+KVzB{3gujEIqIJu3XEtQkXEv`Os~@0Ky1d3V#;AF$%`dl+rJ~AX4RQT| zVcvO!hH?FDJc6pYK4-|XU`q-(Lf*e~o$hl|p!BJM|+8jSKGCKBb@D%GDAfJ2TY;Np$PWaRmCWg*>a{hYo&0ge;%WF>Y%b1)-@$aEy(k!n_WaminQP;Z{C6C`Xck3_(z}>=0I@H3 z0sjfTqe0>%ro~0(A{x)e{jg-Le+mhQUd#Ji+0>gf)gU4mu#b3~f^48yEDf|Ao+MPx z`LoAPk4{YFM&e|GsOhQ5(EG%2e&U5B&!GOnaB(;f(he6-jOI>_C^3*vYjlxF(Bzh6 z4D?p6LJ_xuW~KM)^Ub;WW-(4{km5ciM{!xD1OjtsEz0>P6-(7>eIayzglo!(l7<^H zw`F=WJG*veK1{0;I)V%!7wFSyrxja)UbMa-`aGG?r<=iFCwhp6!ofqbK0l!s3;w>0SqsOF55cqG= z-iE+U2?801BtJ1bRRH z4;=ZQB$S$z8dX?22mss#L@{HF3@`L1(zW2XerrIDBKZdwUxpLy=+Mqa6rNy-1(%7w zX7xyt#|=CIGxbWG2_>dn7&VKo(r9sq%u_P!)ngUq@rV#Uu~I8Fnfu{meuP=1nU+nN zotczzQj}0=ffAp90%EY}Oq=X*l`^tEbu4)S*g#si5?&2G_dmi;f6jD%f!4+9s>>(X@yXyo;(Q z15q9b`G+}5ABt+F7S`rxCA)-Iks|bIG@+PTw~`!mGf0lDFoByhx5FdcoY|6i`6LXr zRm#FIp|W_CDjHTD*aTjI@IBmh(q0vGRg075NpNQh+#WOYGE`3p)$0>x7OF82c3yN| zVSoZG>|r~+(hOclHQ+`{@CJ*wS$u&7d0lArEY2wlnb^$b7F6&iufN5D$X;cXO6}u8 zR5EuGr3r`c#17B47KeXAL=uZo5Jl+&@GeVTCf84J#ls9;!diUSXoGaWUf4F=gv8^7tW`OBHkXAm2F%q z<8k_rGQ#Lm2Q?ttsbJ(~iN-31@WaREv>TrJID0Yrr0Sn|w~+*Xs#==8d%N!@}|RUt}+Gt?LBoqgA|c#!3qZ~ zawMvU^7MS|vKHgl`ip4$#LFUa$q#tQ5LZP$WXQ84b9ZL9_Z&!juJbs4(Krh507S)G z46lnWcUuD|2_ZaYyP>XjKl>aA4N3ye( zNSmwJ25)2&`xd$^F_YDil)>x>&eXu^nNp+pVk)fR92HJBWw8`(19KOl95;Al@auSj z zmSQ9RZHCQ|UrSCSIYbHue86NX4KfpQbzr1pGa>1N6_0I4VyPQbRQld(F9E>+K3>2% zM)<-=8sWwaNC&pg!{vty8_Q-(V42g%b?? z*BgubveD(r+#JXya}%tjCb_aoL=p$0(gUb_MJbm`B9@3pOG9#JOIAnjfgUR(5&Sh= zSVaHdA2b1B;ZeaU2crV=aSP1R zeq=Mf3w6yhAY2B7!HE-`rzY7pl8_)9{?CAgV@pNdGuB47J5g6&GfPJ*dtzcd+ia9- z(PY?oA`5zM7$CE+M5H!%;Op%2E(+qclq_hCCcvpt@4)P^bPV9#vv?8hM;q>=g%q(jgl zv1%aADnW}$2m-}q^j^i=AY(76)yH#0pd`tNlx%NGM&x9@!?B!!Hm70Y-1b>Sy zX_O|d>upj?;i}Q+wdH}HFP?cm_#NK&Ef&Aaq7}|{ioR~ZfZibuzJ|UUPGaf zJ^)BN;uHvg^Vz7wBkJ-B8g{~L`$o^tFZyIXmRH2?*dr-VjM8Tx_}p9aJ*mcV`lP;jEDEOGcZ0( zxXA?sw?OcGQ`c*M`n-Pov|7je2;AAd4)wc~-T%TwcK@mD zbvC0`&0dG^5jue54?le1z(d*Vo;YZzWUoWZE@cs!?Vm*J(d#7GQuaD__YfW|Nv&Z} z=@{Q84Wu|=4o_4E450v6kMYoSSgl4`boj9@&<=EJ@u2vvbSWY;c~&E(%TUeDP+5C% zrj)GoWU4lO{&D`&T%%5RN3%qbQ~gLy>holbTa@*002R%*o6|uCjg& zKEUI?O5VK>0Cnj-NIUj|b9>wV^@D$Rw!2M$mIPAW zZ!KY$4h}AqZL*`R_0M7q*-UKU7dv;77;3OJ-A&Sl+i7gL<$`3Rn8`l&iCeMd#0DvLL>24_O-TI4amt`UGH<%LP z75s&`Lz0;{lO@=lSI!}h&Xh~ym?^FhoB0h(da>M=ZUlb_NT*f;>Lc+IU-O!ew9Npe zV5S{7l$$2@@VM$P7`3b*G47YC<~- zA9Wo@7FH5kQGPdKCB#PYt^y&2$JAS_z)^<1hhGa=Z)Id)fL`lE_VIu%;oytxH=d9F zjrUZQVfM608(;3SEAQEoH0WzkugGM2=Ke0dh&&=V5Oy~Y#^_?a>CAc#XR@5pEnd>4(uzu*;$3+_*Hh@ijJ9kc~>Bw3@p;G}wL)u^dU%B(mt#?zdu4g2@bNL5!Su4rE3GEkvJ##BSI7*h2|~B;nw9JC%5obEz+2;9R|!d z>Y0>|Xcf8Z4h&4m!%L3w8TR<7b-cOoO|qKbMd{6C@oy{C*k%~w4Vfb_!aZH(L}d|w z$MKV&55`L)Jnu@SF`nlsEC#H&8(7~BD({AOmf<~MAlbqs8%-TGJdbxz8kJYI%c@2sVuKKp4Y9`HgwPI%BIdLDLhXW2 z-2Mq36Z{Gbst5eW zjvh)zCD8+k2(D}_9+C6UHxSaJPe<-WEODj2_9JJ@&U*82);eQ_{$s{^?k^srzWa)M&1LC+v z0uhOO-M9?2>uOxXaU|V?^EA9CZiPxwYhT(Ln|i%SJy{8>E;=F(-=~dk#WZaP?sR^R zt82AkYs8>B4x!)1{-sWp?-h z3WemJ0v#Rx7p}D^5&V!<{8mYSB`UrxYz8gJh5zm&$w03q`AOL`Cky^&6p*O=Jb2hE zp?0sIcLf;?9Q=qYP?b7hwWIXJEp6kf?{PU~V~YL;?-`(}wSjpbD(Up$Riq1O+(CiQ zL%iFULhq;T;H&(nvtz{|mp^-ILQf7!t6`adHwhym-C;y|@U?E6sxd(Ob;mzrk` zyUXAaGiLRzu$j=*u(L@l#<7HjC$$>ctq2F-=CqL_@`~=9Nfo<+cypz}txepvO2W#3 zSI2R*T0Y~4T%_k}_`5#bqPvk&{6%!|OQP?kxC=fN7PbfL!`gkE#inlC!|O_3vfktC z+UOuVK)x?%&k+5|TQI@aw`#Sb1 z5%VFVu-+PfK*zvuTVl3&MV>9aJSAtp35Bh8q-ho$e zz)zXb<58LbAJIs;M8=*C*zF|OD6r5cG*lzQ=nES$dQLswC#78Lxzah!<84rXJ=+O~I zpvPOCVFY?~gc0cR3b!l$n2q-}2DUCcp7OMW(?8`O%R~6K`XLR|-tP*oVSH?Lx3_vx z&CA0Yla7nS6l#3EzuEHc<3QzZUg>-l??Zj)@WiiRrolmcd$#j-{|4vsPGo$1Z==}6 z*DtB?;~RMhpghaIzp*;cjCL?nOoRygI7xK&_x9^3 z*{9tnrRMRg?xtbk(t{pC#EEPQvy0wE6^gu{8oy81;inQ=YlbJU${%atsq<~PG8r%V zi2)zv{P92ZgB%9=rc9mV2gc+(K|l+9fOd~H(3yAQBz0K3Q3?8pk=t3^g~DejxLja0 z`5uSyNA8uYdT(?Hz4k8q%Gcgl##Tz|q`2ZFgexDo+_AdIVBU;^(>8^6jGS18kq&NP z6m#tXA|*8#BNzAB=Ww=4#?No2Fd9YJc4eELW9X(;`1g8c!sR+Gl&Y88&Ux}F5)-^m zz{znZjw%U%)04M=`I1o-{CiwjMed*zhF6=z*}3K-AAFJ+AoCgy-1cG{jxwzO$S9JE z`v7<(z<&XMfd7@JmOtZT7oRpI+mA($ql^!piTGqm(pii8aWflxy5U7E}_^S3eu}SjUik=`b3E=_` zFDs1!Hl{0**ht=U@Gs;tSg z<<&}5Z%lAaOaQd@Ak&CH<&h`2G9@r+l4XSmEGnAAMvKS4snh&$p$MH{yRzK zbp2TDo;DodHC$c10m9=7h@Z`i@C*Rtp9m4J`8bas_4KFwi6g!K=nquRqUs%v_j!Rk zX{(3{I=}evS_7g+*VgaJFeW1DQ*PJ}e`812z}Gl|@3HvTEPe?ER3cI7qnLlhR>|Sp zgcwmZ`2c+TauBcdh2h}xPr<1*7-UB(toF~C2B@1qhW0yzewW3!So~)e{{@9aBY$>$ z94<&6r{idF$Il)+HF{icn6}fpRTc{@uA$J;J-$R8yG6y(5&53WvU+T0 zNYHYu(vF>ORnnY%j-YsUe1wM&aX-{Y`jrAa?Nn|!&vxzn2LFl^lFQOfB=oSjn?)8y zlRm}r5#to(?)FQ`sIl+&=N$BZu&A>5pDdVX_UY;$5Sl_E_t598(@lZE`0)SDE|*#S z1^dxRmg}=~$PNTFGr^NAj<9%!#W0IwEGUn`2n!mNV3fu4EKajH!(yBTxk(>A@n01A zJgSK)rukLaUG({ZV(|Ca`7stxv3Q!rQ5HECCs;hk;v|bxEXG*8z~U^67g?+js0Gu! zaD@d7ZTyPo5+Nvw_+`)c2z`ac&$9S=7QfEo>n#2Ti@(X@n=F2h#dlbIz~T>C{7V-9 zfyJM&_-`z}&*Dcclr3ip)hAQe6H?k}Hz6hkTE1*>58FP?;xjBBVDTV}{Vez%Pw)_m zmOc{ni4D_w%;Krpx|^?tPu~JK;+g8??ex~)9bMbH;sZwfoBUcwj*_cvX@@`Dqy6g5 z$7$V*G5rVTZc=vL)z#nM->rYW{XH?ndJFFBzqNlu|3>ud@9)Kx{{Ah3`_RgNoBHqS i-+?~sanIKNb@=Psd(nFx{@sT5?P}`*#HaN2um3-@% 0: + count = 0 + sequence_cnt = len(accession_list) + divisor = utils.get_divisor(sequence_cnt) + if sequence_cnt > 0: if not quiet: print ('fetching sequences: ' + mol_type) - target_file = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) + target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) + target_file = open(target_file_path, 'w') for accession in accession_list: - success = sequenceGet.append_record(target_file, accession, output_format) + success = sequenceGet.write_record(target_file, accession, output_format) if not success: failed_accessions.append(accession) + else: + count += 1 + if count % divisor == 0 and not quiet: + print ('downloaded {0} of {1} sequences'.format(count, sequence_cnt)) + if not quiet: + print ('downloaded {0} of {1} sequences'.format(count, sequence_cnt)) + target_file.close() elif not quiet: print ('no sequences: ' + mol_type) if len(failed_accessions) > 0: print ('Failed to fetch following {0}, format {1}'.format(mol_type, output_format)) - print (failed_accessions.join(',')) + print (','.join(failed_accessions)) def download_sequences(sequence_report, assembly_dir, output_format, quiet): local_sequence_report = os.path.join(assembly_dir, sequence_report) replicon_list, unlocalised_list, unplaced_list, patch_list = parse_sequence_report(local_sequence_report) + wgs_scaffolds, other_unlocalised = extract_wgs_sequences(unlocalised_list) + wgs_unplaced, other_unplaced = extract_wgs_sequences(unplaced_list) download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, quiet) - download_sequence_set(unlocalised_list, UNLOCALISED, assembly_dir, output_format, quiet) - download_sequence_set(unplaced_list, UNPLACED, assembly_dir, output_format, quiet) + download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, quiet) + download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, quiet) download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) + wgs_scaffolds.extend(wgs_unplaced) + return wgs_scaffolds -def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False): +def extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, quiet): + if not quiet: + print ('extracting {0} WGS scaffolds from WGS set file'.format(len(wgs_scaffolds))) + accs = [a.split('.')[0] for a in wgs_scaffolds] + wgs_file_path = os.path.join(assembly_dir, wgs_set + utils.get_wgs_file_ext(output_format)) + target_file_path = os.path.join(assembly_dir, utils.get_filename('wgs_scaffolds', output_format)) + write_line = False + target_file = open(target_file_path, 'w') + with gzip.open(wgs_file_path, 'rb') as f: + for line in f: + if utils.record_start_line(line, output_format): + if utils.extract_acc_from_line(line, output_format) in accs: + write_line = True + else: + write_line = False + target_file.flush() + if write_line: + target_file.write(line) + target_file.flush() + target_file.close() + +def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, quiet=False): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -108,11 +150,23 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False # download sequence report if sequence_report is not None: has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir) + # parse sequence report and download sequences + wgs_scaffolds = [] + wgs_scaffold_cnt = 0 + if has_sequence_report: + wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + wgs_scaffold_cnt = len(wgs_scaffolds) + if wgs_scaffold_cnt > 0: + if not quiet: + print ('Assembly contains {} WGS scaffolds, will fetch WGS set'.format(wgs_scaffold_cnt)) + fetch_wgs = True + else: + fetch_wgs = True # download wgs set if needed if wgs_set is not None and fetch_wgs: if not quiet: print ('fetching wgs set') sequenceGet.download_wgs(assembly_dir, wgs_set, output_format) - # parse sequence report and download sequences - if has_sequence_report: - download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + # extract wgs scaffolds from WGS file + if wgs_scaffold_cnt > 0 and extract_wgs: + extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, quiet) diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index fe2b66f..400e353 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -40,6 +40,8 @@ def set_parser(): help='Destination directory (default is current running directory)') parser.add_argument('-w', '--wgs', action='store_true', help='Download WGS set for each assembly if available (default is false)') + parser.add_argument('-e', '--extract-wgs', action='store_true', + help='Extract WGS scaffolds for each assembly if available (default is false)') parser.add_argument('-m', '--meta', action='store_true', help='Download read or analysis XML in addition to data files (default is false)') parser.add_argument('-i', '--index', action='store_true', @@ -50,7 +52,7 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.4.1') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') return parser @@ -62,6 +64,7 @@ def set_parser(): output_format = args.format dest_dir = args.dest fetch_wgs = args.wgs + extract_wgs = args.extract_wgs fetch_meta = args.meta fetch_index = args.index aspera = args.aspera @@ -93,7 +96,7 @@ def set_parser(): elif utils.is_assembly(accession): if output_format is not None: assemblyGet.check_format(output_format) - assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs) + assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs) else: sys.stderr.write('ERROR: Invalid accession provided\n') sys.exit(1) diff --git a/python3/enaGroupGet.py b/python3/enaGroupGet.py index d8a9ee7..c4af82e 100644 --- a/python3/enaGroupGet.py +++ b/python3/enaGroupGet.py @@ -28,7 +28,7 @@ def set_parser(): parser = argparse.ArgumentParser(prog='enaGroupGet', - description = 'Download data for a given study or sample') + description = 'Download data for a given study or sample, or (for sequence and assembly) taxon') parser.add_argument('accession', help='Study or sample accession or NCBI tax ID to fetch data for') parser.add_argument('-g', '--group', default='read', choices=['sequence', 'wgs', 'assembly', 'read', 'analysis'], @@ -54,7 +54,7 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.4.1') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') return parser def download_report(group, result, accession, temp_file, subtree): @@ -66,77 +66,71 @@ def download_report(group, result, accession, temp_file, subtree): f.flush() f.close() -def download_data(group, data_accession, output_format, group_dir, fetch_wgs, fetch_meta, fetch_index, aspera): +def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera): if group == utils.WGS: print ('Fetching ' + data_accession[:6]) - if aspera: - print ('Aspera not supported for WGS data. Using FTP...') sequenceGet.download_wgs(group_dir, data_accession[:6], output_format) else: print ('Fetching ' + data_accession) if group == utils.ASSEMBLY: - if aspera: - print ('Aspera not supported for assembly data. Using FTP...') - assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, True) + assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, True) elif group in [utils.READ, utils.ANALYSIS]: readGet.download_files(data_accession, output_format, group_dir, fetch_index, fetch_meta, aspera) -def download_data_group(group, accession, output_format, group_dir, fetch_wgs, fetch_meta, fetch_index, aspera, subtree): - temp_file = os.path.join(group_dir, accession + '_temp.txt') - download_report(group, utils.get_group_result(group), accession, temp_file, subtree) - f = open(temp_file) +def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): + temp_file_path = os.path.join(group_dir, accession + '_temp.txt') + download_report(group, utils.get_group_result(group), accession, temp_file_path, subtree) header = True - for line in f: - if header: - header = False - continue - data_accession = line.strip() - download_data(group, data_accession, output_format, group_dir, fetch_wgs, fetch_meta, fetch_index, aspera) - f.close() - os.remove(temp_file) + with open(temp_file_path) as f: + for line in f: + if header: + header = False + continue + data_accession = line.strip() + download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera) + os.remove(temp_file_path) + +def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs): + temp_file_path = os.path.join(group_dir, 'temp.txt') + download_report(utils.SEQUENCE, result, accession, temp_file_path, subtree) + header = True + with open(temp_file_path) as f: + for line in f: + if header: + header = False + continue + data_accession = line.strip() + write_record = False + if result == utils.SEQUENCE_UPDATE_RESULT: + update_accs.append(data_accession) + write_record = True + elif result == utils.SEQUENCE_RELEASE_RESULT: + if data_accession not in update_accs: + write_record = True + if write_record: + sequenceGet.write_record(dest_file, data_accession, output_format) + dest_file.flush() + os.remove(temp_file_path) + return update_accs def download_sequence_group(accession, output_format, group_dir, subtree): - print('Downloading sequences') + print ('Downloading sequences') update_accs = [] - dest_file = os.path.join(group_dir, utils.get_filename(accession + '_sequences', output_format)) + dest_file_path = os.path.join(group_dir, utils.get_filename(accession + '_sequences', output_format)) + dest_file = open(dest_file_path, 'w') #sequence update - temp_file = os.path.join(group_dir, 'temp.txt') - download_report(utils.SEQUENCE, utils.SEQUENCE_UPDATE_RESULT, accession, temp_file, subtree) - f = open(temp_file) - header = True - for line in f: - if header: - header = False - continue - data_accession = line.strip() - update_accs.append(data_accession) - sequenceGet.append_record(dest_file, data_accession, output_format) - f.close() - os.remove(temp_file) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs) #sequence release - temp_file = os.path.join(group_dir, 'temp.txt') - download_report(utils.SEQUENCE, utils.SEQUENCE_RELEASE_RESULT, accession, temp_file, subtree) - f = open(temp_file) - header = True - for line in f: - if header: - header = False - continue - data_accession = line.strip() - if data_accession not in update_accs: - sequenceGet.append_record(dest_file, data_accession, output_format) - f.close() - os.remove(temp_file) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree) + dest_file.close() -def download_group(accession, group, output_format, dest_dir, fetch_wgs, fetch_meta, fetch_index, aspera, subtree): +def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): group_dir = os.path.join(dest_dir, accession) utils.create_dir(group_dir) if group == utils.SEQUENCE: - if aspera: - print ('Aspera not supported for sequence downloads. Using FTP...') download_sequence_group(accession, output_format, group_dir, subtree) else: - download_data_group(group, accession, output_format, group_dir, fetch_wgs, fetch_meta, fetch_index, aspera, subtree) + download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) if __name__ == '__main__': parser = set_parser() @@ -184,7 +178,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, fetch_m if utils.is_taxid(accession) and group in ['read', 'analysis']: print('Sorry, tax ID retrieval not yet supported for read and analysis') sys.exit(1) - download_group(accession, group, output_format, dest_dir, fetch_wgs, fetch_meta, fetch_index, aspera, subtree) + download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) print ('Completed') except Exception: utils.print_error() diff --git a/python3/sequenceGet.py b/python3/sequenceGet.py index b57b90d..2ac1fef 100644 --- a/python3/sequenceGet.py +++ b/python3/sequenceGet.py @@ -23,9 +23,9 @@ import utils -def append_record(dest_file, accession, output_format): +def write_record(dest_file, accession, output_format): url = utils.get_record_url(accession, output_format) - return utils.append_record(url, dest_file) + return utils.write_record(url, dest_file) def download_sequence(dest_dir, accession, output_format): if output_format is None: diff --git a/python3/utils.py b/python3/utils.py index 92eba10..ede4ea1 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -27,6 +27,7 @@ import sys import urllib.request as urlrequest import xml.etree.ElementTree as ElementTree +import urllib.error as urlerror from configparser import SafeConfigParser from http.client import HTTPSConnection @@ -68,7 +69,7 @@ SAMPLE = 'sample' TAXON = 'taxon' -VIEW_URL_BASE = 'http://www.ebi.ac.uk/ena/data/view/' +VIEW_URL_BASE = 'https://www.ebi.ac.uk/ena/data/view/' XML_DISPLAY = '&display=xml' EMBL_DISPLAY = '&display=text' FASTA_DISPLAY = '&display=fasta' @@ -83,7 +84,7 @@ WGS_FTP_BASE = 'ftp://ftp.ebi.ac.uk/pub/databases/ena/wgs' WGS_FTP_DIR = 'pub/databases/ena/wgs' -PORTAL_SEARCH_BASE = 'http://www.ebi.ac.uk/ena/portal/api/search?' +PORTAL_SEARCH_BASE = 'https://www.ebi.ac.uk/ena/portal/api/search?' RUN_RESULT = 'result=read_run' ANALYSIS_RESULT = 'result=analysis' WGS_RESULT = 'result=wgs_set' @@ -106,7 +107,7 @@ sequence_pattern_1 = re.compile('^[A-Z]{1}[0-9]{5}(\.[0-9]+)?$') sequence_pattern_2 = re.compile('^[A-Z]{2}[0-9]{6}(\.[0-9]+)?$') -sequence_pattern_3 = re.compile('^[A-Z]{4}[0-9]{8,9}(\.[0-9]+)?$') +wgs_sequence_pattern = re.compile('^[A-Z]{4}[0-9]{8,9}(\.[0-9]+)?$') coding_pattern = re.compile('^[A-Z]{3}[0-9]{5}(\.[0-9]+)?$') wgs_prefix_pattern = re.compile('^[A-Z]{4}[0-9]{2}$') wgs_master_pattern = re.compile('^[A-Z]{4}[0-9]{2}[0]{6}$') @@ -125,8 +126,10 @@ enaBrowserTools_path = os.path.dirname(os.path.dirname(__file__)) def is_sequence(accession): - return sequence_pattern_1.match(accession) or sequence_pattern_2.match(accession) \ - or sequence_pattern_3.match(accession) + return sequence_pattern_1.match(accession) or sequence_pattern_2.match(accession) + +def is_wgs_sequence(accession): + return wgs_sequence_pattern.match(accession) def is_coding(accession): return coding_pattern.match(accession) @@ -235,9 +238,17 @@ def is_available(accession): url = get_record_url('Taxon:{0}'.format(accession), XML_FORMAT) else: url = get_record_url(accession, XML_FORMAT) - response = urlrequest.urlopen(url) - record = ElementTree.parse(response).getroot() - return (not 'entry is not found' in record.text) and (len(record.getchildren()) > 0) + try: + response = urlrequest.urlopen(url) + record = ElementTree.parse(response).getroot() + return (not 'entry is not found' in record.text) and (len(record.getchildren()) > 0) + except urlerror.URLError as e: + if 'CERTIFICATE_VERIFY_FAILED' in str(e): + print ('Error verifying SSL certificate. Have you run "Install Certificates" as part of your Python3 installation?') + print ('This is a commonly missed step in Python3 installation on a Mac.') + print ('Please run the following from a terminal window (update to your Python3 version as needed):') + print ('open "/Applications/Python 3.6/Install Certificates.command"') + raise def get_filename(base_name, output_format): if output_format == XML_FORMAT: @@ -267,14 +278,16 @@ def download_record(dest_dir, accession, output_format): print ("Error downloading read record: {0}".format(e)) return False -def append_record(url, dest_file): +def write_record(url, dest_file): try: response = urlrequest.urlopen(url) - f = open(dest_file, 'ab') + linenum = 1 for line in response: - chars = f.write(line) - f.flush() - f.close() + if linenum == 1 and line.startswith('Entry:'): + return False + chars = dest_file.write(line) + linenum += 1 + dest_file.flush() return True except Exception: return False @@ -352,13 +365,22 @@ def set_aspera_variables(filepath): try: parser = SafeConfigParser() parser.read(filepath) + # set and check binary location global ASPERA_BIN ASPERA_BIN = parser.get('aspera', 'ASPERA_BIN') + if not os.path.exists(ASPERA_BIN): + print ('Aspera binary ({0}) does not exist. Defaulting to FTP transfer'.format(ASPERA_BIN)) + return False + if not os.access(ASPERA_BIN, os.X_OK): + print ('You do not have permissions to execute the aspera binary ({0}). Defaulting to FTP transfer'.format(ASPERA_BIN)) + return False + # set and check private key location global ASPERA_PRIVATE_KEY ASPERA_PRIVATE_KEY = parser.get('aspera', 'ASPERA_PRIVATE_KEY') if not os.path.exists(ASPERA_PRIVATE_KEY): - print('Private key file ({0}) does not exist. Defaulting to FTP transfer'.format(ASPERA_PRIVATE_KEY)) + print ('Private key file ({0}) does not exist. Defaulting to FTP transfer'.format(ASPERA_PRIVATE_KEY)) return False + # set non-file variables global ASPERA_SPEED ASPERA_SPEED = parser.get('aspera', 'ASPERA_SPEED') global ASPERA_OPTIONS @@ -389,8 +411,6 @@ def set_aspera(aspera_filepath): def asperaretrieve(url, dest_dir, dest_file): try: - if not os.path.exists(ASPERA_BIN) or not os.path.exists(ASPERA_PRIVATE_KEY): - raise FileNotFoundError('Aspera not available. Check your ascp binary path and your private key file is specified correctly') logdir=os.path.abspath(os.path.join(dest_dir, "logs")) print ('Creating', logdir) create_dir(logdir) @@ -587,3 +607,26 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + +def get_divisor(record_cnt): + if record_cnt <= 10000: + return 1000 + elif record_cnt <= 50000: + return 5000 + return 10000 + +def record_start_line(line, output_format): + if output_format == FASTA_FORMAT: + return line.startswith(b'>') + elif output_format == EMBL_FORMAT: + return line.startswith(b'ID ') + else: + return False + +def extract_acc_from_line(line, output_format): + if output_format == FASTA_FORMAT: + return line.split(b'|')[1] + elif output_format == EMBL_FORMAT: + return line.split()[1][:-1] + else: + return '' From 83da583154671cbdde470ca630043612a4daa423 Mon Sep 17 00:00:00 2001 From: nicsilvester Date: Thu, 10 May 2018 08:44:01 +0100 Subject: [PATCH 10/18] v1.5.1 release Issue 35 fix Removal of wrongly committed files --- python/assemblyGet.py | 2 +- python/assemblyGet.pyc | Bin 6569 -> 0 bytes python/readGet.pyc | Bin 4258 -> 0 bytes python/sequenceGet.pyc | Bin 2844 -> 0 bytes python/utils.pyc | Bin 24461 -> 0 bytes python3/assemblyGet.py | 4 ++-- python3/enaGroupGet.py | 2 +- python3/sequenceGet.pyc | Bin 0 -> 2849 bytes python3/utils.py | 2 +- 9 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 python/assemblyGet.pyc delete mode 100644 python/readGet.pyc delete mode 100644 python/sequenceGet.pyc delete mode 100644 python/utils.pyc create mode 100644 python3/sequenceGet.pyc diff --git a/python/assemblyGet.py b/python/assemblyGet.py index 2b890fb..901555d 100644 --- a/python/assemblyGet.py +++ b/python/assemblyGet.py @@ -82,7 +82,7 @@ def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, divisor = utils.get_divisor(sequence_cnt) if sequence_cnt > 0: if not quiet: - print 'fetching sequences: ' + mol_type + print 'fetching {0} sequences: {1}'.format(sequence_cnt, mol_type) target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) target_file = open(target_file_path, 'w') for accession in accession_list: diff --git a/python/assemblyGet.pyc b/python/assemblyGet.pyc deleted file mode 100644 index d5e60dc593ed1787a018b404519d5f056b86eedc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6569 zcmcIpO>-Mb8Sa*4*|Oz0alTm~A}m`9B=NE=yAYNoj^iXQ93?w)@Z#F6YN8q0Badg~ z>DJmI)+u}9!f)UQaN5IGapO1eH#ks5)n4Iw-ky;hZzwo8k=5$y>5sSHkLP{5Gx4vv znZND*W3!|3eSz;zh!)v@im{II8@3i z)zeaz)K*FDOsVXlvJRsL6BA};F^M{`$>QC#GuRCiYZkX6yBH?^oh;o)hxIg# z&0_z#-AX!hhxR6#C}?Lp?Cj(VT*b5D=9IpqtNOboF&3rgcNt$%=)nr@)f2ae z{_j40Uk$Z7goWVrVufBw4NK|}em|jhN-8^Dye-p4c-z`Ltu_!itp**VX}RbVZ>_zy z+Wf8PW^L)t%9@!+b1U2%gh?l?dtpD#EQ~E(8&sBP3`)$T(v6ZpDmFZ@Qfa11D4rl% z&caTbxgK$p?8rK?|16A8T(K7++g51XCYQaPiR|q`7GJ|FGlN3u3wlA<^f@hB=5|I} z2AHUW@R*j@|BIHL-Y0U+4ks8ihw>PdMf+{)3CID}qx~X!AV^>wKD0K(3iQmTokF>B zztwK7tc$`Pdn@-=Kb7X+t$3}~M@{kn2;EeJL%dTgXeEz`A-0L(gxrX+nMW)TLwQ6I zY2M#*anR4gZge1Xe*rf}gq^q*xF)TDC1DBMc1ylX*(kR&zygeL7S2900U*g`s#|-_IZr)eZTwG_~G& z)V=ScaDL5rIM|Kb-6#&7?x^Hr?j#fBn|mY56EWKbXtW{FFDV6uQ~Er7x8~Hw2F9_d z?PGu!)aG;UOUBCYFPR96tK$iz3fy}_RXuU`9{7`Lv)7^ zO{&9+I-FA7a)F5wYKEDh8cm>dQAbrnHOLHoI8+stEc64wS!l$wI&limqA^TAj;sKy zxAJ&xb@@?KT=7YB?a}hm+G=a%u0!4C*<@F7PuLP%C=n6kVKN`PzPFvFNub{x0}?1T2U<9(8FM z>XFG|%OyXD3Pi0DA#vK{0AG~vfWUP(GDm8+!)#4y-NVLGiYP$s|=-jr1hKS-}V%px(2rS3n#ApC7mWTn6L7NTR_ zUQH1TlLgLVu4wT3@y^ggQg*)kmIm4962<=RKLF{+5yJHnZAdC7CNu>sgrj#uknSki z0@2N_bE_^PyzcWWekg0I3mh10HNkrI$vD&{TG0 zGEDs4(0d0x1<;aBwO(u}@{!$AeKbxz(b=rg)Q zq@;DJazV_$qT;v{V;L?B2894D3wM%3z-MKKKLQE~5Md5$BX|&PpGnvvRH1zb0&JOO zXo8c%*%{H8P~h|BBu;;5MQbrv^J2$2R#!VR7C{g7`Su%M55t1ND^c#p-;SiH~T z=P1U&UYxGQ6CJQ0$*s=$L34}LYENNw(amMs6Za$1>u>pyiO?#*Lxy+>F$^Qz8nzHjV?N~>^D^Cqu#7|Hh$J%WzAXu4ln2lqsi49e z0b)851sxA*i8fAZzoTb#hgx_AabGZy2&Sdf4t)8~GChJqhPzmTAP8o}4+rLw3D)K_}Rv)EUoCs(p)oSn7=-KIJ&Pqhew8SfHP#2_!f6D?&9G`ocSq= zF$J@I_v-h2CBPsMK^XFU7I%y8qeUUR2g5t=CA5nRQ8@20M=)fC{>W|#Cc^CsK;Uo* zXhyp9bH`HLMtE;?JQJ!ti(=d7J&$9#`Hh=ko18>3O}oHDGPDy9Y*NJ0o(nGdM>HCy zb9cx@D3>niOQi)}Db@6O1T;TaKw zWzljrNuS#A0I#c%&^@x5CS+m4&WqWm_ua$e-WF5elZ@2>Bg8&jTyk0?vHhL9+WycX5!wcUvER;$gfF} zf3L?80hspsfl|=f5O#wpFSJF&krQPy%9MB@Zr&l z?2x5*nCq_Cyc(5jdk=N*%y{psXtWVmUBz)Dc}*(Q_6>|%<#~~pNtzXhje{(=jq@xV zHBQrCG}O9HPj4QC-e#-inM>bYzuDvhP3d-7SPe+{0IWkBI*IRDmvkt%L%AKu?Wo-F z?sy4Azjc23KSk*Mqu9wFH6t*{dP3jj`jT27I%(ztQyC}f)f=gD<0+7YU{8Bci6*9F zU;YT^`ZWw<>Y4)U(S7}1F`5d@(kX2I2`?keydH0NkJ!C~3peE=HD1lg5V4&RfZ$uF zkPe*PXF@%FHo;Anjkc#&PZH>vHbU8$=sF5|au6)g@w{w|4e|=MqiEj3Mzs%^<%5rz z`c1uQOo*6<#_Jq5Fa%ya-&cseQEw9_CrYckf60TEFo?NrX7E(aJ9xUZmJf4WFB{Qr z>|woTr1er6x>)g{ea&2E-t-`6yQND5vGNbwpO1mQN%4V@GA$+Qh5wzsr#2+SW6zUi zWFsBT3|AW+h-nr9k%yz=i}X=I9##am0l9OrDafO(7eABXEK|ij=H11_*+;_UL3y5) zS8$LPX$S&v+#6QqIp+5e+d-9>1PmGZj_0>M@wcXJL6etplr@tJZZCiV8B12j+r)K` zI%j?BF2fzH8p)>EGE6yNYpw)MYK+j%d^4;=I=Fir_ydTuJ7QGX?k5djUj%7IZrLb zEaSgw7V+@^o|!dWs(cp@P1c|vAsa?PXSGnMKx7at#2Q5!u|fb1Ul42I%McMn(6?oP zhHP8l1;Pu~A)q7da87q)oo?Xk{F{p?ucjRAxZZZ2l+0KDjt%7c`P=MvVhWMTEzi%(>@PT7NXT&xWlz$Q|+ zWw62an~m1D2C0Zws51Z`Zk`cy_ANhqY!H4NUHn`ID-hO{;T;*=nQVNm8^Kwlu9J=>2ciCw2 z_Mq~iWA=J)dr3G_;<(I;5%3VJyJ&U;3No4i)(tdgS2s;@L+?M>K#ll^l*yu|y#)*T z8$d-Mn_k%ef>vb@_{M_3CYxB|6{os+Iffc%Ps^7!ypJWl#Cbbv7{=!Sl=^w-CoiZ*{W(pk4(`zZ+G##?+!Js1>t@xm8fN6v7v{sdrel#+rJ~=JZer|ug X0a|Mk4XsD@s5X!PdaYKg)aw5RnZ(1d diff --git a/python/sequenceGet.pyc b/python/sequenceGet.pyc deleted file mode 100644 index 3f4c17610c3eaa7e4bd3fb8574cef1341340b80a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2844 zcmd5;-EQ1O6h7nq%Z3eUNn270gsFtI2*`pE5<&O>;t{yz0pL5wYllSTrrHaV_4v%0@%((}JKs$0ucgIb|M+t^pxK{; z_kBF(dlVsliejSO%2u?iO20$94uuZ=gbp8%3y%9q&i6mYN>~+L$O!>=N42+)i_}rrEJ? zEzg3nx_FTkW*t^<$LP%Tk|;=H-Fuh@`LRx{=^aG2rxSl8OHa{ZJ56KLo1WRjH0c@r zL#~rR_qAP{o_P%UT|9=@MCvtFp}`J8<0}{dAtX9lplp?_q6q+TD8VjZ3m`L~Y77)0 zGJQsQQojVS{}|oTGV1b_1H+rxN&LN7yEb+AwG9qkM#9C0-De+Oyw3)9?mqkIqRa4F zUeF*-w1+p@S5RzjZ9Ex2`r6xC-?q!Bhv{h&r+zrY**4)TMkyfYmhga6yAscsd(bC@gOKSrgZ;d4dVTVz2%z*w*nV1gG{ z>IhQFOW7aKVeg=b%s5X@Ko-zThvU-&Gd9}VRWw!0yVgOzs~WF-E2Q^01eY+*9XbZ6 znjQ;&2X(_CCAF+>I6_noU4_?gr~n!e&t(=?l)d*N$|9c&f+D@a+ImUl!rzdzhrc2I zCY^)IlJ*y{n`C}0{peZxnc!&pg=J~dDx>63|1soA_%Gq%^$sH5i1~oc)cv+UauJP! z|Ne;|#gZ2<$OJRQ7sFWlM!Nxo!Vj#wIaqhQp>KVY@0t60FIw}1wfyK^H`;g4(j2*^ zW7i%=M&>YwX}o_4E=L<>EJQc}y~G?9E_9b=~K+o=c{uX(t$@#M<~b`j=iXE^jW zN2AS$B5~ZCdvX?g+}@QTud+mPa+>M==&4BMG~bJ(fVo&nrOBt$(%~@)=FKec1_pf( zkKxQ9wJ2G*s#YO?P07ZaMKk%FDA0{6X-K+LV1K%mHmmMyX`$D*3>$f|Tfl zgRJ=M=kl|Xa4Na;A4J0!BN4no#H;^5v+ND5k9m`%*TiW?Id26WyxUyd;zDS}VYwzg z9!Ju9oeP$*b%4({)^IsEzkyNcH44F(h79=bi62K{d2F(H zNYYogiqH61U5t2{UA}z$E7>{A9<7hI*UQtCFCbfRYaluVmbHcd4mz@2@Z8|G@GH%3uOr-By>mJRmEnRX_no7 zb};obQ(PnD{^Rj(JwB;L89ZLK_cn@48i_TZE0-+d^W!wkWBmn_+OQnq9^iDGcD=pO KZnl@&%l`n+a)ZbK diff --git a/python/utils.pyc b/python/utils.pyc deleted file mode 100644 index a3d1deaed4f8649dda6395104949da8ac4e1513f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24461 zcmd6PdvILWdEd7H2oQWfBvQ0qD_N!?i3BK$qMnop5=#jb2|{;4ky@HsE%#gi3odrS zy$ga61z8ehS+Y~vNt8#EG<7CPJ<~erxN+Q>HZ!fKlYjb{wn?3|{ip38?R3U%CvDuO zPVJ_@-|yUecNc=_$^tqSwFloh_uO;O`QG1m&b{4#x_QH&*ZxzXZ1I2H_#@>SbL*_M&0MdQwwuFPJ5=s8hp}!k*KdVe&26y4 zE^`=bx4BJLxXl~}+GB2u6+UHds}*iHx6KN7nA>iJJI(E|!d>QeTH$VUw^$)-?p7<@ zV{Vrf?lrgD3ZFK2n-%Uex5o;fG50Ae>@|106+Ub34l4|pyVDByo4d;j516~#3J;pg zT4A5Ld#teE+`U#9H1}yM44J#n3J1)6#tILa+iQh`8sTAcpS8jvjq`}P0V^CfcfS=L zHTQrO9y9l#6&^RY&k9eN+i!&@%?(=NDRV|VRMJ9 zaLn8zRyc0%uoXtkJ!*xVxyP(M>RGN%gp@6G?SM)w4-8r|OGI^@OVDlIpXn zo=>XJsrph<9aXiER8OjUA*nvE>gST`DOE2f)zhlJoK(kDEn3)PGq2c8sSUWqOT}&zX9diddr`n;bLRw9&7Bu8YwjfhHFE_4 zb#oU4%$fU~fGg%M3TT*nSwLj2D4=QX6#?_+N&>E$yCh)2Tv@<1b56jbxlq8X<|YMP zH#a5VHFMJfmdsTIyl!qrz~{|f7Vw6-s(?4m%?fzSTus0Yb9Di4o0}8x1#?#fyko8* z;EU!W0q>e?3V6@lynrv6yDH$z<`x9}l(}mH{*t*x0q>i8RlrxwT^I0GbFT^bX>&^g ze#YGE0)E!q=LLLV?hOGyXYNe_KX2|W0l#4GhJar*_qKq)Z0-vJe#zWB0)E-t7X|!^ zxpxKp6?5+i_*HXX67W|kR{4Q1L!DOs0{@K-WB@j|p~#me!{hbZWMyi+)QG}Hvj>f( zs5}?l4q#}m)SMn_)`v>bT-YcTT~r#Z&xN%pnr`BL0Nt4CU(XHsc@|6d1y}5hnOalA9Ubs9t(x=6Z5+^(`u5Ryqrf-ezMC;2JhW8gP zUVmum!h!vdU%dXv(!l2i1wMHHGxxRkI2iXh+}Y#q))2EdjrcZARx3bj zZa5jFbEf&E87nAL6H}*AJ08a``aB9K*Vl|@z42LV!p+UtOjq0j&2*#7mNHh`%1h{$ zwm|dDth3kT4=i;V)Y?p+z1GEXX4cycv!!lx7za*Nzg_OL`s)UD&FV&8MK^f1mwR9- zIH56V&5@l=kW_GeZUxOvae*@%Ysl!TCgqo|kCWT`vY!(|~ zx!!Qa`9@V*cCy}>h4v-M&!0Y3JaHyCJv%JphEG*U0mb4tu8cPsgi@mh8e9DACk=hvdhyecQ~jaf~Ah@4>u(vndmI z?HHeXyo0M)g*Hfe4ppd>q?(Y2KABjGn1Y`-S-IA+#`KzZw~}j3vh;lIDkKRKC^UE8 zyh4Z05!Qt2R7EM(n?H?Ov>t^)F8S{hNoGJ%7Q+T3fYxOGbaXKP6tSU;Ik@=?xFRo^ zYx(dxZYq5gW4MqB?bc4{=)7b4c-Aj-x<8SskY!4J^tGQeF^4&~!h&D@iKk9iA#rWj ze&VrmP9FGcpKRK(tNs6O+VKN_<7U(TICfJp5kt0Q3B`xR=ssBLxUe?Hl8Oe5q_FYp zH(S+%xY(h$AL!KEsJ7|tL{Za=^HGI@(B-Qn@A>dFy>f7+?hmht-inQupTJ+|R5)GB z-1RY4RWOdb)=obWNg&Z2;{Q@((Ua=8(e@K*s^H9;&XszQ>;#v)=F-`I^JcUC7~&EH z$>f7PpfzC%##8Os7TVk8L!D?87+3AThsm9D@>OzliYd=^5PE4@@opX zG6>=F=W08w2j2bV4c6FenM?-4oImf7oYTC=d;mjNzk(DL#blx}A1a7?u~eN8bBzY_ zDSXC>QZ)+KVzB{3gujEIqIJu3XEtQkXEv`Os~@0Ky1d3V#;AF$%`dl+rJ~AX4RQT| zVcvO!hH?FDJc6pYK4-|XU`q-(Lf*e~o$hl|p!BJM|+8jSKGCKBb@D%GDAfJ2TY;Np$PWaRmCWg*>a{hYo&0ge;%WF>Y%b1)-@$aEy(k!n_WaminQP;Z{C6C`Xck3_(z}>=0I@H3 z0sjfTqe0>%ro~0(A{x)e{jg-Le+mhQUd#Ji+0>gf)gU4mu#b3~f^48yEDf|Ao+MPx z`LoAPk4{YFM&e|GsOhQ5(EG%2e&U5B&!GOnaB(;f(he6-jOI>_C^3*vYjlxF(Bzh6 z4D?p6LJ_xuW~KM)^Ub;WW-(4{km5ciM{!xD1OjtsEz0>P6-(7>eIayzglo!(l7<^H zw`F=WJG*veK1{0;I)V%!7wFSyrxja)UbMa-`aGG?r<=iFCwhp6!ofqbK0l!s3;w>0SqsOF55cqG= z-iE+U2?801BtJ1bRRH z4;=ZQB$S$z8dX?22mss#L@{HF3@`L1(zW2XerrIDBKZdwUxpLy=+Mqa6rNy-1(%7w zX7xyt#|=CIGxbWG2_>dn7&VKo(r9sq%u_P!)ngUq@rV#Uu~I8Fnfu{meuP=1nU+nN zotczzQj}0=ffAp90%EY}Oq=X*l`^tEbu4)S*g#si5?&2G_dmi;f6jD%f!4+9s>>(X@yXyo;(Q z15q9b`G+}5ABt+F7S`rxCA)-Iks|bIG@+PTw~`!mGf0lDFoByhx5FdcoY|6i`6LXr zRm#FIp|W_CDjHTD*aTjI@IBmh(q0vGRg075NpNQh+#WOYGE`3p)$0>x7OF82c3yN| zVSoZG>|r~+(hOclHQ+`{@CJ*wS$u&7d0lArEY2wlnb^$b7F6&iufN5D$X;cXO6}u8 zR5EuGr3r`c#17B47KeXAL=uZo5Jl+&@GeVTCf84J#ls9;!diUSXoGaWUf4F=gv8^7tW`OBHkXAm2F%q z<8k_rGQ#Lm2Q?ttsbJ(~iN-31@WaREv>TrJID0Yrr0Sn|w~+*Xs#==8d%N!@}|RUt}+Gt?LBoqgA|c#!3qZ~ zawMvU^7MS|vKHgl`ip4$#LFUa$q#tQ5LZP$WXQ84b9ZL9_Z&!juJbs4(Krh507S)G z46lnWcUuD|2_ZaYyP>XjKl>aA4N3ye( zNSmwJ25)2&`xd$^F_YDil)>x>&eXu^nNp+pVk)fR92HJBWw8`(19KOl95;Al@auSj z zmSQ9RZHCQ|UrSCSIYbHue86NX4KfpQbzr1pGa>1N6_0I4VyPQbRQld(F9E>+K3>2% zM)<-=8sWwaNC&pg!{vty8_Q-(V42g%b?? z*BgubveD(r+#JXya}%tjCb_aoL=p$0(gUb_MJbm`B9@3pOG9#JOIAnjfgUR(5&Sh= zSVaHdA2b1B;ZeaU2crV=aSP1R zeq=Mf3w6yhAY2B7!HE-`rzY7pl8_)9{?CAgV@pNdGuB47J5g6&GfPJ*dtzcd+ia9- z(PY?oA`5zM7$CE+M5H!%;Op%2E(+qclq_hCCcvpt@4)P^bPV9#vv?8hM;q>=g%q(jgl zv1%aADnW}$2m-}q^j^i=AY(76)yH#0pd`tNlx%NGM&x9@!?B!!Hm70Y-1b>Sy zX_O|d>upj?;i}Q+wdH}HFP?cm_#NK&Ef&Aaq7}|{ioR~ZfZibuzJ|UUPGaf zJ^)BN;uHvg^Vz7wBkJ-B8g{~L`$o^tFZyIXmRH2?*dr-VjM8Tx_}p9aJ*mcV`lP;jEDEOGcZ0( zxXA?sw?OcGQ`c*M`n-Pov|7je2;AAd4)wc~-T%TwcK@mD zbvC0`&0dG^5jue54?le1z(d*Vo;YZzWUoWZE@cs!?Vm*J(d#7GQuaD__YfW|Nv&Z} z=@{Q84Wu|=4o_4E450v6kMYoSSgl4`boj9@&<=EJ@u2vvbSWY;c~&E(%TUeDP+5C% zrj)GoWU4lO{&D`&T%%5RN3%qbQ~gLy>holbTa@*002R%*o6|uCjg& zKEUI?O5VK>0Cnj-NIUj|b9>wV^@D$Rw!2M$mIPAW zZ!KY$4h}AqZL*`R_0M7q*-UKU7dv;77;3OJ-A&Sl+i7gL<$`3Rn8`l&iCeMd#0DvLL>24_O-TI4amt`UGH<%LP z75s&`Lz0;{lO@=lSI!}h&Xh~ym?^FhoB0h(da>M=ZUlb_NT*f;>Lc+IU-O!ew9Npe zV5S{7l$$2@@VM$P7`3b*G47YC<~- zA9Wo@7FH5kQGPdKCB#PYt^y&2$JAS_z)^<1hhGa=Z)Id)fL`lE_VIu%;oytxH=d9F zjrUZQVfM608(;3SEAQEoH0WzkugGM2=Ke0dh&&=V5Oy~Y#^_?a>CAc#XR@5pEnd>4(uzu*;$3+_*Hh@ijJ9kc~>Bw3@p;G}wL)u^dU%B(mt#?zdu4g2@bNL5!Su4rE3GEkvJ##BSI7*h2|~B;nw9JC%5obEz+2;9R|!d z>Y0>|Xcf8Z4h&4m!%L3w8TR<7b-cOoO|qKbMd{6C@oy{C*k%~w4Vfb_!aZH(L}d|w z$MKV&55`L)Jnu@SF`nlsEC#H&8(7~BD({AOmf<~MAlbqs8%-TGJdbxz8kJYI%c@2sVuKKp4Y9`HgwPI%BIdLDLhXW2 z-2Mq36Z{Gbst5eW zjvh)zCD8+k2(D}_9+C6UHxSaJPe<-WEODj2_9JJ@&U*82);eQ_{$s{^?k^srzWa)M&1LC+v z0uhOO-M9?2>uOxXaU|V?^EA9CZiPxwYhT(Ln|i%SJy{8>E;=F(-=~dk#WZaP?sR^R zt82AkYs8>B4x!)1{-sWp?-h z3WemJ0v#Rx7p}D^5&V!<{8mYSB`UrxYz8gJh5zm&$w03q`AOL`Cky^&6p*O=Jb2hE zp?0sIcLf;?9Q=qYP?b7hwWIXJEp6kf?{PU~V~YL;?-`(}wSjpbD(Up$Riq1O+(CiQ zL%iFULhq;T;H&(nvtz{|mp^-ILQf7!t6`adHwhym-C;y|@U?E6sxd(Ob;mzrk` zyUXAaGiLRzu$j=*u(L@l#<7HjC$$>ctq2F-=CqL_@`~=9Nfo<+cypz}txepvO2W#3 zSI2R*T0Y~4T%_k}_`5#bqPvk&{6%!|OQP?kxC=fN7PbfL!`gkE#inlC!|O_3vfktC z+UOuVK)x?%&k+5|TQI@aw`#Sb1 z5%VFVu-+PfK*zvuTVl3&MV>9aJSAtp35Bh8q-ho$e zz)zXb<58LbAJIs;M8=*C*zF|OD6r5cG*lzQ=nES$dQLswC#78Lxzah!<84rXJ=+O~I zpvPOCVFY?~gc0cR3b!l$n2q-}2DUCcp7OMW(?8`O%R~6K`XLR|-tP*oVSH?Lx3_vx z&CA0Yla7nS6l#3EzuEHc<3QzZUg>-l??Zj)@WiiRrolmcd$#j-{|4vsPGo$1Z==}6 z*DtB?;~RMhpghaIzp*;cjCL?nOoRygI7xK&_x9^3 z*{9tnrRMRg?xtbk(t{pC#EEPQvy0wE6^gu{8oy81;inQ=YlbJU${%atsq<~PG8r%V zi2)zv{P92ZgB%9=rc9mV2gc+(K|l+9fOd~H(3yAQBz0K3Q3?8pk=t3^g~DejxLja0 z`5uSyNA8uYdT(?Hz4k8q%Gcgl##Tz|q`2ZFgexDo+_AdIVBU;^(>8^6jGS18kq&NP z6m#tXA|*8#BNzAB=Ww=4#?No2Fd9YJc4eELW9X(;`1g8c!sR+Gl&Y88&Ux}F5)-^m zz{znZjw%U%)04M=`I1o-{CiwjMed*zhF6=z*}3K-AAFJ+AoCgy-1cG{jxwzO$S9JE z`v7<(z<&XMfd7@JmOtZT7oRpI+mA($ql^!piTGqm(pii8aWflxy5U7E}_^S3eu}SjUik=`b3E=_` zFDs1!Hl{0**ht=U@Gs;tSg z<<&}5Z%lAaOaQd@Ak&CH<&h`2G9@r+l4XSmEGnAAMvKS4snh&$p$MH{yRzK zbp2TDo;DodHC$c10m9=7h@Z`i@C*Rtp9m4J`8bas_4KFwi6g!K=nquRqUs%v_j!Rk zX{(3{I=}evS_7g+*VgaJFeW1DQ*PJ}e`812z}Gl|@3HvTEPe?ER3cI7qnLlhR>|Sp zgcwmZ`2c+TauBcdh2h}xPr<1*7-UB(toF~C2B@1qhW0yzewW3!So~)e{{@9aBY$>$ z94<&6r{idF$Il)+HF{icn6}fpRTc{@uA$J;J-$R8yG6y(5&53WvU+T0 zNYHYu(vF>ORnnY%j-YsUe1wM&aX-{Y`jrAa?Nn|!&vxzn2LFl^lFQOfB=oSjn?)8y zlRm}r5#to(?)FQ`sIl+&=N$BZu&A>5pDdVX_UY;$5Sl_E_t598(@lZE`0)SDE|*#S z1^dxRmg}=~$PNTFGr^NAj<9%!#W0IwEGUn`2n!mNV3fu4EKajH!(yBTxk(>A@n01A zJgSK)rukLaUG({ZV(|Ca`7stxv3Q!rQ5HECCs;hk;v|bxEXG*8z~U^67g?+js0Gu! zaD@d7ZTyPo5+Nvw_+`)c2z`ac&$9S=7QfEo>n#2Ti@(X@n=F2h#dlbIz~T>C{7V-9 zfyJM&_-`z}&*Dcclr3ip)hAQe6H?k}Hz6hkTE1*>58FP?;xjBBVDTV}{Vez%Pw)_m zmOc{ni4D_w%;Krpx|^?tPu~JK;+g8??ex~)9bMbH;sZwfoBUcwj*_cvX@@`Dqy6g5 z$7$V*G5rVTZc=vL)z#nM->rYW{XH?ndJFFBzqNlu|3>ud@9)Kx{{Ah3`_RgNoBHqS i-+?~sanIKNb@=Psd(nFx{@sT5?P}`*#HaN2um3-@% 0: if not quiet: - print ('fetching sequences: ' + mol_type) + print ('fetching {0} sequences: {1}'.format(sequence_cnt, mol_type)) target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) - target_file = open(target_file_path, 'w') + target_file = open(target_file_path, 'wb') for accession in accession_list: success = sequenceGet.write_record(target_file, accession, output_format) if not success: diff --git a/python3/enaGroupGet.py b/python3/enaGroupGet.py index c4af82e..32deb26 100644 --- a/python3/enaGroupGet.py +++ b/python3/enaGroupGet.py @@ -117,7 +117,7 @@ def download_sequence_group(accession, output_format, group_dir, subtree): print ('Downloading sequences') update_accs = [] dest_file_path = os.path.join(group_dir, utils.get_filename(accession + '_sequences', output_format)) - dest_file = open(dest_file_path, 'w') + dest_file = open(dest_file_path, 'wb') #sequence update update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs) #sequence release diff --git a/python3/sequenceGet.pyc b/python3/sequenceGet.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56dcc87819cbcf6c6c98b67f2fd6af26f2280532 GIT binary patch literal 2849 zcmd5;&2HpG5U%$8WkN!B3A+S@z^sHc2*`j|NC+Xc$!>NQv6&5?M2H+1Id*5p6UQFh z?PQ}(#9?^=4xBk~`3hSaYC~Sz*q_8PUi^7&D3q0+h zvw$U{zwp-&YEbKV_y(wY2;o(8fe*VwcgE=60H+H_eWH zYk3vS)xnEIm`!NiJ)<+zO`;%;b@xFU=db2R22?3NI1TNf+&PUM+=m#kySK7L>x-64d@b)S)^(l zDa6S1ILwp!WdwVVFdTCXI=p1x@GkZe{~*?`P2EFngR#q@aIs_e*+&;2u!60-&py8B zaC|L0)K3%b;Z61>h^_6-$HRwDyzPx$yMlI@o+feXhqFN2CTztlMa=ooCFWt2c|4sb z6Stt5T%tes0Nb0qX6aWM{I__;MF>($_`ju^McnzA*)NIvT__a{pDW7VCJO-KFbY)G zXaHPXX-j+yT*~$^1KHaUks0R63BUq)>2P>DGDD-ST|-yp+_eGnUDbKzTLHbtCvX|X z`A3fPrdl5!zl*lvBPF$>Za4x|KDsJkzoLQI0C+B=u!8KJ7eN*YT>upM4b;|4C>QpI z^pDyvw{`Mo@i20D!)cvM6 zaB&<3{{0g_iX|^zkO^i7U+l!%H`)!r6nxB{1MOA|2^o4EN=Yq`(?s%#iHxCgXs5zt?DNjv=HoB#TL$l7 zZ)eBb8Vt4`2*>eg9?4nkaeqgqyb2S^$!Vq!qo=}^)BGTg0tREnl_sA~OM}NCm^-w* z8<_QdJce_F)S_hJnpy+@H68vh+$08j`9ZBzEGKnJ)fA*GV-tLOv7 z3RGecN?GyQPvvJN;#6|`Uyz0`2Eus#h`s+m!|YA?$G}O_YxFe3oVSVr-YpidvJhPH zv0S4ck5AHjjRlk1{2{~jHFpr$QctD9cvJQ@1FQ^6qbdP z=|j@Kx>$V1g>^AvH#>a)_(!^Po;}~56xl3Hg_kvFMl#sgP-mIkd*o^( z=OPtaKaH%E_@cIQ*!(z7Pf>M>QY@8!u9p~LpJ1$mquCNY_LNzSkkU))bye(^v1Zxq zXQQc~nc^lPS0InK>v63XZt&P`?;VIsAc;1oYnOQO`EeTNvHpT_ZJ3d86>yen?RvY_ KZnl@(D}Mv3HiXmw literal 0 HcmV?d00001 diff --git a/python3/utils.py b/python3/utils.py index ede4ea1..990ef15 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -283,7 +283,7 @@ def write_record(url, dest_file): response = urlrequest.urlopen(url) linenum = 1 for line in response: - if linenum == 1 and line.startswith('Entry:'): + if linenum == 1 and line.startswith(b'Entry:'): return False chars = dest_file.write(line) linenum += 1 From e12fb2c04b93d8ae9f0f7a792adf60d12b29510c Mon Sep 17 00:00:00 2001 From: nicsilvester Date: Thu, 10 May 2018 08:53:09 +0100 Subject: [PATCH 11/18] Update to README with reported fix for homebrew installed python 3 on a Mac --- README.md | 43 ++++++++++++++++++++++++++++++++++++++++ python3/sequenceGet.pyc | Bin 2849 -> 0 bytes 2 files changed, 43 insertions(+) delete mode 100644 python3/sequenceGet.pyc diff --git a/README.md b/README.md index 28764b8..c78c78e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,49 @@ the Mac can correctly authenticate against the servers. To do this, run the foll the Python version with the correct version of Python 3 that you have installed: open "/Applications/Python 3.6/Install Certificates.command" +We have had a report from a user than when Python 3 was installed using homebrew, the following code needed to be run instead: +``` +# install_certifi.py +# +# sample script to install or update a set of default Root Certificates +# for the ssl module. Uses the certificates provided by the certifi package: +# https://pypi.python.org/pypi/certifi + +import os +import os.path +import ssl +import stat +import subprocess +import sys + +STAT_0o775 = ( stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR + | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP + | stat.S_IROTH | stat.S_IXOTH ) + +openssl_dir, openssl_cafile = os.path.split( + ssl.get_default_verify_paths().openssl_cafile) + +print(" -- pip install --upgrade certifi") +subprocess.check_call([sys.executable, + "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi"]) + +import certifi + +# change working directory to the default SSL directory +os.chdir(openssl_dir) +relpath_to_certifi_cafile = os.path.relpath(certifi.where()) +print(" -- removing any existing file or link") +try: + os.remove(openssl_cafile) +except FileNotFoundError: + pass +print(" -- creating symlink to certifi certificate bundle") +os.symlink(relpath_to_certifi_cafile, openssl_cafile) +print(" -- setting permissions") +os.chmod(openssl_cafile, STAT_0o775) +print(" -- update complete") +``` + ## Installing and running the scripts Download the [latest release](https://github.com/enasequence/enaBrowserTools/releases/latest) and extract it to the preferred location on your computer. You will now have the enaBrowserTools folder, containing both the python 2 and 3 option scripts. If you are using a Unix/Linux or Mac computer, we suggest you add the following aliases to your .bashrc or .bash_profile file. Where INSTALLATION_DIR is the location where you have saved the enaBrowserTools to and PYTHON_CHOICE will depend on whether you are using the Python 2 or Python 3 scripts. diff --git a/python3/sequenceGet.pyc b/python3/sequenceGet.pyc deleted file mode 100644 index 56dcc87819cbcf6c6c98b67f2fd6af26f2280532..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2849 zcmd5;&2HpG5U%$8WkN!B3A+S@z^sHc2*`j|NC+Xc$!>NQv6&5?M2H+1Id*5p6UQFh z?PQ}(#9?^=4xBk~`3hSaYC~Sz*q_8PUi^7&D3q0+h zvw$U{zwp-&YEbKV_y(wY2;o(8fe*VwcgE=60H+H_eWH zYk3vS)xnEIm`!NiJ)<+zO`;%;b@xFU=db2R22?3NI1TNf+&PUM+=m#kySK7L>x-64d@b)S)^(l zDa6S1ILwp!WdwVVFdTCXI=p1x@GkZe{~*?`P2EFngR#q@aIs_e*+&;2u!60-&py8B zaC|L0)K3%b;Z61>h^_6-$HRwDyzPx$yMlI@o+feXhqFN2CTztlMa=ooCFWt2c|4sb z6Stt5T%tes0Nb0qX6aWM{I__;MF>($_`ju^McnzA*)NIvT__a{pDW7VCJO-KFbY)G zXaHPXX-j+yT*~$^1KHaUks0R63BUq)>2P>DGDD-ST|-yp+_eGnUDbKzTLHbtCvX|X z`A3fPrdl5!zl*lvBPF$>Za4x|KDsJkzoLQI0C+B=u!8KJ7eN*YT>upM4b;|4C>QpI z^pDyvw{`Mo@i20D!)cvM6 zaB&<3{{0g_iX|^zkO^i7U+l!%H`)!r6nxB{1MOA|2^o4EN=Yq`(?s%#iHxCgXs5zt?DNjv=HoB#TL$l7 zZ)eBb8Vt4`2*>eg9?4nkaeqgqyb2S^$!Vq!qo=}^)BGTg0tREnl_sA~OM}NCm^-w* z8<_QdJce_F)S_hJnpy+@H68vh+$08j`9ZBzEGKnJ)fA*GV-tLOv7 z3RGecN?GyQPvvJN;#6|`Uyz0`2Eus#h`s+m!|YA?$G}O_YxFe3oVSVr-YpidvJhPH zv0S4ck5AHjjRlk1{2{~jHFpr$QctD9cvJQ@1FQ^6qbdP z=|j@Kx>$V1g>^AvH#>a)_(!^Po;}~56xl3Hg_kvFMl#sgP-mIkd*o^( z=OPtaKaH%E_@cIQ*!(z7Pf>M>QY@8!u9p~LpJ1$mquCNY_LNzSkkU))bye(^v1Zxq zXQQc~nc^lPS0InK>v63XZt&P`?;VIsAc;1oYnOQO`EeTNvHpT_ZJ3d86>yen?RvY_ KZnl@(D}Mv3HiXmw From bb95101235eef5c6b4360e586ec9c95dfeea741d Mon Sep 17 00:00:00 2001 From: nicsilvester Date: Thu, 10 May 2018 10:20:48 +0100 Subject: [PATCH 12/18] Added expanded flag (to expand CON embl format records) --- .gitignore | 4 ++++ README.md | 4 ++++ python/assemblyGet.py | 20 ++++++++++---------- python/enaDataGet.py | 7 +++++-- python/enaGroupGet.py | 29 ++++++++++++++++------------- python/sequenceGet.py | 8 +++++--- python/utils.py | 4 +++- python3/assemblyGet.py | 18 +++++++++--------- python3/enaDataGet.py | 7 +++++-- python3/enaGroupGet.py | 30 ++++++++++++++++++------------ python3/sequenceGet.py | 8 +++++--- python3/utils.py | 4 +++- 12 files changed, 87 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 0a3b76e..ed50a40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ ### Mac ### .DS_Store **/.DS_Store + +### compiled python ### +python/*.pyc +python3/__pycache__ diff --git a/README.md b/README.md index c78c78e..3e2d8e3 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,8 @@ optional arguments: (default is false) -e, --extract-wgs Extract WGS scaffolds for each assembly if available (default is false) + -exp, --expanded Expand CON scaffolds when downloading embl format + (default is false) -m, --meta Download read or analysis XML in addition to data files (default is false) -i, --index Download CRAM index files with submitted CRAM files, @@ -215,6 +217,8 @@ optional arguments: (default is false) -e, --extract-wgs Extract WGS scaffolds for each assembly if available (default is false) + -exp, --expanded Expand CON scaffolds when downloading embl format + (default is false) -m, --meta Download read or analysis XML in addition to data files (default is false) -i, --index Download CRAM index files with submitted CRAM files, diff --git a/python/assemblyGet.py b/python/assemblyGet.py index 901555d..1250a86 100644 --- a/python/assemblyGet.py +++ b/python/assemblyGet.py @@ -75,7 +75,7 @@ def extract_wgs_sequences(accession_list): other_sequences = [a for a in accession_list if not utils.is_wgs_sequence(a)] return wgs_sequences, other_sequences -def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, quiet): +def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, expanded, quiet): failed_accessions = [] count = 0 sequence_cnt = len(accession_list) @@ -86,7 +86,7 @@ def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) target_file = open(target_file_path, 'w') for accession in accession_list: - success = sequenceGet.write_record(target_file, accession, output_format) + success = sequenceGet.write_record(target_file, accession, output_format, expanded) if not success: failed_accessions.append(accession) else: @@ -102,15 +102,15 @@ def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, print 'Failed to fetch following {0}, format {1}'.format(mol_type, output_format) print ','.join(failed_accessions) -def download_sequences(sequence_report, assembly_dir, output_format, quiet): +def download_sequences(sequence_report, assembly_dir, output_format, expanded, quiet): local_sequence_report = os.path.join(assembly_dir, sequence_report) replicon_list, unlocalised_list, unplaced_list, patch_list = parse_sequence_report(local_sequence_report) - wgs_scaffolds, other_unlocalised = _sequences(unlocalised_list) + wgs_scaffolds, other_unlocalised = extract_wgs_sequences(unlocalised_list) wgs_unplaced, other_unplaced = extract_wgs_sequences(unplaced_list) - download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, quiet) - download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, quiet) - download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, quiet) - download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) + download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, expanded, quiet) + download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, expanded, quiet) + download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, expanded, quiet) + download_sequence_set(patch_list, PATCH, assembly_dir, output_format, expanded, quiet) wgs_scaffolds.extend(wgs_unplaced) return wgs_scaffolds @@ -135,7 +135,7 @@ def extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, q target_file.flush() target_file.close() -def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, quiet=False): +def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, expanded, quiet=False): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -153,7 +153,7 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs wgs_scaffolds = [] wgs_scaffold_cnt = 0 if has_sequence_report: - wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, expanded, quiet) wgs_scaffold_cnt = len(wgs_scaffolds) if wgs_scaffold_cnt > 0: if not quiet: diff --git a/python/enaDataGet.py b/python/enaDataGet.py index 6d93c3c..603d245 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -42,6 +42,8 @@ def set_parser(): help='Download WGS set for each assembly if available (default is false)') parser.add_argument('-e', '--extract-wgs', action='store_true', help='Extract WGS scaffolds for each assembly if available (default is false)') + parser.add_argument('-exp', '--expanded', action='store_true', + help='Expand CON scaffolds when downloading embl format (default is false)') parser.add_argument('-m', '--meta', action='store_true', help='Download read or analysis XML in addition to data files (default is false)') parser.add_argument('-i', '--index', action='store_true', @@ -65,6 +67,7 @@ def set_parser(): dest_dir = args.dest fetch_wgs = args.wgs extract_wgs = args.extract_wgs + expanded = args.expanded fetch_meta = args.meta fetch_index = args.index aspera = args.aspera @@ -84,7 +87,7 @@ def set_parser(): elif utils.is_sequence(accession): if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_sequence(dest_dir, accession, output_format) + sequenceGet.download_sequence(dest_dir, accession, output_format, expanded) elif utils.is_analysis(accession): if output_format is not None: readGet.check_read_format(output_format) @@ -96,7 +99,7 @@ def set_parser(): elif utils.is_assembly(accession): if output_format is not None: assemblyGet.check_format(output_format) - assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs) + assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, expanded) else: sys.stderr.write('ERROR: Invalid accession provided\n') sys.exit(1) diff --git a/python/enaGroupGet.py b/python/enaGroupGet.py index 5aee110..0b3c572 100644 --- a/python/enaGroupGet.py +++ b/python/enaGroupGet.py @@ -44,6 +44,8 @@ def set_parser(): help='Download WGS set for each assembly if available (default is false)') parser.add_argument('-e', '--extract-wgs', action='store_true', help='Extract WGS scaffolds for each assembly if available (default is false)') + parser.add_argument('-exp', '--expanded', action='store_true', + help='Expand CON scaffolds when downloading embl format (default is false)') parser.add_argument('-m', '--meta', action='store_true', help='Download read or analysis XML in addition to data files (default is false)') parser.add_argument('-i', '--index', action='store_true', @@ -68,18 +70,18 @@ def download_report(group, result, accession, temp_file, subtree): f.flush() f.close() -def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera): +def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera): if group == utils.WGS: print 'Fetching ' + data_accession[:6] sequenceGet.download_wgs(group_dir, data_accession[:6], output_format) else: print 'Fetching ' + data_accession if group == utils.ASSEMBLY: - assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, True) + assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, expanded, True) elif group in [utils.READ, utils.ANALYSIS]: readGet.download_files(data_accession, output_format, group_dir, fetch_index, fetch_meta, aspera) -def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): +def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): temp_file_path = os.path.join(group_dir, accession + '_temp.txt') download_report(group, utils.get_group_result(group), accession, temp_file_path, subtree) header = True @@ -89,10 +91,10 @@ def download_data_group(group, accession, output_format, group_dir, fetch_wgs, e header = False continue data_accession = line.strip() - download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera) + download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera) os.remove(temp_file_path) -def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs): +def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs, expanded): temp_file_path = os.path.join(group_dir, 'temp.txt') download_report(utils.SEQUENCE, result, accession, temp_file_path, subtree) header = True @@ -110,29 +112,29 @@ def download_sequence_result(dest_file, group_dir, result, accession, subtree, u if data_accession not in update_accs: write_record = True if write_record: - sequenceGet.write_record(dest_file, data_accession, output_format) + sequenceGet.write_record(dest_file, data_accession, output_format, expanded) dest_file.flush() os.remove(temp_file_path) return update_accs -def download_sequence_group(accession, output_format, group_dir, subtree): +def download_sequence_group(accession, output_format, group_dir, subtree, expanded): print 'Downloading sequences' update_accs = [] dest_file_path = os.path.join(group_dir, utils.get_filename(accession + '_sequences', output_format)) dest_file = open(dest_file_path, 'w') #sequence update - update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs, expanded) #sequence release - update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree, update_accs, expanded) dest_file.close() -def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): +def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): group_dir = os.path.join(dest_dir, accession) utils.create_dir(group_dir) if group == utils.SEQUENCE: - download_sequence_group(accession, output_format, group_dir, subtree) + download_sequence_group(accession, output_format, group_dir, subtree, expanded) else: - download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) + download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) if __name__ == '__main__': @@ -145,6 +147,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract dest_dir = args.dest fetch_wgs = args.wgs extract_wgs = args.extract_wgs + expanded = args.expanded fetch_meta = args.meta fetch_index = args.index aspera = args.aspera @@ -182,7 +185,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract if utils.is_taxid(accession) and group in ['read', 'analysis']: print 'Sorry, tax ID retrieval not yet supported for read and analysis' sys.exit(1) - download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) + download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) print 'Completed' except Exception: utils.print_error() diff --git a/python/sequenceGet.py b/python/sequenceGet.py index 8a1bd21..780c986 100644 --- a/python/sequenceGet.py +++ b/python/sequenceGet.py @@ -22,14 +22,16 @@ import utils -def write_record(dest_file, accession, output_format): +def write_record(dest_file, accession, output_format, expanded=False): url = utils.get_record_url(accession, output_format) + if expanded: + url = url + '&expanded=true' return utils.write_record(url, dest_file) -def download_sequence(dest_dir, accession, output_format): +def download_sequence(dest_dir, accession, output_format, expanded): if output_format is None: output_format = utils.EMBL_FORMAT - success = utils.download_record(dest_dir, accession, output_format) + success = utils.download_record(dest_dir, accession, output_format, expanded) if not success: print 'Unable to fetch file for {0}, format {1}'.format(accession, output_format) return success diff --git a/python/utils.py b/python/utils.py index ec79b8a..895b54f 100644 --- a/python/utils.py +++ b/python/utils.py @@ -259,10 +259,12 @@ def get_destination_file(dest_dir, accession, output_format): def download_single_record(url, dest_file): urllib.urlretrieve(url, dest_file) -def download_record(dest_dir, accession, output_format): +def download_record(dest_dir, accession, output_format, expanded=False): try: dest_file = get_destination_file(dest_dir, accession, output_format) url = get_record_url(accession, output_format) + if (expanded): + url = url + '&expanded=true' download_single_record(url, dest_file) return True except Exception: diff --git a/python3/assemblyGet.py b/python3/assemblyGet.py index 06ea6cb..f99f27c 100644 --- a/python3/assemblyGet.py +++ b/python3/assemblyGet.py @@ -76,7 +76,7 @@ def extract_wgs_sequences(accession_list): other_sequences = [a for a in accession_list if not utils.is_wgs_sequence(a)] return wgs_sequences, other_sequences -def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, quiet): +def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, expanded, quiet): failed_accessions = [] count = 0 sequence_cnt = len(accession_list) @@ -87,7 +87,7 @@ def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, target_file_path = os.path.join(assembly_dir, utils.get_filename(mol_type, output_format)) target_file = open(target_file_path, 'wb') for accession in accession_list: - success = sequenceGet.write_record(target_file, accession, output_format) + success = sequenceGet.write_record(target_file, accession, output_format, expanded) if not success: failed_accessions.append(accession) else: @@ -103,15 +103,15 @@ def download_sequence_set(accession_list, mol_type, assembly_dir, output_format, print ('Failed to fetch following {0}, format {1}'.format(mol_type, output_format)) print (','.join(failed_accessions)) -def download_sequences(sequence_report, assembly_dir, output_format, quiet): +def download_sequences(sequence_report, assembly_dir, output_format, expanded, quiet): local_sequence_report = os.path.join(assembly_dir, sequence_report) replicon_list, unlocalised_list, unplaced_list, patch_list = parse_sequence_report(local_sequence_report) wgs_scaffolds, other_unlocalised = extract_wgs_sequences(unlocalised_list) wgs_unplaced, other_unplaced = extract_wgs_sequences(unplaced_list) - download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, quiet) - download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, quiet) - download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, quiet) - download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) + download_sequence_set(replicon_list, REPLICON, assembly_dir, output_format, expanded, quiet) + download_sequence_set(other_unlocalised, UNLOCALISED, assembly_dir, output_format, expanded, quiet) + download_sequence_set(other_unplaced, UNPLACED, assembly_dir, output_format, expanded, quiet) + download_sequence_set(patch_list, PATCH, assembly_dir, output_format, expanded, quiet) wgs_scaffolds.extend(wgs_unplaced) return wgs_scaffolds @@ -136,7 +136,7 @@ def extract_wgs_scaffolds(assembly_dir, wgs_scaffolds, wgs_set, output_format, q target_file.flush() target_file.close() -def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, quiet=False): +def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, expanded, quiet=False): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -154,7 +154,7 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs wgs_scaffolds = [] wgs_scaffold_cnt = 0 if has_sequence_report: - wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) + wgs_scaffolds = download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, expanded, quiet) wgs_scaffold_cnt = len(wgs_scaffolds) if wgs_scaffold_cnt > 0: if not quiet: diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index 400e353..d49a4a3 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -42,6 +42,8 @@ def set_parser(): help='Download WGS set for each assembly if available (default is false)') parser.add_argument('-e', '--extract-wgs', action='store_true', help='Extract WGS scaffolds for each assembly if available (default is false)') + parser.add_argument('-exp', '--expanded', action='store_true', + help='Expand CON scaffolds when downloading embl format (default is false)') parser.add_argument('-m', '--meta', action='store_true', help='Download read or analysis XML in addition to data files (default is false)') parser.add_argument('-i', '--index', action='store_true', @@ -65,6 +67,7 @@ def set_parser(): dest_dir = args.dest fetch_wgs = args.wgs extract_wgs = args.extract_wgs + expanded = args.expanded fetch_meta = args.meta fetch_index = args.index aspera = args.aspera @@ -84,7 +87,7 @@ def set_parser(): elif utils.is_sequence(accession): if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_sequence(dest_dir, accession, output_format) + sequenceGet.download_sequence(dest_dir, accession, output_format, expanded) elif utils.is_analysis(accession): if output_format is not None: readGet.check_read_format(output_format) @@ -96,7 +99,7 @@ def set_parser(): elif utils.is_assembly(accession): if output_format is not None: assemblyGet.check_format(output_format) - assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs) + assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, extract_wgs, expanded) else: sys.stderr.write('ERROR: Invalid accession provided\n') sys.exit(1) diff --git a/python3/enaGroupGet.py b/python3/enaGroupGet.py index 32deb26..80a554d 100644 --- a/python3/enaGroupGet.py +++ b/python3/enaGroupGet.py @@ -42,6 +42,10 @@ def set_parser(): help='Destination directory (default is current running directory)') parser.add_argument('-w', '--wgs', action='store_true', help='Download WGS set for each assembly if available (default is false)') + parser.add_argument('-e', '--extract-wgs', action='store_true', + help='Extract WGS scaffolds for each assembly if available (default is false)') + parser.add_argument('-exp', '--expanded', action='store_true', + help='Expand CON scaffolds when downloading embl format (default is false)') parser.add_argument('-m', '--meta', action='store_true', help='Download read or analysis XML in addition to data files (default is false)') parser.add_argument('-i', '--index', action='store_true', @@ -66,18 +70,18 @@ def download_report(group, result, accession, temp_file, subtree): f.flush() f.close() -def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera): +def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera): if group == utils.WGS: print ('Fetching ' + data_accession[:6]) sequenceGet.download_wgs(group_dir, data_accession[:6], output_format) else: print ('Fetching ' + data_accession) if group == utils.ASSEMBLY: - assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, True) + assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, expanded, True) elif group in [utils.READ, utils.ANALYSIS]: readGet.download_files(data_accession, output_format, group_dir, fetch_index, fetch_meta, aspera) -def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): +def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): temp_file_path = os.path.join(group_dir, accession + '_temp.txt') download_report(group, utils.get_group_result(group), accession, temp_file_path, subtree) header = True @@ -87,10 +91,10 @@ def download_data_group(group, accession, output_format, group_dir, fetch_wgs, e header = False continue data_accession = line.strip() - download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera) + download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera) os.remove(temp_file_path) -def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs): +def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs, expanded): temp_file_path = os.path.join(group_dir, 'temp.txt') download_report(utils.SEQUENCE, result, accession, temp_file_path, subtree) header = True @@ -113,24 +117,24 @@ def download_sequence_result(dest_file, group_dir, result, accession, subtree, u os.remove(temp_file_path) return update_accs -def download_sequence_group(accession, output_format, group_dir, subtree): +def download_sequence_group(accession, output_format, group_dir, subtree, expanded): print ('Downloading sequences') update_accs = [] dest_file_path = os.path.join(group_dir, utils.get_filename(accession + '_sequences', output_format)) dest_file = open(dest_file_path, 'wb') #sequence update - update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_UPDATE_RESULT, accession, subtree, update_accs, expanded) #sequence release - update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree) + update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree, update_accs, expanded) dest_file.close() -def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree): +def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): group_dir = os.path.join(dest_dir, accession) utils.create_dir(group_dir) if group == utils.SEQUENCE: - download_sequence_group(accession, output_format, group_dir, subtree) + download_sequence_group(accession, output_format, group_dir, subtree, expanded) else: - download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) + download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) if __name__ == '__main__': parser = set_parser() @@ -141,6 +145,8 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract output_format = args.format dest_dir = args.dest fetch_wgs = args.wgs + extract_wgs = args.extract_wgs + expanded = args.expanded fetch_meta = args.meta fetch_index = args.index aspera = args.aspera @@ -178,7 +184,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract if utils.is_taxid(accession) and group in ['read', 'analysis']: print('Sorry, tax ID retrieval not yet supported for read and analysis') sys.exit(1) - download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree) + download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) print ('Completed') except Exception: utils.print_error() diff --git a/python3/sequenceGet.py b/python3/sequenceGet.py index 2ac1fef..d7923de 100644 --- a/python3/sequenceGet.py +++ b/python3/sequenceGet.py @@ -23,14 +23,16 @@ import utils -def write_record(dest_file, accession, output_format): +def write_record(dest_file, accession, output_format, expanded=False): url = utils.get_record_url(accession, output_format) + if expanded: + url = url + '&expanded=true' return utils.write_record(url, dest_file) -def download_sequence(dest_dir, accession, output_format): +def download_sequence(dest_dir, accession, output_format, expanded): if output_format is None: output_format = utils.EMBL_FORMAT - success = utils.download_record(dest_dir, accession, output_format) + success = utils.download_record(dest_dir, accession, output_format, expanded) if not success: print ('Unable to fetch file for {0}, format {1}'.format(accession, output_format)) diff --git a/python3/utils.py b/python3/utils.py index 990ef15..0d8180e 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -268,10 +268,12 @@ def get_destination_file(dest_dir, accession, output_format): def download_single_record(url, dest_file): urlrequest.urlretrieve(url, dest_file) -def download_record(dest_dir, accession, output_format): +def download_record(dest_dir, accession, output_format, expanded=False): try: dest_file = get_destination_file(dest_dir, accession, output_format) url = get_record_url(accession, output_format) + if expanded: + url = url + '&expanded=true' download_single_record(url, dest_file) return True except Exception as e: From 0e5a03b6f18d843827bdc09fe423b620016130b1 Mon Sep 17 00:00:00 2001 From: froggleston Date: Fri, 11 May 2018 17:53:59 +0100 Subject: [PATCH 13/18] Update python3 code to use the output handler stuff from the python version --- python/utils.py | 3 +- python3/assemblyGet.py | 6 +- python3/enaDataGet.py | 6 +- python3/sequenceGet.py | 22 +++---- python3/utils.py | 138 ++++++++++++++++++++++++++++++++++++----- 5 files changed, 139 insertions(+), 36 deletions(-) diff --git a/python/utils.py b/python/utils.py index cfb4030..cb00068 100644 --- a/python/utils.py +++ b/python/utils.py @@ -409,7 +409,6 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) -# _do_aspera_transfer(aspera_line, handler) _do_aspera_pexpect(aspera_line, handler) return True except Exception as e: @@ -457,7 +456,7 @@ def _do_aspera_pexpect(cmd, handler): if handler and callable(handler): handler(json.dumps(output)) else: - sys.stdout.write("%s%% transferred, %s, %s\n" % (output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])), + sys.stdout.write("%s%% transferred, %s, %s\n" % (output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])) thread.close() def _do_aspera_transfer(cmd, handler): diff --git a/python3/assemblyGet.py b/python3/assemblyGet.py index 65de13d..5779fdd 100644 --- a/python3/assemblyGet.py +++ b/python3/assemblyGet.py @@ -94,7 +94,7 @@ def download_sequences(sequence_report, assembly_dir, output_format, quiet): download_sequence_set(unplaced_list, UNPLACED, assembly_dir, output_format, quiet) download_sequence_set(patch_list, PATCH, assembly_dir, output_format, quiet) -def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False): +def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False, handler=None): if output_format is None: output_format = utils.EMBL_FORMAT assembly_dir = os.path.join(dest_dir, accession) @@ -107,12 +107,12 @@ def download_assembly(dest_dir, accession, output_format, fetch_wgs, quiet=False has_sequence_report = False # download sequence report if sequence_report is not None: - has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir) + has_sequence_report = utils.get_ftp_file(sequence_report, assembly_dir, handler) # download wgs set if needed if wgs_set is not None and fetch_wgs: if not quiet: print ('fetching wgs set') - sequenceGet.download_wgs(assembly_dir, wgs_set, output_format) + sequenceGet.download_wgs(assembly_dir, wgs_set, output_format, handler) # parse sequence report and download sequences if has_sequence_report: download_sequences(sequence_report.split('/')[-1], assembly_dir, output_format, quiet) diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index a7020ca..96e2a70 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -78,14 +78,14 @@ def set_parser(): if utils.is_wgs_set(accession): if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_wgs(dest_dir, accession, output_format) + sequenceGet.download_wgs(dest_dir, accession, output_format, handler) elif not utils.is_available(accession): sys.stderr.write('ERROR: Record does not exist or is not available for accession provided\n') sys.exit(1) elif utils.is_sequence(accession): if output_format is not None: sequenceGet.check_format(output_format) - sequenceGet.download_sequence(dest_dir, accession, output_format) + sequenceGet.download_sequence(dest_dir, accession, output_format, handler) elif utils.is_analysis(accession): if output_format is not None: readGet.check_read_format(output_format) @@ -97,7 +97,7 @@ def set_parser(): elif utils.is_assembly(accession): if output_format is not None: assemblyGet.check_format(output_format) - assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs) + assemblyGet.download_assembly(dest_dir, accession, output_format, fetch_wgs, handler) else: sys.stderr.write('ERROR: Invalid accession provided\n') sys.exit(1) diff --git a/python3/sequenceGet.py b/python3/sequenceGet.py index b57b90d..8dc9a35 100644 --- a/python3/sequenceGet.py +++ b/python3/sequenceGet.py @@ -27,43 +27,43 @@ def append_record(dest_file, accession, output_format): url = utils.get_record_url(accession, output_format) return utils.append_record(url, dest_file) -def download_sequence(dest_dir, accession, output_format): +def download_sequence(dest_dir, accession, output_format, handler=None): if output_format is None: output_format = utils.EMBL_FORMAT - success = utils.download_record(dest_dir, accession, output_format) + success = utils.download_record(dest_dir, accession, output_format, handler) if not success: print ('Unable to fetch file for {0}, format {1}'.format(accession, output_format)) -def download_wgs(dest_dir, accession, output_format): +def download_wgs(dest_dir, accession, output_format, handler=None): if utils.is_unversioned_wgs_set(accession): - return download_unversioned_wgs(dest_dir, accession, output_format) + return download_unversioned_wgs(dest_dir, accession, output_format, handler) else: - return download_versioned_wgs(dest_dir, accession, output_format) + return download_versioned_wgs(dest_dir, accession, output_format, handler) -def download_versioned_wgs(dest_dir, accession, output_format): +def download_versioned_wgs(dest_dir, accession, output_format, handler=None): prefix = accession[:6] if output_format is None: output_format = utils.EMBL_FORMAT public_set_url = utils.get_wgs_ftp_url(prefix, utils.PUBLIC, output_format) supp_set_url = utils.get_wgs_ftp_url(prefix, utils.SUPPRESSED, output_format) - success = utils.get_ftp_file(public_set_url, dest_dir) + success = utils.get_ftp_file(public_set_url, dest_dir, handler) if not success: - success = utils.get_ftp_file(supp_set_url, dest_dir) + success = utils.get_ftp_file(supp_set_url, dest_dir, handler) if not success: print ('No WGS set file available for {0}, format {1}'.format(accession, output_format)) print ('Please contact ENA (datasubs@ebi.ac.uk) if you feel this set should be available') -def download_unversioned_wgs(dest_dir, accession, output_format): +def download_unversioned_wgs(dest_dir, accession, output_format, handler=None): prefix = accession[:4] if output_format is None: output_format = utils.EMBL_FORMAT public_set_url = utils.get_nonversioned_wgs_ftp_url(prefix, utils.PUBLIC, output_format) if public_set_url is not None: - utils.get_ftp_file(public_set_url, dest_dir) + utils.get_ftp_file(public_set_url, dest_dir, handler) else: supp_set_url = utils.get_nonversioned_wgs_ftp_url(prefix, utils.SUPPRESSED, output_format) if supp_set_url is not None: - utils.get_ftp_file(supp_set_url, dest_dir) + utils.get_ftp_file(supp_set_url, dest_dir, handler) else: print ('No WGS set file available for {0}, format {1}'.format(accession, output_format)) print ('Please contact ENA (datasubs@ebi.ac.uk) if you feel this set should be available') diff --git a/python3/utils.py b/python3/utils.py index e502bf1..856b798 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -25,6 +25,11 @@ import ssl import subprocess import sys +import urllib +import pexpect +import time +import shlex +import json import urllib.request as urlrequest import xml.etree.ElementTree as ElementTree @@ -279,11 +284,12 @@ def append_record(url, dest_file): except Exception: return False -def get_ftp_file(ftp_url, dest_dir): +def get_ftp_file(ftp_url, dest_dir, handler=None): try: filename = ftp_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - urlrequest.urlretrieve(ftp_url, dest_file) + response = urllib.urlopen(ftp_url) + chunk_read(response, file_handle=dest_file, handler=handler) return True except Exception as e: print ("Error with FTP transfer: {0}".format(e)) @@ -306,12 +312,14 @@ def get_md5(filepath): hash_md5.update(chunk) return hash_md5.hexdigest() -def check_md5(filepath, expected_md5): +def check_md5(filepath, expected_md5, handler): generated_md5 = get_md5(filepath) if expected_md5 != generated_md5: - print ('MD5 mismatch for downloaded file ' + filepath + '. Deleting file') - print ('generated md5', generated_md5) - print ('expected md5', expected_md5) + if not handler: + handler=sys.stdout.write + handler('MD5 mismatch for downloaded file ' + filepath + '. Deleting file') + handler('Generated md5: %s' % generated_md5) + handler('Expected md5: %s' % expected_md5) os.remove(filepath) return False return True @@ -326,23 +334,24 @@ def file_exists(file_url, dest_dir, md5): return True return False -def get_ftp_file_with_md5_check(ftp_url, dest_dir, md5): +def get_ftp_file_with_md5_check(ftp_url, dest_dir, md5, handler=None): try: filename = ftp_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - urlrequest.urlretrieve(ftp_url, dest_file) - return check_md5(dest_file, md5) + response = urllib.urlopen(ftp_url) + chunk_read(response, file_handle=dest_file, handler=handler) + return check_md5(dest_file, md5, handler=handler) except Exception as e: sys.stderr.write("Error with FTP transfer: {0}\n".format(e)) return False -def get_aspera_file_with_md5_check(aspera_url, dest_dir, md5): +def get_aspera_file_with_md5_check(aspera_url, dest_dir, md5, handler=None): try: filename = aspera_url.split('/')[-1] dest_file = os.path.join(dest_dir, filename) - success = asperaretrieve(aspera_url, dest_dir, dest_file) + success = asperaretrieve(aspera_url, dest_dir, dest_file, handler=handler) if success: - return check_md5(dest_file, md5) + return check_md5(dest_file, md5, handler=handler) return False except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) @@ -404,22 +413,67 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - #print(aspera_line) - #subprocess.call(aspera_line, shell=True) # this blocks so we're fine to wait and return True - _do_aspera_transfer(aspera_line, handler) + _do_aspera_pexpect(aspera_line, handler) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False +def _do_aspera_pexpect(cmd, handler): + thread = pexpect.spawn(cmd, timeout=None) + cpl = thread.compile_pattern_list([pexpect.EOF, '(.+)']) + started = 0 + + while True: + i = thread.expect_list(cpl, timeout=None) + if i == 0: + if started == 0: + if handler and callable(handler): + handler("Error initiating transfer") + else: + sys.stderr.write("Error initiating transfer") + break + elif i == 1: + started = 1 + output = dict() + pexp_match = thread.match.group(1) + prev_file = '' + tokens_to_match = ["Mb/s"] + units_to_match = ["KB", "MB", "GB"] + rates_to_match = ["Kb/s", "kb/s", "Mb/s", "mb/s", "Gb/s", "gb/s"] + end_of_transfer = False + if any(tm in pexp_match.decode("utf-8") for tm in tokens_to_match): + tokens = pexp_match.decode("utf-8").split(" ") + if 'ETA' in tokens: + pct_completed = [x for x in tokens if len(x) > 1 and x[-1] == '%'] + if pct_completed: + output["pct_completed"] = pct_completed[0][:-1] + + bytes_transferred = [x for x in tokens if len(x) > 2 and x[-2:] in units_to_match] + if bytes_transferred: + output["bytes_transferred"] = bytes_transferred[0] + + transfer_rate = [x for x in tokens if len(x) > 4 and x[-4:] in rates_to_match] + if transfer_rate: + output["transfer_rate"] = transfer_rate[0] + + if handler and callable(handler): + handler(json.dumps(output)) + else: + sys.stdout.write("%s%% transferred, %s, %s\n" % (output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])) + thread.close() + def _do_aspera_transfer(cmd, handler): - p = Popen(cmd, stdout=PIPE, bufsize=1) + p = Popen(shlex.split(cmd), + stdout=PIPE, + stderr=STDOUT, + bufsize=1) with p.stdout: for line in iter(p.stdout.readline, b''): if handler and callable(handler): handler(line) else: - print line, + sys.stdout.write(line), p.wait() pass @@ -599,3 +653,53 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + +def chunk_report(f, bytes_so_far, chunk_size, total_size, handler=None): + global start_time + if bytes_so_far == chunk_size: + start_time = time.time() + return + duration = time.time() - start_time + progress_size = int(bytes_so_far) + speed = int(progress_size / ((1024 * 1024) * duration)) + percent = int(progress_size * 100 /total_size) + line = "\r%s: %0.f%%, %d MB, %d MB/s" % (f, percent, progress_size / (1024 * 1024), speed) + + if handler and callable(handler): + handler(line) + else: + sys.stdout.write(line) + + if bytes_so_far >= total_size: + if handler and callable(handler): + handler('\n') + else: + sys.stdout.write('\n') + +def chunk_read(response, file_handle, chunk_size=104857, tick_rate=10, report_hook=chunk_report, handler=None): + total_size = response.info().getheader('Content-Length').strip() + bytes_so_far = 0 + + with open(file_handle, 'wb') as f: + base = os.path.basename(file_handle) + if total_size is None: # cannot chunk - no content length + f.write(response.content) + else: + tick = 0 + total_size = int(total_size) + while 1: + chunk = response.read(chunk_size) + bytes_so_far += len(chunk) + + if not chunk: + break + + f.write(chunk) + + if report_hook: + if tick == 0 or tick % int(tick_rate) == 0: + tick = 0 + report_hook(base, bytes_so_far, chunk_size, total_size, handler) + tick += 1 + + return bytes_so_far From d0b18ad41a73e8680e6eaebbb3430e6b4c6c76dd Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 16 May 2018 17:21:42 +0100 Subject: [PATCH 14/18] Update handler for enaGroupGet --- python/enaGroupGet.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/enaGroupGet.py b/python/enaGroupGet.py index 0b3c572..7d46c6a 100644 --- a/python/enaGroupGet.py +++ b/python/enaGroupGet.py @@ -58,6 +58,9 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') + parser.add_argument('-r', '--redirect-handler', default=None, + choices=['queue', 'file'], + help="""File download progress handler. Specify an output handler to process the download progress. Default is no handler (output is printed to stdout). 'queue' redirects all output to a queue handler, such as RabbitMQ. 'file' redirects to a file handle (default is [current_file_download.log]).""") parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') return parser @@ -70,18 +73,18 @@ def download_report(group, result, accession, temp_file, subtree): f.flush() f.close() -def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera): +def download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera, handler=None): if group == utils.WGS: print 'Fetching ' + data_accession[:6] - sequenceGet.download_wgs(group_dir, data_accession[:6], output_format) + sequenceGet.download_wgs(group_dir, data_accession[:6], output_format, handler) else: print 'Fetching ' + data_accession if group == utils.ASSEMBLY: - assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, expanded, True) + assemblyGet.download_assembly(group_dir, data_accession, output_format, fetch_wgs, extract_wgs, expanded, True, handler) elif group in [utils.READ, utils.ANALYSIS]: - readGet.download_files(data_accession, output_format, group_dir, fetch_index, fetch_meta, aspera) + readGet.download_files(data_accession, output_format, group_dir, fetch_index, fetch_meta, aspera, handler) -def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): +def download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded, handler=None): temp_file_path = os.path.join(group_dir, accession + '_temp.txt') download_report(group, utils.get_group_result(group), accession, temp_file_path, subtree) header = True @@ -91,7 +94,7 @@ def download_data_group(group, accession, output_format, group_dir, fetch_wgs, e header = False continue data_accession = line.strip() - download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera) + download_data(group, data_accession, output_format, group_dir, fetch_wgs, extract_wgs, expanded, fetch_meta, fetch_index, aspera, handler) os.remove(temp_file_path) def download_sequence_result(dest_file, group_dir, result, accession, subtree, update_accs, expanded): @@ -128,13 +131,13 @@ def download_sequence_group(accession, output_format, group_dir, subtree, expand update_accs = download_sequence_result(dest_file, group_dir, utils.SEQUENCE_RELEASE_RESULT, accession, subtree, update_accs, expanded) dest_file.close() -def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded): +def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded, handler=None): group_dir = os.path.join(dest_dir, accession) utils.create_dir(group_dir) if group == utils.SEQUENCE: download_sequence_group(accession, output_format, group_dir, subtree, expanded) else: - download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) + download_data_group(group, accession, output_format, group_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded, handler) if __name__ == '__main__': @@ -153,6 +156,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract aspera = args.aspera aspera_settings = args.aspera_settings subtree = args.subtree + handler = args.redirect_handler if aspera or aspera_settings is not None: aspera = utils.set_aspera(aspera_settings) @@ -185,7 +189,7 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract if utils.is_taxid(accession) and group in ['read', 'analysis']: print 'Sorry, tax ID retrieval not yet supported for read and analysis' sys.exit(1) - download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) + download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded, handler) print 'Completed' except Exception: utils.print_error() From 5f30ae46510f66c90a3c243fff3e5fd4b3897865 Mon Sep 17 00:00:00 2001 From: froggleston Date: Thu, 17 May 2018 16:49:23 +0100 Subject: [PATCH 15/18] Add fasp url to aspera output string/json --- python/utils.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/python/utils.py b/python/utils.py index 452eb5a..533f109 100644 --- a/python/utils.py +++ b/python/utils.py @@ -261,14 +261,10 @@ def get_destination_file(dest_dir, accession, output_format): return os.path.join(dest_dir, filename) return None -def download_single_record(url, dest_file, handler=None): +def download_single_record(url, dest_file): urllib.urlretrieve(url, dest_file) -<<<<<<< HEAD -def download_record(dest_dir, accession, output_format, handler=None): -======= def download_record(dest_dir, accession, output_format, expanded=False): ->>>>>>> upstream_master try: dest_file = get_destination_file(dest_dir, accession, output_format) url = get_record_url(accession, output_format) @@ -426,13 +422,13 @@ def asperaretrieve(url, dest_dir, dest_file, handler=None): key=ASPERA_PRIVATE_KEY, speed=ASPERA_SPEED, ) - _do_aspera_pexpect(aspera_line, handler) + _do_aspera_pexpect(url, aspera_line, handler) return True except Exception as e: sys.stderr.write("Error with Aspera transfer: {0}\n".format(e)) return False -def _do_aspera_pexpect(cmd, handler): +def _do_aspera_pexpect(url, cmd, handler): thread = pexpect.spawn(cmd, timeout=None) cpl = thread.compile_pattern_list([pexpect.EOF, '(.+)']) started = 0 @@ -470,10 +466,12 @@ def _do_aspera_pexpect(cmd, handler): if transfer_rate: output["transfer_rate"] = transfer_rate[0] + output["url"] = url + if handler and callable(handler): handler(json.dumps(output)) else: - sys.stdout.write("%s%% transferred, %s, %s\n" % (output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])) + sys.stdout.write("%s : %s%% transferred, %s, %s\n" % (output["url"], output["pct_completed"], output["bytes_transferred"], output["transfer_rate"])) thread.close() def _do_aspera_transfer(cmd, handler): From 70346ac60e32ed0a77d7a39bb08306398f46a312 Mon Sep 17 00:00:00 2001 From: Suran Jayathilaka Date: Mon, 9 Jul 2018 11:27:42 +0100 Subject: [PATCH 16/18] removed anon auth. --- python/utils.py | 4 ---- python3/utils.py | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/python/utils.py b/python/utils.py index 895b54f..7b12484 100644 --- a/python/utils.py +++ b/python/utils.py @@ -35,9 +35,6 @@ ASPERA_OPTIONS = '' # set any extra aspera options ASPERA_SPEED = '100M' # set aspera download speed -ANON_USER = 'anon' -ANON_PWD = 'anon' - SUPPRESSED = 'suppressed' PUBLIC = 'public' @@ -449,7 +446,6 @@ def get_nonversioned_wgs_ftp_url(wgs_set, status, output_format): def get_report_from_portal(url): request = urllib2.Request(url) - request.add_header('Authorization', b'Basic ' + base64.b64encode(ANON_USER + b':' + ANON_PWD)) return urllib2.urlopen(request) def download_report_from_portal(url, dest_file): diff --git a/python3/utils.py b/python3/utils.py index 0d8180e..02f4088 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -37,8 +37,6 @@ ASPERA_OPTIONS = '' # set any extra aspera options ASPERA_SPEED = '100M' # set aspera download speed -ANON_AUTH = b'anon:anon' - SUPPRESSED = 'suppressed' PUBLIC = 'public' @@ -461,9 +459,7 @@ def get_nonversioned_wgs_ftp_url(wgs_set, status, output_format): return base_url + '/' + max(files) def get_report_from_portal(url): - userAndPass = base64.b64encode(ANON_AUTH).decode("ascii") - headers = { 'Authorization' : 'Basic %s' % userAndPass } - request = urlrequest.Request(url, headers=headers) + request = urlrequest.Request(url) gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1) return urlrequest.urlopen(request, context=gcontext) From 714f9eb62ef6f34371e3f1a0987eb8c0821c226c Mon Sep 17 00:00:00 2001 From: Suran Jayathilaka Date: Fri, 10 Aug 2018 14:53:57 +0100 Subject: [PATCH 17/18] version corrected --- python/enaDataGet.py | 2 +- python/enaGroupGet.py | 2 +- python3/enaDataGet.py | 2 +- python3/enaGroupGet.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/enaDataGet.py b/python/enaDataGet.py index 603d245..a469e06 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -54,7 +54,7 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') return parser diff --git a/python/enaGroupGet.py b/python/enaGroupGet.py index 0b3c572..1e29433 100644 --- a/python/enaGroupGet.py +++ b/python/enaGroupGet.py @@ -58,7 +58,7 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') return parser def download_report(group, result, accession, temp_file, subtree): diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index d49a4a3..395262d 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -54,7 +54,7 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') return parser diff --git a/python3/enaGroupGet.py b/python3/enaGroupGet.py index 80a554d..9c0ff75 100644 --- a/python3/enaGroupGet.py +++ b/python3/enaGroupGet.py @@ -58,7 +58,7 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') return parser def download_report(group, result, accession, temp_file, subtree): From ee3fd542dc551f3701b2fe4c36fe4e9e0a8ff89b Mon Sep 17 00:00:00 2001 From: Suran Jayathilaka Date: Fri, 10 Aug 2018 15:30:54 +0100 Subject: [PATCH 18/18] added exception stack trace printing. increased version to 1.5.3 --- python/enaDataGet.py | 4 +++- python/enaGroupGet.py | 4 +++- python/utils.py | 2 +- python3/enaDataGet.py | 4 +++- python3/enaGroupGet.py | 4 +++- python3/utils.py | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/python/enaDataGet.py b/python/enaDataGet.py index a469e06..fce020d 100644 --- a/python/enaDataGet.py +++ b/python/enaDataGet.py @@ -25,6 +25,7 @@ import assemblyGet import readGet import utils +import traceback def set_parser(): parser = argparse.ArgumentParser(prog='enaDataGet', @@ -54,7 +55,7 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.3') return parser @@ -105,5 +106,6 @@ def set_parser(): sys.exit(1) print 'Completed' except Exception: + traceback.print_exc() utils.print_error() sys.exit(1) diff --git a/python/enaGroupGet.py b/python/enaGroupGet.py index 1e29433..b02c1fb 100644 --- a/python/enaGroupGet.py +++ b/python/enaGroupGet.py @@ -25,6 +25,7 @@ import assemblyGet import readGet import utils +import traceback def set_parser(): parser = argparse.ArgumentParser(prog='enaGroupGet', @@ -58,7 +59,7 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.3') return parser def download_report(group, result, accession, temp_file, subtree): @@ -188,5 +189,6 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) print 'Completed' except Exception: + traceback.print_exc() utils.print_error() sys.exit(1) diff --git a/python/utils.py b/python/utils.py index 7b12484..f0ab277 100644 --- a/python/utils.py +++ b/python/utils.py @@ -589,7 +589,7 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') - sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance. with the above error details.\n') def get_divisor(record_cnt): if record_cnt <= 10000: diff --git a/python3/enaDataGet.py b/python3/enaDataGet.py index 395262d..e0d06ab 100644 --- a/python3/enaDataGet.py +++ b/python3/enaDataGet.py @@ -25,6 +25,7 @@ import assemblyGet import readGet import utils +import traceback def set_parser(): parser = argparse.ArgumentParser(prog='enaDataGet', @@ -54,7 +55,7 @@ def set_parser(): parser.add_argument('-as', '--aspera-settings', default=None, help="""Use the provided settings file, will otherwise check for environment variable or default settings file location.""") - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.3') return parser @@ -105,5 +106,6 @@ def set_parser(): sys.exit(1) print ('Completed') except Exception: + traceback.print_exc() utils.print_error() sys.exit(1) diff --git a/python3/enaGroupGet.py b/python3/enaGroupGet.py index 9c0ff75..5c311a9 100644 --- a/python3/enaGroupGet.py +++ b/python3/enaGroupGet.py @@ -25,6 +25,7 @@ import assemblyGet import readGet import utils +import traceback def set_parser(): parser = argparse.ArgumentParser(prog='enaGroupGet', @@ -58,7 +59,7 @@ def set_parser(): for environment variable or default settings file location.""") parser.add_argument('-t', '--subtree', action='store_true', help='Include subordinate taxa (taxon subtree) when querying with NCBI tax ID (default is false)') - parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.2') + parser.add_argument('-v', '--version', action='version', version='%(prog)s 1.5.3') return parser def download_report(group, result, accession, temp_file, subtree): @@ -187,5 +188,6 @@ def download_group(accession, group, output_format, dest_dir, fetch_wgs, extract download_group(accession, group, output_format, dest_dir, fetch_wgs, extract_wgs, fetch_meta, fetch_index, aspera, subtree, expanded) print ('Completed') except Exception: + traceback.print_exc() utils.print_error() sys.exit(1) diff --git a/python3/utils.py b/python3/utils.py index 02f4088..8ee731d 100644 --- a/python3/utils.py +++ b/python3/utils.py @@ -604,7 +604,7 @@ def is_empty_dir(target_dir): def print_error(): sys.stderr.write('ERROR: Something unexpected went wrong please try again.\n') - sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance.\n') + sys.stderr.write('If problem persists, please contact datasubs@ebi.ac.uk for assistance, with the above error details.\n') def get_divisor(record_cnt): if record_cnt <= 10000: