diff --git a/hanasitter.py b/hanasitter.py index c3f5d64..255bd07 100644 --- a/hanasitter.py +++ b/hanasitter.py @@ -10,14 +10,14 @@ def printHelp(): - print(" ") + print(" ") print("DESCRIPTION: ") print(" The HANA sitter checks regularly (def. 1h) if HANA is online and primary. If so, it starts to track. Tracking includes ") print(" regularly (def. 1m) checks if HANA is responsive. If it is not, it will record. Recording could include writing call stacks ") print(" of all active threads and/or record run time dumps and/or indexserver gstacks and/or kernel profiler traces. By default ") print(" nothing is recorded. If HANA is responsive it will check for too many critical features of HANA. By default this is checking ") print(" if there are more than 30 active threads. If there is, it will record (see above). After it is done recording it will by ") - print(" default exit, but could also restart, if so wanted. ") + print(" default exit, but could also restart, if so wanted. ") print(" After it has concluded that all was good, it will wait (def. 1h) and then start all over to check again if HANA is online ") print(" and primary. See also SAP Note 2399979. ") print(" ") @@ -34,7 +34,7 @@ def printHelp(): print(" -cpu a 4 items list to control the cpu check: cpu type, number checks, interval, max average CPU in %, default: 0,0,0,100 ") print(" Possible cpu types are: 0 = not used, 1 = user cpu, 2 = system cpu, 3 = both user and system ") print(" -pt ping timeout [seconds], time it waits before the DB is considered unresponsive (select * from dummy), if set to 0 the ping test will ") - print(" not be done, default: 60 seconds ") + print(" not be done, default: 60 seconds ") print(' -cf list of features surrounded by two "s; the -cf flag has two modes, 1. One Column Mode and 2. Where Clause Mode ') print(" 1. One Column Mode: any sys.m_* view, a column in that view, the column value (wildcards, *, before and/or after are possible) and ") print(" max number allowed feature occations, i.e. ") @@ -51,12 +51,12 @@ def printHelp(): print(' Note: Sometimes it is needed to have a , in the SQL of the WHERE clause for -cf, e.g. ADD_SECONDS(CURRENT_TIME, -60), then use -cd 2 ') print(" -if number checks and intervals of checks, every odd item of this list specifies how many times each feature check (see -cf) should be executed") print(" and every even item specifies how many seconds it waits between each check, then the in the -cf flag is the maximum ") - print(" allowed average value, e.g. ,,...,,, ") + print(" allowed average value, e.g. ,,...,,, ") print(" default: [] (not used) so if you only require one check per feature, do not use -if ") print(" -tf feature check time out [seconds], time it waits before the DB is considered unresponsive during a feature check ") - print(" (see -cf), if -if is used this time out will be added to the interval and then multiplied with number checks, default: 60 seconds ") + print(" (see -cf), if -if is used this time out will be added to the interval and then multiplied with number checks, default: 60 seconds ") print(" -lf log features [true/false], logging ALL information of ALL critical features (beware: could be costly!), default: false ") - print(" -ci check interval [seconds], time it waits before it checks cpu, pings and check features again, default: 60 seconds ") + print(" -ci check interval [seconds], time it waits before it checks cpu, pings and check features again, default: 60 seconds ") print(" -ar time to sleep after recording [seconds], if negative it exits, default: -1 ") print(" *** RECORDINGS (GStacks and/or Kernel Profiler Traces and/or Call Stacks and/or RTE dumps and/or Output from Custom SQL) *** ") print(" -rm recording mode [1, 2 or 3], 1 = each requested recording types are done one after each other with the order above, ") @@ -71,21 +71,21 @@ def printHelp(): print(" the critical feature is above allowed limit per host, default: false ") print(" Note: -hm is not supported for gstack (-ng), but for the other recording possibilities (-np, -nc, and -nr) ") print(" -ng number indexserver gstacks created if the DB is considered unresponsive (Note: gstack blocks the indexserver! See SAP Note 2000000 ") - print(' "Call stack generation via gstack"), default: 0 (not used) ') + print(' "Call stack generation via gstack"), default: 0 (not used) ') print(" -ig gstacks interval [seconds], for -rm = 1: time it waits after a gstack, ") print(" for -rm = 2: time it waits after a gstack, ") print(" for -rm = 3: time the thread waits after a gstack, default: 60 seconds ") - print(" -np number indexserver kernel profiler traces created if the DB is considered unresponsive: default: 0 (not used) ") + print(" -np number indexserver kernel profiler traces created if the DB is considered unresponsive: default: 0 (not used) ") print(" -dp profiler duration [seconds], how long time it is tracing, default: 60 seconds (more info: SAP Note 1804811) ") print(" -wp profiler wait time [milliseconds], wait time after callstacks of all running threads have been taken, default 0 ") print(" -ip profiler interval [seconds], for -rm = 1: time it waits after a profiler trace, ") print(" for -rm = 2: time it waits after a profiler trace, ") print(" for -rm = 3: time the thread waits after a profiler trace, default: 60 seconds ") - print(" -nc number call stacks created if the DB is considered unresponsive: default: 0 (not used) ") + print(" -nc number call stacks created if the DB is considered unresponsive: default: 0 (not used) ") print(" -ic call stacks interval [seconds], for -rm = 1: time it waits after a call stack, ") print(" for -rm = 2: time it waits after a call stack, ") print(" for -rm = 3: time the thread waits after a call stack, default: 60 seconds ") - print(" -nr number rte dumps created if the DB is considered unresponsive, default: 0 (not used) ") + print(" -nr number rte dumps created if the DB is considered unresponsive, default: 0 (not used) ") print(" Note: output is restricted to these folders /tmp, $HOME, $DIR_INSTANCE/work, and $SAP_RETRIEVAL_PATH ") print(" -ir rte dumps interval [seconds], for -rm = 1: time it waits after an rte dump, ") print(" for -rm = 2: time it waits after an rte dump, ") @@ -112,7 +112,7 @@ def printHelp(): print(" -oc output configuration [true/false], logs all parameters set by the flags and where the flags were set, i.e. what flag file ") print(" (one of the files listed in -ff) or if it was set via a flag specified on the command line, default = false ") print(" -en email notification, ,,... default: (not used) ") - print(" -enc email client, to explicitly specify the email client (e.g mail, mailx, mutt, ...,), only useful if -en if used, default: mailx ") + print(" -enc email client, to explicitly specify the email client (e.g mail, mailx, mutt, ...,), only useful if -en if used, default: mailx ") print(" -ens sender's email, to explicitly specify sender's email address, only useful if -en if used, default: (sender's email configured used) ") print(" -enm mail server, to explicitly specify mail server, only useful if -en is used, default: (mail server configured used) ") print(' NOTE: For this to work you have to install the linux program "sendmail" and add a line similar to DSsmtp.intra.ourcompany.com in the file ') @@ -120,34 +120,34 @@ def printHelp(): print(" -so standard out switch [true/false], switch to write to standard out, default: true ") print(" -ff flag file(s), a comma seperated list of full paths to a files that contain input flags, each flag in a new line, all lines in the file ") print(" that do not start with a flag (a minus) are considered comments, default: '' (not used) ") - print(" -ssl turns on ssl certificate [true/false], makes it possible to use SAP HANA Sitter despite SSL, default: false ") + print(" -ssl turns on ssl certificate [true/false], makes it possible to use SAP HANA Sitter despite SSL, default: false ") print(" -vlh virtual local host, if hanacleaner runs on a virtual host this has to be specified, default: '' (physical host is assumed) ") print(" -hc host checking [true/false], checks if the host is the same as in cdtrace and provided in hdbuserkey, might be necessary to turn to false ") - print(" e.g. if you for some reason must provide full host name in hdbuserkey (it will still give warnings though), default: true ") - print(" -k DB user key, this one has to be maintained in hdbuserstore, i.e. as adm do ") + print(" e.g. if you for some reason must provide full host name in hdbuserkey (it will still give warnings though), default: true ") + print(" -k DB user key, this one has to be maintained in hdbuserstore, i.e. as adm do ") print(" > hdbuserstore SET , default: SYSTEMKEY ") - print(" ") + print(" ") print(" ") print("EXAMPLE (if > 20 THREAD_STATE=Running, or > 30 THREAD_STATE=Semaphore Wait are found 2 RTE dumps and 3 GStacks will be recorded ") print(" in parallel, i.e. RTE1&GStack1, RTE2&GStack2, GStack3): ") print(' > python hanasitter.py -cf "M_SERVICE_THREADS,THREAD_STATE,Running,30,M_SERVICE_THREADS,THREAD_STATE,Semaphore Wait,20" -nr 2 -ng 3 -rm 3 ') print(" ") print("EXAMPLE (if, on average from 3 checks with 5s interval, > 30 THREAD_STATE=Running, or if any column from the table VARINUM has been unloaded, ") - print(" then record two call stacks) ") + print(" then record two call stacks) ") print(' > python hanasitter.py -cf "M_SERVICE_THREADS,THREAD_STATE,Running,30,M_CS_UNLOADS,TABLE_NAME,VARINUM,1" -if 3,5,1,0 -nc 2 ') print(" ") - print("EXAMPLE (Here a where clause is given, if more than 3 active indexserver threads runs longer than about 5 days (duration is in ms)) ") + print("EXAMPLE (Here a where clause is given, if more than 3 active indexserver threads runs longer than about 5 days (duration is in ms)) ") print(''' > python hanasitter.py -cf "M_SERVICE_THREADS,WHERE,IS_ACTIVE='TRUE' and SERVICE_NAME='indexserver' and DURATION>420000000,3" -nc 2 ''') print(" ") print("EXAMPLE (if average system CPU >95% or Ping > 30 seconds, 2 Call Stacks are recorded, or else it will try again after 120 seconds, after ") - print(" recording it will sleep for one hour before it starts to track again): ") + print(" recording it will sleep for one hour before it starts to track again): ") print(" > python hanasitter.py -cpu 2,5,2,95 -pt 30 -ci 120 -nc 2 -ar 3600 ") print(" ") print("EXAMPLE (if there are more then 10 threads from the Application user AUSER123 or from the DB user DUSER123 record 2 RTE dumps): ") print(' > python hanasitter.py -cf "M_SERVICE_THREADS,APPLICATION_USER_NAME,AUSER123,10,M_SERVICE_THREADS,USER_NAME,DUSER123,10" -nr 2 ') print(" ") print("EXAMPLE (if there are more then 5 threads with a thread method that starts with PlanExecutor or with a thread type that ") - print(" includes Attribute or that is executed from any user starting with DUSER12, then 5 GStacks are recorded ") + print(" includes Attribute or that is executed from any user starting with DUSER12, then 5 GStacks are recorded ") print(' > python hanasitter.py -cf "M_SERVICE_THREADS,THREAD_METHOD,PlanExecutor*,5,M_SERVICE_THREADS,THREAD_TYPE,*Attribute*,5,M_SERVICE_THREADS,USER_NAME,DUSER12*,5" -ng 5 ') print(" ") print("EXAMPLE (use -cd 2 to use ; as deliminiter of -cf instead of ,) ") @@ -184,7 +184,7 @@ def printHelp(): print(" ") print("CURRENT KNOWN LIMITATIONS (i.e. TODO LIST): ") print(" 1. Record in parallel for different Scale-Out Nodes (should work for some recording types, e.g. RTE dumps --> TODO) ") - print(" 2. If a CPU only happens on one Host, possible to record on only one Host --> not possible to do this with SAR ") + print(" 2. If a CPU only happens on one Host, possible to record on only one Host --> not possible to do this with SAR ") print(" 4. Let HANASitter first check that there is no other hanasitter process running --> refuse to run --> TODO (but can be done with cron, see slides)") print(" 5. Read config file, -ff, after hanasitter slept, so that it will allow dynamic changes ") print(" 6. Make the PING check specific for HOSTS (and only record for that host) --> not possible! Could be done hint ROUTE_TO(, ...) ") @@ -197,12 +197,12 @@ def printHelp(): print(" ") print(" ") os._exit(1) - + def printDisclaimer(): - print(" ") + print(" ") print("ANY USAGE OF HANASITTER ASSUMES THAT YOU HAVE UNDERSTOOD AND AGREED THAT: ") print(" 1. HANASitter is NOT SAP official software, so normal SAP support of HANASitter cannot be assumed ") - print(" 2. HANASitter is open source ") + print(" 2. HANASitter is open source ") print(' 3. HANASitter is provided "as is" ') print(' 4. HANASitter is to be used on "your own risk" ') print(" 5. HANASitter is a one-man's hobby (developed, maintained and supported only during non-working hours) ") @@ -224,17 +224,17 @@ class RTESetting: def __init__(self, num_rtedumps, rtedumps_interval): self.num_rtedumps = num_rtedumps self.rtedumps_interval = rtedumps_interval - + class CallStackSetting: def __init__(self, num_callstacks, callstacks_interval): self.num_callstacks = num_callstacks self.callstacks_interval = callstacks_interval - + class GStackSetting: def __init__(self, num_gstacks, gstacks_interval): self.num_gstacks = num_gstacks self.gstacks_interval = gstacks_interval - + class KernelProfileSetting: def __init__(self, num_kprofs, kprofs_interval, kprofs_duration, kprofs_wait): self.num_kprofs = num_kprofs @@ -275,8 +275,8 @@ def printEmailNotification(self): # - Single container in SAP HANA 1.0 and # - Default tenant starting SAP HANA 2.0 SPS2 # -#If indexserver port ≥ 3**40 then SQL port is always indexserver port +1, valid for -# - All MDC tenants until HANA 2.0 SPS1 and +#If indexserver port ≥ 3**40 then SQL port is always indexserver port +1, valid for +# - All MDC tenants until HANA 2.0 SPS1 and # - Starting HANA 2 SPS2 with second tenant within a MDC system class Tenant: @@ -296,13 +296,13 @@ def printTenant(self): print("TenantDB: ", self.DBName, " Indexserver Port: ", self.indexserverPort, " Sql Port: ", self.sqlPort) def getIndexserverPortString(self): return str(self.indexserverPort) - + class HDBCONS: def __init__(self, local_host, hosts, local_dbinstance, is_mdc, is_tenant, communicationPort, SID, rte_mode, tenantDBName = None): self.local_host = local_host self.local_dbinstance = local_dbinstance self.hosts = hosts - self.hostsForRecording = hosts # at first assume all, also true unloss host_mode + self.hostsForRecording = hosts # at first assume all, also true unloss host_mode self.is_scale_out = (len(hosts) > 1) self.is_mdc = is_mdc self.is_tenant = is_tenant @@ -344,7 +344,7 @@ def clear(self): #subprocess.check_output("rm -r "+path, shell=True) dummout = run_command("rm -r "+path) - + class CommunicationManager: def __init__(self, dbuserkey, out_dir, log_dir, std_out, hdbsql_string, log_features): self.dbuserkey = dbuserkey @@ -352,8 +352,8 @@ def __init__(self, dbuserkey, out_dir, log_dir, std_out, hdbsql_string, log_feat self.log_dir = log_dir self.std_out = std_out self.hdbsql_string = hdbsql_string - self.log_features = log_features - + self.log_features = log_features + class CriticalFeature: def __init__(self, view, feature, value, limit, killSession = False): self.text = "" @@ -367,11 +367,11 @@ def __init__(self, view, feature, value, limit, killSession = False): # IF THERE IS A > THEN TRY TO SPLIT TO A MAX_REPEAT AND A VALUE if '>' in value: # to find string before > X number times where X is the integer after > self.maxRepeat = value.rsplit('>',1)[1] #rsplit allows other >s in the value - if is_integer(self.maxRepeat): #if not, then this > was not intended for repeat + if is_integer(self.maxRepeat): #if not, then this > was not intended for repeat value = value.rsplit('>',1)[0] #where-clause to find rows where the column 'feature' contains the string 'value' more than 'maxRepeat' times self.whereClause = "length("+feature+") - length(replace("+feature+", '"+value+"', '')) > "+str(int(self.maxRepeat)*len(value)) # IF NOT MANAGED TO SPLIT THEN FIRST CORRECT WILDCARDS AND THEN CREATE THE WHERE CLAUSE - if not is_integer(self.maxRepeat): + if not is_integer(self.maxRepeat): if value[0] == '*' and value[-1] == '*': #wildcards, "*", before and after value = "'%"+value[1:-1]+"%'" elif value[0] == '*': #wildcard, "*", before @@ -383,11 +383,11 @@ def __init__(self, view, feature, value, limit, killSession = False): if value[1] == '%' or value[-1] == '%': self.whereClause = feature + " like " + value #where-clause with wildcard(s) else: - self.whereClause = feature + " = " + value #where-clause without wildcard(s) + self.whereClause = feature + " = " + value #where-clause without wildcard(s) if self.view == 'M_ACTIVE_STATEMENTS': # to avoid finding itself: - self.whereClause += " and STATEMENT_STRING not like '%M_ACTIVE_STATEMENTS%'" + self.whereClause += " and STATEMENT_STRING not like '%M_ACTIVE_STATEMENTS%'" self.value = value - self.limitIsMinimumNumberCFAllowed = (limit[0] == '>') # so default and < then maximum number CF allowed + self.limitIsMinimumNumberCFAllowed = (limit[0] == '>') # so default and < then maximum number CF allowed if limit[0] in ['<', '>']: limit = limit[1:] if not is_integer(limit): @@ -411,11 +411,11 @@ def setIterations(self, iterations, interval): self.interval = interval def setText(self, text): self.text = text - + ######################## DEFINE FUNCTIONS ################################ def run_command(cmd): - if sys.version_info[0] == 2: + if sys.version_info[0] == 2: out = subprocess.check_output(cmd, shell=True).strip("\n") elif sys.version_info[0] == 3: out = subprocess.run(cmd, shell=True, capture_output=True, text=True).stdout.strip("\n") @@ -432,14 +432,14 @@ def is_integer(s): return True except ValueError: return False - + def is_email(s): s = s.split('@') if not len(s) == 2: return False return '.' in s[1] - -def checkAndConvertBooleanFlag(boolean, flagstring): + +def checkAndConvertBooleanFlag(boolean, flagstring): boolean = boolean.lower() if boolean not in ("false", "true"): print("INPUT ERROR: ", flagstring, " must be either 'true' or 'false'. Please see --help for more information.") @@ -452,11 +452,11 @@ def checkIfAcceptedFlag(word): print("INPUT ERROR: ", word, " is not one of the accepted input flags. Please see --help for more information.") os._exit(1) -def is_online(dbinstance, comman): #Checks if all services are GREEN and if there exists an indexserver (if not this is a Stand-By) +def is_online(dbinstance, comman): #Checks if all services are GREEN and if there exists an indexserver (if not this is a Stand-By) process = subprocess.Popen(['sapcontrol', '-nr', dbinstance, '-function', 'GetProcessList'], stdout=subprocess.PIPE) out, err = process.communicate() out = out.decode() - number_services = out.count(" HDB ") + out.count(" Local Secure Store") + number_services = out.count(" HDB ") + out.count(" Local Secure Store") number_running_services = out.count("GREEN") number_indexservers = int(out.count("hdbindexserver")) # if not indexserver this is Stand-By test_ok = (str(err) == "None") @@ -464,16 +464,16 @@ def is_online(dbinstance, comman): #Checks if all services are GREEN and if ther printout = "Online Check , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , - , "+str(test_ok)+" , "+str(result)+" , # index services: "+str(number_indexservers)+", # running services: "+str(number_running_services)+" out of "+str(number_services) log(printout, comman) return result - + def is_secondary(comman): process = subprocess.Popen(['hdbnsutil', '-sr_state'], stdout=subprocess.PIPE) out, err = process.communicate() out = out.decode() test_ok = (str(err) == "None") result = "active primary site" in out # then it is secondary! - printout = "Primary Check , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , - , "+str(test_ok)+" , "+str(not result)+" , " + printout = "Primary Check , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , - , "+str(test_ok)+" , "+str(not result)+" , " log(printout, comman) - return result + return result def is_multitenant_database_container(local_dbinstance): is_mdc = False @@ -485,12 +485,12 @@ def is_multitenant_database_container(local_dbinstance): def ping_db(comman, output): with open(os.devnull, 'w') as devnull: # just to get no stdout in case HANA is offline try: - #output[0] = subprocess.check_output(comman.hdbsql_string+''' -j -A -U '''+comman.dbuserkey+''' "select * from dummy"''', shell=True, stderr=devnull) - output[0] = run_command(comman.hdbsql_string+''' -j -A -U '''+comman.dbuserkey+''' "select * from dummy"''') #this might be a problem ... from https://docs.python.org/3/library/subprocess.html#subprocess.getoutput : + #output[0] = subprocess.check_output(comman.hdbsql_string+''' -j -A -U '''+comman.dbuserkey+''' "select * from dummy"''', shell=True, stderr=devnull) + output[0] = run_command(comman.hdbsql_string+''' -j -A -U '''+comman.dbuserkey+''' "select * from dummy"''') #this might be a problem ... from https://docs.python.org/3/library/subprocess.html#subprocess.getoutput : #The stdout and stderr arguments may not be supplied at the same time as capture_output. If you wish to capture and combine both streams into one, use stdout=PIPE and stderr=STDOUT instead of capture_output. except: pass - + def hana_ping(ping_timeout, comman): pause = ping_timeout/10. lifetime = 0 @@ -510,11 +510,11 @@ def hana_ping(ping_timeout, comman): os._exit(1) if not pinged and not hanging: # then still investigating if offline offline = lifetime > ping_timeout - if not offline: + if not offline: time.sleep(pause) # e.g. if ping timeout is 60 seconds it will retry after 6 seconds if HANA is offline - lifetime += pause + lifetime += pause return [hanging, offline] - + def is_number(s): try: float(s) @@ -524,21 +524,21 @@ def is_number(s): def prio_def(prio_number): prios = {1:"RTE", 2:"Call Stacks", 3:"G-Stacks", 4:"Kernel Profiler", 5:"Custom SQL"} - return prios[prio_number] + return prios[prio_number] def recording_prio_convert(recording_prio): recordings = [] for rec in recording_prio: recordings.append(prio_def(rec)) return " ".join(recordings) - + def file_lines_with_word(file_name, word): lines = [] with open(file_name) as f: for line in f: if word in line: lines.append(line) - return lines + return lines def clean_outputs(minRetainedOutputDays, comman): path = comman.out_dir @@ -546,7 +546,7 @@ def clean_outputs(minRetainedOutputDays, comman): #subprocess.check_output("find "+path+"/* -mtime +"+str(minRetainedOutputDays)+" -delete", shell=True) dummyout = run_command("find "+path+"/* -mtime +"+str(minRetainedOutputDays)+" -delete") nFilesAfter = len([name for name in os.listdir(path)]) - return nFilesBefore - nFilesAfter + return nFilesBefore - nFilesAfter def clean_logs(minRetainedLogDays, comman): path = comman.log_dir @@ -554,10 +554,10 @@ def clean_logs(minRetainedLogDays, comman): #subprocess.check_output("find "+path+"/hanasitterlog* -mtime +"+str(minRetainedLogDays)+" -delete", shell=True) dummyout = run_command("find "+path+"/hanasitterlog* -mtime +"+str(minRetainedLogDays)+" -delete") nFilesAfter = len([name for name in os.listdir(path) if "hanasitterlog" in name]) - return nFilesBefore - nFilesAfter + return nFilesBefore - nFilesAfter def tenant_names_and_ports(daemon_file): - tenantDBNames = [] + tenantDBNames = [] tenantIndexserverPorts = [] ports_first_halfs = [] ports_second_halfs = [] @@ -637,7 +637,7 @@ def cpu_too_high(cpu_check_params, comman): log(printout, comman, sendEmail = too_high_cpu) return any_cpu_too_high -def stop_session(cf, comman): +def stop_session(cf, comman): #connExists = int(subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where VIEW_COLUMN_NAME = 'CONNECTION_ID' and VIEW_NAME = '"+cf.view+"'\"", shell=True).strip(' ')) connExists = int(run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where VIEW_COLUMN_NAME = 'CONNECTION_ID' and VIEW_NAME = '"+cf.view+"'\"").strip(' ')) if connExists: @@ -664,8 +664,8 @@ def stop_session(cf, comman): log("Session "+connId+" got disconnected by itself before HANASitter tried", comman) else: log("WARNING, the view in the Critical Feature has no CONNECTION_ID column, so the session for this Critical Feature cannot be killed", comman) - - + + def feature_check(cf, nbrCFsPerHost, critical_feature_info, host_mode, comman): # cf = critical_feature, # comman = communication manager #CHECKS @@ -675,17 +675,17 @@ def feature_check(cf, nbrCFsPerHost, critical_feature_info, host_mode, comman): log("INPUT ERROR, the view given as first entry in the -cf flag, "+cf.view+", does not exist. Please see --help for more information.", comman) os._exit(1) if not cf.whereMode: - #columnExists = int(subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = '"+cf.feature+"'\"", shell=True).strip(' ')) - columnExists = int(run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = '"+cf.feature+"'\"").strip(' ')) + #columnExists = int(subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = '"+cf.feature+"'\"", shell=True).strip(' ')) + columnExists = int(run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = '"+cf.feature+"'\"").strip(' ')) if not columnExists: log("INPUT ERROR, the view "+cf.view+" does not have the column "+cf.feature+". Please see --help for more information.", comman) os._exit(1) if host_mode: - #hostColumnExists = int(subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = 'HOST'\"", shell=True).strip(' ')) - hostColumnExists = int(run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = 'HOST'\"").strip(' ')) + #hostColumnExists = int(subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = 'HOST'\"", shell=True).strip(' ')) + hostColumnExists = int(run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select count(*) from sys.m_monitor_columns where view_name = '"+cf.view+"' and view_column_name = 'HOST'\"").strip(' ')) if not hostColumnExists: log("INPUT ERROR, you have specified host mode with -hf, but the view "+cf.view+" does not have a HOST column. Please see --help for more information.", comman) - os._exit(1) + os._exit(1) nbrCFSum = {} for iteration in range(cf.nbrIterations): # EXECUTE @@ -693,11 +693,11 @@ def feature_check(cf, nbrCFsPerHost, critical_feature_info, host_mode, comman): if host_mode: #hostsInView = subprocess.check_output(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select distinct HOST from SYS."+cf.view+"\"", shell=True).strip(' ').split('\n') hostsInView = run_command(comman.hdbsql_string+" -j -A -a -x -Q -U "+comman.dbuserkey+" \"select distinct HOST from SYS."+cf.view+"\"").strip(' ').split('\n') - hostsInView = [h for h in hostsInView if h != ''] + hostsInView = [h for h in hostsInView if h != ''] for host in hostsInView: #nCFsPerHost.append([int(subprocess.check_output(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "select count(*) from SYS.'+cf.view+' where '+cf.whereClause+' and HOST = \''+host+'\'"', shell=True).split('|')[5].replace(" ", "")), host]) nCFsPerHost.append([int(run_command(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "select count(*) from SYS.'+cf.view+' where '+cf.whereClause+' and HOST = \''+host+'\'"').split('|')[5].replace(" ", "")), host]) - else: + else: #nCFsPerHost.append([int(subprocess.check_output(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "select count(*) from SYS.'+cf.view+' where '+cf.whereClause+'"', shell=True).split('|')[5].replace(" ", "")), '']) nCFsPerHost.append([int(run_command(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "select count(*) from SYS.'+cf.view+' where '+cf.whereClause+'"').split('|')[5].replace(" ", "")), '']) # COLLECT INFO @@ -710,13 +710,13 @@ def feature_check(cf, nbrCFsPerHost, critical_feature_info, host_mode, comman): else: nbrCFSum[cfHost[1]] = cfHost[0] # CRITICAL FEATURE CHECK INTERVALL - time.sleep(float(cf.interval)) + time.sleep(float(cf.interval)) # GET AVERAGE for h, nCF in nbrCFSum.items(): nbrCFSum[h] = int ( float(nCF) / float(cf.nbrIterations) ) - nbrCFsPerHost[0] = nbrCFSum #output + nbrCFsPerHost[0] = nbrCFSum #output + - def record_gstack(gstacks_interval, comman): #pid = subprocess.check_output("pgrep hdbindexserver", shell=True).strip("\n").strip(" ") pid = run_command("pgrep hdbindexserver").strip("\n").strip(" ") @@ -724,15 +724,15 @@ def record_gstack(gstacks_interval, comman): filename = (comman.out_dir+"/gstack_"+pid+"_"+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+".txt") os.system('gstack '+pid+' > '+filename) stop_time = datetime.now() - printout = "GStack Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename + printout = "GStack Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename log(printout, comman) time.sleep(gstacks_interval) return printout - + def record_kprof(kprofiler, hdbcons, comman): # SAP Note 1804811 out_dir = comman.out_dir+"/" total_printout = "" - for hdbcon_string, host, tmp_dir in zip(hdbcons.hdbcons_strings, hdbcons.hosts, hdbcons.temp_host_output_dirs): + for hdbcon_string, host, tmp_dir in zip(hdbcons.hdbcons_strings, hdbcons.hosts, hdbcons.temp_host_output_dirs): if host in hdbcons.hostsForRecording: tenantDBString = hdbcons.tenantDBName+"_" if hdbcons.is_tenant else "" filename_cpu = ("kernel_profiler_cpu_"+host+"_"+hdbcons.SID+"_"+hdbcons.communicationPort+"_"+"_"+tenantDBString+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+".dot") @@ -741,10 +741,10 @@ def record_kprof(kprofiler, hdbcons, comman): # SAP Note 1804811 start_time = datetime.now() os.system(hdbcon_string+'profiler clear" > '+out_dir+filename_kprof_log) os.system(hdbcon_string+'profiler start -w '+str(kprofiler.kprofs_wait)+'" > '+out_dir+filename_kprof_log) - time.sleep(kprofiler.kprofs_duration) - os.system(hdbcon_string+'profiler stop" > '+out_dir+filename_kprof_log) + time.sleep(kprofiler.kprofs_duration) + os.system(hdbcon_string+'profiler stop" > '+out_dir+filename_kprof_log) os.system(hdbcon_string+'profiler print -o '+tmp_dir+filename_cpu+','+tmp_dir+filename_wait+'" > '+out_dir+filename_kprof_log) - os.system(hdbcon_string+'profiler clear" > '+out_dir+filename_kprof_log) # added to avoid an entry in M_KERNEL_PROFILER + os.system(hdbcon_string+'profiler clear" > '+out_dir+filename_kprof_log) # added to avoid an entry in M_KERNEL_PROFILER stop_time = datetime.now() if "[ERROR]" in open(out_dir+filename_kprof_log).read(): printout = "Kernel Profiler , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , False , None , "+out_dir+filename_kprof_log @@ -755,9 +755,9 @@ def record_kprof(kprofiler, hdbcons, comman): # SAP Note 1804811 log(printout, comman) total_printout += printout time.sleep(kprofiler.kprofs_interval) - return total_printout - - + return total_printout + + def record_callstack(callstacks_interval, hdbcons, comman): total_printout = "" for hdbcon_string, host in zip(hdbcons.hdbcons_strings, hdbcons.hosts): @@ -767,12 +767,12 @@ def record_callstack(callstacks_interval, hdbcons, comman): start_time = datetime.now() os.system(hdbcon_string+'context list -s" > '+filename) stop_time = datetime.now() - printout = "Call Stack Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename + printout = "Call Stack Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename log(printout, comman) total_printout += printout time.sleep(callstacks_interval) - return total_printout - + return total_printout + def record_rtedump(rtedumps_interval, hdbcons, comman): total_printout = "" for hdbcon_string, host in zip(hdbcons.hdbcons_strings, hdbcons.hosts): @@ -782,7 +782,7 @@ def record_rtedump(rtedumps_interval, hdbcons, comman): if hdbcons.rte_mode == 0: # normal rte dump filename = (comman.out_dir+"/rtedump_normal_"+host+"_"+hdbcons.SID+"_"+hdbcons.communicationPort+"_"+tenantDBString+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+".trc") os.system(hdbcon_string+'runtimedump dump -c" > '+filename) # have to dump to std with -c and then to a file with > since in case of scale-out -f does not work - elif hdbcons.rte_mode == 1: # light rte dump + elif hdbcons.rte_mode == 1: # light rte dump filename = (comman.out_dir+"/rtedump_light_"+host+"_"+hdbcons.SID+"_"+hdbcons.communicationPort+"_"+tenantDBString+datetime.now().strftime("%Y-%m-%d_%H-%M-%S")+".trc") os.system(hdbcon_string+'runtimedump dump -c -s STACK_SHORT,THREADS" > '+filename) os.system(hdbcon_string+'statreg print -h -n M_JOBEXECUTORS_" >> '+filename) @@ -796,7 +796,7 @@ def record_rtedump(rtedumps_interval, hdbcons, comman): log(printout, comman) total_printout += printout time.sleep(rtedumps_interval) - return total_printout + return total_printout def record_customsql(customsql, hdbcons, comman): tenantDBString = hdbcons.tenantDBName+"_" if hdbcons.is_tenant else "" @@ -805,14 +805,14 @@ def record_customsql(customsql, hdbcons, comman): start_time = datetime.now() #customsql_output = subprocess.check_output(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "'+customsql.custom_sql_recording+'"', shell=True) customsql_output = run_command(comman.hdbsql_string+' -j -A -U '+comman.dbuserkey+' "'+customsql.custom_sql_recording+'"') - customsql_output_file.write(customsql_output) + customsql_output_file.write(customsql_output) customsql_output_file.flush() customsql_output_file.close() stop_time = datetime.now() - printout = "Custom SQL Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename + printout = "Custom SQL Record , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , - , "+filename log(printout, comman) time.sleep(customsql.custom_sql_interval) - return printout + return printout def record(recording_mode, rte, callstack, gstack, kprofiler, customsql, recording_prio, hdbcons, comman): if recording_mode == 1: @@ -822,17 +822,17 @@ def record(recording_mode, rte, callstack, gstack, kprofiler, customsql, recordi record_rtedump(rte.rtedumps_interval, hdbcons, comman) if p == 2: for i in range(callstack.num_callstacks): - record_callstack(callstack.callstacks_interval, hdbcons, comman) + record_callstack(callstack.callstacks_interval, hdbcons, comman) if p == 3: - for i in range(gstack.num_gstacks): - record_gstack(gstack.gstacks_interval, comman) - if p == 4: - for i in range(kprofiler.num_kprofs): - record_kprof(kprofiler, hdbcons, comman) - if p == 5: - for i in range(customsql.num_custom_sql_recordings): - record_customsql(customsql, hdbcons, comman) - elif recording_mode == 2: + for i in range(gstack.num_gstacks): + record_gstack(gstack.gstacks_interval, comman) + if p == 4: + for i in range(kprofiler.num_kprofs): + record_kprof(kprofiler, hdbcons, comman) + if p == 5: + for i in range(customsql.num_custom_sql_recordings): + record_customsql(customsql, hdbcons, comman) + elif recording_mode == 2: max_nbr_recordings = max(gstack.num_gstacks, kprofiler.num_kprofs, callstack.num_callstacks, rte.num_rtedumps) for i in range(max_nbr_recordings): for p in recording_prio: @@ -844,25 +844,25 @@ def record(recording_mode, rte, callstack, gstack, kprofiler, customsql, recordi record_callstack(callstack.callstacks_interval, hdbcons, comman) if p == 3: if i < gstack.num_gstacks: - record_gstack(gstack.gstacks_interval, comman) - if p == 4: + record_gstack(gstack.gstacks_interval, comman) + if p == 4: if i < kprofiler.num_kprofs: record_kprof(kprofiler, hdbcons, comman) - if p == 5: + if p == 5: if i < customsql.num_custom_sql_recordings: record_customsql(customsql, hdbcons, comman) else: record_in_parallel(rte, callstack, gstack, kprofiler, customsql, hdbcons, comman) return True -def record_in_parallel(rte, callstack, gstack, kprofiler, customsql, hdbcons, comman): +def record_in_parallel(rte, callstack, gstack, kprofiler, customsql, hdbcons, comman): max_nbr_recordings = max(gstack.num_gstacks, kprofiler.num_kprofs, callstack.num_callstacks, rte.num_rtedumps, customsql.num_custom_sql_recordings) - for i in range(max_nbr_recordings): + for i in range(max_nbr_recordings): nbr_recording_types = sum(x > i for x in [rte.num_rtedumps, callstack.num_callstacks, gstack.num_gstacks, kprofiler.num_kprofs, customsql.num_custom_sql_recordings]) pool = Pool(nbr_recording_types) # need as many threads as number of recording types rec_types = [] if rte.num_rtedumps > i: - rec_types.append((1, rte, hdbcons, comman)) # 1 = RTE + rec_types.append((1, rte, hdbcons, comman)) # 1 = RTE if callstack.num_callstacks > i: rec_types.append((2, callstack, hdbcons, comman)) # 2 = CallStacks if gstack.num_gstacks > i: @@ -875,10 +875,10 @@ def record_in_parallel(rte, callstack, gstack, kprofiler, customsql, hdbcons, co if comman.std_out: for j in range(len(results)): log(results[j], comman) - pool.close() + pool.close() pool.join() - -def parallel_recording_wrapper(rec_types): + +def parallel_recording_wrapper(rec_types): return parallel_recording(*rec_types) def parallel_recording(record_type, recorder, hdbcons, comman): @@ -893,7 +893,7 @@ def parallel_recording(record_type, recorder, hdbcons, comman): else: return record_customsql(recorder, hdbcons, CommunicationManager(comman.dbuserkey, comman.out_dir, comman.log_dir, False, comman.hdbsql_string, comman.log_features)) -def tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack, kprofiler, customsql, recording_prio, critical_features, feature_check_timeout, cpu_check_params, minRetainedLogDays, minRetainedOutputDays, host_mode, local_dbinstance, comman, hdbcons): +def tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack, kprofiler, customsql, recording_prio, critical_features, feature_check_timeout, cpu_check_params, minRetainedLogDays, minRetainedOutputDays, host_mode, local_dbinstance, comman, hdbcons): recorded = False offline = False while not recorded: @@ -910,13 +910,13 @@ def tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack start_time = datetime.now() [hanging, offline] = hana_ping(ping_timeout, comman) stop_time = datetime.now() - if offline: + if offline: comment = "DB is offline, will exit the tracker without recording (if DB is online, check that the key can be used with hdbsql)" elif hanging: comment = "No response from DB within "+str(ping_timeout)+" seconds" else: comment = "DB responded faster than "+str(ping_timeout)+" seconds" - log("Ping Check , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , "+str(not hanging and not offline)+" , "+comment, comman, sendEmail = hanging or offline) + log("Ping Check , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , - , "+str(not hanging and not offline)+" , "+comment, comman, sendEmail = hanging or offline) if hanging: recorded = record(recording_mode, rte, callstack, gstack, kprofiler, customsql, recording_prio, hdbcons, comman) if offline: @@ -940,7 +940,7 @@ def tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack info_message = "Hang situation during feature-check detected" printout = "Feature Check "+str(chid)+" , "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" , "+str(stop_time-start_time)+" , "+str(not hanging)+" , "+str(not hanging)+" , "+info_message log(printout, comman, sendEmail = hanging) - else: + else: for host, nCFs in nbrCFsPerHost[0].items(): wrong_number_critical_features = (cf.limitIsMinimumNumberCFAllowed and nCFs < cf.limit) or (not cf.limitIsMinimumNumberCFAllowed and nCFs > cf.limit) info_message = "# CFs = "+str(nCFs)+" "+host+", "+cf.cfInfo @@ -968,7 +968,7 @@ def tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack nCleaned = clean_outputs(minRetainedOutputDays, comman) log(str(nCleaned)+" hanasitter output files were removed", comman) return [recorded, offline] - + def cdalias(alias, local_dbinstance): # alias e.g. cdtrace, cdhdb, ... #command_run = subprocess.check_output(['/bin/bash', '-i', '-c', "alias "+alias]).split("alias")[1] process = subprocess.Popen(['/bin/bash', '-i', '-c', "alias "+alias], stdout=subprocess.PIPE) @@ -984,19 +984,19 @@ def cdalias(alias, local_dbinstance): # alias e.g. cdtrace, cdhdb, ... piece = out.decode().strip("\n") path = path + '/' + piece + '/' path = path.replace("[0-9][0-9]", local_dbinstance) # if /bin/bash shows strange HDB[0-9][0-9] we force correct instance on it - return path - + return path + def log(message, comman, file_name = "", sendEmail = False): if comman.std_out: print(message) if file_name == "": file_name = "hanasitterlog" logfile = open(comman.log_dir+"/"+file_name+"_"+datetime.now().strftime("%Y-%m-%d"+".txt").replace(" ", "_"), "a") - logfile.write(message+"\n") + logfile.write(message+"\n") logfile.flush() logfile.close() global emailNotification - if sendEmail and emailNotification: #sends email IF this call of log() wants it AND IF -en flag has been specified + if sendEmail and emailNotification: #sends email IF this call of log() wants it AND IF -en flag has been specified message = 'Hi Team, \nAn odd event reported on the server. Here below are the details:\n'+message mailstring = 'echo "'+message+'" | '+emailNotification.emailClient+' -s "Message from HANASitter about '+emailNotification.SID+'" ' if emailNotification.mailServer: @@ -1006,7 +1006,7 @@ def log(message, comman, file_name = "", sendEmail = False): mailstring += ",".join(emailNotification.receiverEmails) #output = subprocess.check_output(mailstring, shell=True) output = run_command(mailstring) - + def main(): ##################### CHECK PYTHON VERSION ########### if sys.version_info[0] != 2 and sys.version_info[0] != 3: @@ -1025,7 +1025,7 @@ def main(): host_mode = "false" num_rtedumps = 0 #how many rtedumps? rtedumps_interval = 60 #seconds - rte_mode = 0 # either 0 or 1 + rte_mode = 0 # either 0 or 1 num_custom_sql_recordings = 0 #how many custom sqls? custom_sql_interval = 60 #seconds custom_sql_recording = '' #custom sql dump @@ -1059,17 +1059,17 @@ def main(): ssl = "false" virtual_local_host = "" #default: assume physical local host host_check = "true" - dbuserkey = 'SYSTEMKEY' # This KEY has to be maintained in hdbuserstore - # so that hdbuserstore LIST gives e.g. + dbuserkey = 'SYSTEMKEY' # This KEY has to be maintained in hdbuserstore + # so that hdbuserstore LIST gives e.g. # KEY SYSTEMKEY # ENV : mo-fc8d991e0:30015 # USER: SYSTEM cpu_check_params = ['0', '0','0','100'] # by default no cpu check - + ##################### CHECK INPUT ARGUMENTS ################# if len(sys.argv) == 1: print("INPUT ERROR: hanasitter needs input arguments. Please see --help for more information.") - os._exit(1) + os._exit(1) if len(sys.argv) != 2 and len(sys.argv) % 2 == 0: print("INPUT ERROR: Wrong number of input arguments. Please see --help for more information.") os._exit(1) @@ -1077,22 +1077,22 @@ def main(): if i % 2 != 0: if sys.argv[i][0] != '-': print("INPUT ERROR: Every second argument has to be a flag, i.e. start with -. Please see --help for more information.") - os._exit(1) - - - ##################### PRIMARY INPUT ARGUMENTS #################### - flag_log = {} + os._exit(1) + + + ##################### PRIMARY INPUT ARGUMENTS #################### + flag_log = {} if '-h' in sys.argv or '--help' in sys.argv: - printHelp() + printHelp() if '-d' in sys.argv or '--disclaimer' in sys.argv: - printDisclaimer() + printDisclaimer() flag_files = getParameterListFromCommandLine(sys.argv, '-ff', flag_log, flag_files) - + ############ CONFIGURATION FILE ################### for flag_file in flag_files: with open(flag_file, 'r') as fin: for line in fin: - firstWord = line.strip(' ').split(' ')[0] + firstWord = line.strip(' ').split(' ')[0] if firstWord[0:1] == '-': checkIfAcceptedFlag(firstWord) flagValue = line.strip(' ').split('"')[1].strip('\n').strip('\r') if line.strip(' ').split(' ')[1][0] == '"' else line.strip(' ').split(' ')[1].strip('\n').strip('\r') @@ -1117,9 +1117,9 @@ def main(): kprofs_interval = getParameterFromFile(firstWord, '-ip', flagValue, flag_file, flag_log, kprofs_interval) kprofs_duration = getParameterFromFile(firstWord, '-dp', flagValue, flag_file, flag_log, kprofs_duration) kprofs_wait = getParameterFromFile(firstWord, '-wp', flagValue, flag_file, flag_log, kprofs_wait) - cf_deliminiter_mode = '1' # default: deliminiter is , + cf_deliminiter_mode = '1' # default: deliminiter is , cf_deliminiter_mode = getParameterFromFile(firstWord, '-cd', flagValue, flag_file, flag_log, cf_deliminiter_mode) - if cf_deliminiter_mode == '2': + if cf_deliminiter_mode == '2': critical_features = getParameterListFromFile(firstWord, '-cf', flagValue, flag_file, flag_log, critical_features, ';') else: critical_features = getParameterListFromFile(firstWord, '-cf', flagValue, flag_file, flag_log, critical_features) @@ -1143,8 +1143,8 @@ def main(): host_check = getParameterFromFile(firstWord, '-hc', flagValue, flag_file, flag_log, host_check) dbuserkey = getParameterFromFile(firstWord, '-k', flagValue, flag_file, flag_log, dbuserkey) cpu_check_params = getParameterListFromFile(firstWord, '-cpu', flagValue, flag_file, flag_log, cpu_check_params) - - ##################### INPUT ARGUMENTS (these would overwrite whats in the configuration file(s)) #################### + + ##################### INPUT ARGUMENTS (these would overwrite whats in the configuration file(s)) #################### for word in sys.argv: if word[0:1] == '-': checkIfAcceptedFlag(word) @@ -1169,9 +1169,9 @@ def main(): kprofs_interval = getParameterFromCommandLine(sys.argv, '-ip', flag_log, kprofs_interval) kprofs_duration = getParameterFromCommandLine(sys.argv, '-dp', flag_log, kprofs_duration) kprofs_wait = getParameterFromCommandLine(sys.argv, '-wp', flag_log, kprofs_wait) - cf_deliminiter_mode = '1' # default: deliminiter is , + cf_deliminiter_mode = '1' # default: deliminiter is , cf_deliminiter_mode = getParameterFromCommandLine(sys.argv, '-cd', flag_log, cf_deliminiter_mode) - if cf_deliminiter_mode == '2': + if cf_deliminiter_mode == '2': critical_features = getParameterListFromCommandLine(sys.argv, '-cf', flag_log, critical_features, ';') else: critical_features = getParameterListFromCommandLine(sys.argv, '-cf', flag_log, critical_features) @@ -1196,12 +1196,12 @@ def main(): virtual_local_host = getParameterFromCommandLine(sys.argv, '-vlh', flag_log, virtual_local_host) host_check = getParameterFromCommandLine(sys.argv, '-hc', flag_log, host_check) dbuserkey = getParameterFromCommandLine(sys.argv, '-k', flag_log, dbuserkey) - cpu_check_params = getParameterListFromCommandLine(sys.argv, '-cpu', flag_log, cpu_check_params) - + cpu_check_params = getParameterListFromCommandLine(sys.argv, '-cpu', flag_log, cpu_check_params) + ############ GET LOCAL HOST, LOCAL SQL PORT, LOCAL INSTANCE and SID ########## #local_host = subprocess.check_output("hostname", shell=True).replace('\n','') if virtual_local_host == "" else virtual_local_host local_host = run_command("hostname").replace('\n','') if virtual_local_host == "" else virtual_local_host - #key_environment = subprocess.check_output('''hdbuserstore LIST '''+dbuserkey, shell=True) + #key_environment = subprocess.check_output('''hdbuserstore LIST '''+dbuserkey, shell=True) key_environment = run_command('''hdbuserstore LIST '''+dbuserkey) if "NOT FOUND" in key_environment: print("ERROR, the key ", dbuserkey, " is not maintained in hdbuserstore.") @@ -1213,7 +1213,7 @@ def main(): key_environment = key_environment.split('\n') key_environment = [ke for ke in key_environment if ke and not ke == 'Operation succeed.'] ENV = key_environment[1].replace(' ENV : ','').replace(';',',').split(',') - key_hosts = [env.split(':')[0] for env in ENV] + key_hosts = [env.split(':')[0] for env in ENV] if not local_host in key_hosts and not 'localhost' in key_hosts: #Turned out this check was not needed. A user that executed HANASitter from a non-possible future master with virtual host name virt2 only wanted #possible future masters in the hdbuserstore: virt1:30413,virt3:30413,virt4:30413, so he executed HANASitter on virt2 with -vlh virt2 --> worked fine @@ -1223,11 +1223,11 @@ def main(): elif not local_host in key_hosts and 'localhost' in key_hosts: local_host_index = 0 else: - local_host_index = key_hosts.index(local_host) + local_host_index = key_hosts.index(local_host) ### host_check, -hc host_check = checkAndConvertBooleanFlag(host_check, "-hc") - key_sqlports = [env.split(':')[1] for env in ENV] - local_sqlport = key_sqlports[local_host_index] + key_sqlports = [env.split(':')[1] for env in ENV] + local_sqlport = key_sqlports[local_host_index] dbinstances = [port[1:3] for port in key_sqlports] if not all(x == dbinstances[0] for x in dbinstances): if host_check: @@ -1246,15 +1246,15 @@ def main(): log_dir = log_dir.replace(" ","_") if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) - + ############ CHECK AND CONVERT INPUT PARAMETERS FOR COMMUNICATION MANAGER and OLINE TEST ################ ### out_config, -oc - out_config = checkAndConvertBooleanFlag(out_config, "-oc") + out_config = checkAndConvertBooleanFlag(out_config, "-oc") if out_config: parameter_string = "\n".join("{}\t{}".format(k, "= "+v[0]+" from "+v[1]) for k, v in flag_log.items()) - log("\nHANASitter executed "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" with\n"+parameter_string+"\nas "+dbuserkey+": "+'\n'.join(key_environment), CommunicationManager(dbuserkey, out_dir, log_dir, True, "", False)) + log("\nHANASitter executed "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" with\n"+parameter_string+"\nas "+dbuserkey+": "+'\n'.join(key_environment), CommunicationManager(dbuserkey, out_dir, log_dir, True, "", False)) else: - log("\nHANASitter executed "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" with \n"+" ".join(sys.argv)+"\nas "+dbuserkey+": "+'\n'.join(key_environment), CommunicationManager(dbuserkey, out_dir, log_dir, True, "", False)) + log("\nHANASitter executed "+datetime.now().strftime("%Y-%m-%d %H:%M:%S")+" with \n"+" ".join(sys.argv)+"\nas "+dbuserkey+": "+'\n'.join(key_environment), CommunicationManager(dbuserkey, out_dir, log_dir, True, "", False)) ### std_out, -so std_out = checkAndConvertBooleanFlag(std_out, "-so") ### ssl, -ssl @@ -1266,15 +1266,15 @@ def main(): log_features = checkAndConvertBooleanFlag(log_features, "-lf") if log_features and len(critical_features) == 0: log("INPUT ERROR: -lf is True even though -cf is empty, i.e. no critical feature specified. This does not make sense. Please see --help for more information.", CommunicationManager(dbuserkey, out_dir, log_dir, std_out, hdbsql_string, False)) - os._exit(1) - ### online_test_interval, -oi + os._exit(1) + ### online_test_interval, -oi if not is_integer(online_test_interval): log("INPUT ERROR: -oi must be an integer. Please see --help for more information.", CommunicationManager(dbuserkey, out_dir, log_dir, std_out, hdbsql_string, False)) os._exit(1) online_test_interval = int(online_test_interval) - + ############# COMMUNICATION MANAGER ############## - comman = CommunicationManager(dbuserkey, out_dir, log_dir, std_out, hdbsql_string, log_features) + comman = CommunicationManager(dbuserkey, out_dir, log_dir, std_out, hdbsql_string, log_features) ### First Online-Check ### wasOnline = True @@ -1283,18 +1283,18 @@ def main(): wasOnline = False time.sleep(float(online_test_interval)) # wait online_test_interval seconds before again checking if HANA is running - ### MDC or not, SystemDB or Tenant ### + ### MDC or not, SystemDB or Tenant ### is_mdc = is_multitenant_database_container(local_dbinstance) - tenantIndexserverPorts = [] - #output = subprocess.check_output('HDB info', shell=True).splitlines(1) + tenantIndexserverPorts = [] + #output = subprocess.check_output('HDB info', shell=True).splitlines(1) output = run_command('HDB info').splitlines(1) tenantIndexserverPorts = [line.split(' ')[-1].strip('\n') for line in output if "hdbindexserver -port" in line] tenantDBNames = [line.split(' ')[0].replace('adm','').replace('usr','').upper() for line in output if "hdbindexserver -port" in line] # only works if high-isolated (below we get the names in case of low isolated) - #output = subprocess.check_output('ls -l '+cdalias('cdhdb', local_dbinstance)+local_host+'/lock', shell=True).splitlines(1) + #output = subprocess.check_output('ls -l '+cdalias('cdhdb', local_dbinstance)+local_host+'/lock', shell=True).splitlines(1) output = run_command('ls -l '+cdalias('cdhdb', local_dbinstance)+local_host+'/lock').splitlines(1) - nameserverPort = [line.split('@')[1].replace('.pid','') for line in output if "hdbnameserver" in line][0].strip('\n') + nameserverPort = [line.split('@')[1].replace('.pid','') for line in output if "hdbnameserver" in line][0].strip('\n') if not tenantDBNames: print("WARNING: Something went wrong, it passed online tests but still no tenant names were found. Is this HANA 1? HANA 1 is not supported as of May 2021.") @@ -1311,7 +1311,7 @@ def main(): if is_mdc: for dbname, port in zip(tenantDBNames, tenantIndexserverPorts): testTenant = Tenant(dbname, port, local_dbinstance, SID) - if testTenant.sqlPort == int(local_sqlport) or testTenant.DBName == DATABASE: # then the sql port provided in hdbuserstore key is a tenant, or we checking the database name + if testTenant.sqlPort == int(local_sqlport) or testTenant.DBName == DATABASE: # then the sql port provided in hdbuserstore key is a tenant, or we checking the database name tenantDBName = testTenant.DBName is_tenant = True communicationPort = testTenant.getIndexserverPortString() # indexserver port for the tenant @@ -1338,11 +1338,11 @@ def main(): ### HOST(S) USED BY THIS DB ### used_hosts = [] - for potential_host in hosts_worker_and_standby: + for potential_host in hosts_worker_and_standby: #if '@'+communicationPort in subprocess.check_output('ls -l '+cdalias('cdhdb', local_dbinstance)+potential_host+'/lock', shell=True): if '@'+communicationPort in run_command('ls -l '+cdalias('cdhdb', local_dbinstance)+potential_host+'/lock'): - used_hosts.append(potential_host) - + used_hosts.append(potential_host) + ############ CHECK AND CONVERT THE REST OF THE INPUT PARAMETERS ################ ### ping_timeout, -pt if not is_integer(ping_timeout): @@ -1364,7 +1364,7 @@ def main(): recording_mode = int(recording_mode) if not recording_mode in [1, 2, 3]: print("INPUT ERROR: The -rm flag must be either 1, 2, or 3. Please see --help for more information.") - os._exit(1) + os._exit(1) ### recording_prio, -rp if not len(recording_prio) == 5: print("INPUT ERROR: The -rp flag must be followed by 5 items, seperated by comma. Please see --help for more information.") @@ -1375,10 +1375,10 @@ def main(): recording_prio = [int(rec) for rec in recording_prio] if not (recording_prio[0] in [1,2,3,4,5] or recording_prio[1] in [1,2,3,4,5] or recording_prio[2] in [1,2,3,4,5] or recording_prio[3] in [1,2,3,4,5] or recording_prio[4] in [1,2,3,4,5]): print("INPUT ERROR: The -rp flag must be followed by integers of the values withing [1-5]. Please see --help for more information.") - os._exit(1) + os._exit(1) if [rec for rec in recording_prio if recording_prio.count(rec) > 1]: print("INPUT ERROR: The -rp flag must not contain dublicates. Please see --help for more information.") - os._exit(1) + os._exit(1) ### host_mode, -hm host_mode = checkAndConvertBooleanFlag(host_mode, "-hm") if host_mode and not (len(hosts_worker_and_standby) > 1): @@ -1387,10 +1387,10 @@ def main(): host_mode = False if host_mode and log_features: log("INPUT ERROR, it is not supported to log features (-lf) if host mode (-hm) is used. Please see --help for more information.", comman) - os._exit(1) + os._exit(1) if host_mode and num_gstacks: log("INPUT ERROR, gstack recording (-ng) is not supported in host mode (-hm). Please see --help for more information.", comman) - os._exit(1) + os._exit(1) ### num_rtedumps, -nr if not is_integer(num_rtedumps): log("INPUT ERROR: -nr must be an integer. Please see --help for more information.", comman) @@ -1425,7 +1425,7 @@ def main(): log("INPUT ERROR: The -cs flag specifies something allthough -ns is not. This makes no sense. Please see --help for more information.", comman) os._exit(1) if not custom_sql_recording[0:6].upper() == 'SELECT': - log('INPUT ERROR: The -cs flag must be a SELECT statement. Please see --help for more information.', comman) + log('INPUT ERROR: The -cs flag must be a SELECT statement. Please see --help for more information.', comman) os._exit(1) ### num_callstacks, -nc if not is_integer(num_callstacks): @@ -1436,7 +1436,7 @@ def main(): if not is_integer(callstacks_interval): log("INPUT ERROR: -ic must be an integer. Please see --help for more information.", comman) os._exit(1) - callstacks_interval = int(callstacks_interval) + callstacks_interval = int(callstacks_interval) ### num_gstacks, -ng if not is_integer(num_gstacks): log("INPUT ERROR: -ng must be an integer. Please see --help for more information.", comman) @@ -1456,7 +1456,7 @@ def main(): if not is_integer(kprofs_interval): log("INPUT ERROR: -ip must be an integer. Please see --help for more information.", comman) os._exit(1) - kprofs_interval = int(kprofs_interval) + kprofs_interval = int(kprofs_interval) ### kprofs_duration, -dp if not is_integer(kprofs_duration): log("INPUT ERROR: -dp must be an integer. Please see --help for more information.", comman) @@ -1471,7 +1471,7 @@ def main(): if not is_integer(feature_check_timeout): log("INPUT ERROR: -tf must be an integer. Please see --help for more information.", comman) os._exit(1) - feature_check_timeout = int(feature_check_timeout) + feature_check_timeout = int(feature_check_timeout) ### after_recorded, -ar if not is_integer(after_recorded): log("INPUT ERROR: -ar must be an integer. Please see --help for more information.", comman) @@ -1497,11 +1497,11 @@ def main(): critical_features = [critical_features[i*4:i*4+4] for i in range(len(critical_features)//4)] # // is "integer division" in Python 3 else: print("ERROR: Wrong Python version") - os._exit(1) + os._exit(1) critical_features = [CriticalFeature(cf[0], cf[1], cf[2], cf[3]) for cf in critical_features] #testing cf[3] is done in the class ### cf_texts, -ct if cf_texts: - if not len(cf_texts) == len(critical_features): + if not len(cf_texts) == len(critical_features): log("INPUT ERROR: -ct must be a list with the length same as number of critical features. Please see --help for more information.", comman) os._exit(1) cf_texts = [ct.replace("_", ' ') for ct in cf_texts] @@ -1605,15 +1605,15 @@ def main(): else: printout = "Host = "+str(local_host)+", SID = "+SID+", DB Instance = "+str(local_dbinstance)+", MDC SystemDB, Nameserver Port = "+str(communicationPort) else: - printout = "Host = "+str(local_host)+", SID = "+SID+", DB Instance = "+str(local_dbinstance) + printout = "Host = "+str(local_host)+", SID = "+SID+", DB Instance = "+str(local_dbinstance) if (len(hosts_worker_and_standby) > 1): printout += "\nScale Out DB System with hosts: "+", ".join([h for h in hosts_worker_and_standby]) if is_mdc: - if is_tenant: + if is_tenant: printout += "\nTenant DB "+tenantDBName+"@"+SID+" uses host(s): "+", ".join([h for h in used_hosts]) else: printout += "\nSystemDB@"+SID+" uses host(s): "+", ".join([h for h in used_hosts]) - log(printout, comman) + log(printout, comman) log("Online, Primary and Not-Secondary Check: Interval = "+str(online_test_interval)+" seconds", comman) if ping_timeout == 0: log("Ping Check: None", comman) @@ -1631,12 +1631,12 @@ def main(): else: printout += " allows only "+str(cf.limit)+" times that "+cf.whereClauseDescription if cf.nbrIterations > 1: - printout += " as an average from "+str(cf.nbrIterations)+" checks with "+str(cf.interval)+" seconds intervals" + printout += " as an average from "+str(cf.nbrIterations)+" checks with "+str(cf.interval)+" seconds intervals" log(printout, comman) if log_features: log("All information for all features that are in one of the above critical feature states is recorded in the "+comman.log_dir+"/criticalFeatures log", comman) log("Recording mode: "+str(recording_mode), comman) - log("Recording Type , Number Recordings , Intervals [seconds] , Durations [seconds] , Wait [milliseconds]", comman) + log("Recording Type , Number Recordings , Intervals [seconds] , Durations [seconds] , Wait [milliseconds]", comman) log("GStack , "+str(num_gstacks)+" , "+str(gstacks_interval)+" , ", comman) log("Kernel Profiler , "+str(num_kprofs)+" , "+str(kprofs_interval)+" , "+str(kprofs_duration)+" , "+str(kprofs_wait), comman) log("Call Stack , "+str(num_callstacks)+" , "+str(callstacks_interval)+" , ", comman) @@ -1671,7 +1671,7 @@ def main(): if num_kprofs: #only if we write kernel profiler dumps will we need temporary output folders hdbcons.create_temp_output_directories(host_check) #create temporary output folders wasOnline = True - while True: + while True: if is_online(local_dbinstance, comman) and not is_secondary(comman): wasOnline = True [recorded, offline] = tracker(ping_timeout, check_interval, recording_mode, rte, callstack, gstack, kprofiler, customsql, recording_prio, critical_features, feature_check_timeout, cpu_check_params, minRetainedLogDays, minRetainedOutputDays, host_mode, local_dbinstance, comman, hdbcons) @@ -1687,7 +1687,7 @@ def main(): log("\nOne of the online checks found out that this HANA instance is not online. HANASitter will now have a "+str(online_test_interval)+" seconds break.\n", comman, sendEmail = wasOnline) wasOnline = False time.sleep(float(online_test_interval)) # wait online_test_interval seconds before again checking if HANA is running - #except: + #except: except Exception as e: print("HANASitter stopped with the exception: ", e) hdbcons.clear() #remove temporary output folders before exit @@ -1696,9 +1696,9 @@ def main(): print("HANASitter was stopped with ctrl-c") hdbcons.clear() #remove temporary output folders before exit sys.exit() - - + + if __name__ == '__main__': main() - +