From 02bf52c6c6b5fa466a0022ffd42cc4d5faa8b19e Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Tue, 10 Nov 2015 15:32:09 +1300 Subject: [PATCH 01/10] Added method to detect system memory --- easybuild/tools/systemtools.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 569a9536cb..eceb432da2 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -62,6 +62,7 @@ MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' PROC_CPUINFO_FP = '/proc/cpuinfo' +PROC_MEMINFO_FP = '/proc/meminfo' CPU_FAMILIES = [ARM, AMD, INTEL, POWER] VENDORS = { @@ -105,6 +106,25 @@ def get_core_count(): """NO LONGER SUPPORTED: use get_avail_core_count() instead""" _log.nosupport("get_core_count() is replaced by get_avail_core_count()", '2.0') +def get_total_memory(): + + # Return total memory as an integer, a number of megabytes. + + memtotal = 0 + os_type = get_os_type() + + if os_type == LINUX and os.path.exists(PROC_MEMINFO_FP): + with open(PROC_MEMINFO_FP, 'r') as meminfo: + for memline in meminfo: + memline_sub = re.sub(r'^MemTotal:\s*(\d+)\s*kB$', r'\1', memline) + if memline_sub != memline: + memtotal = memline_sub / 1024 + + if memtotal = 0: + raise SystemToolsException('Can not determine total memory on this system') + else: + return memtotal + def get_cpu_vendor(): """ From 4aaf51fd7bdfe13b13ece3a535e8d09ba09e418c Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Tue, 17 Nov 2015 08:08:13 +1300 Subject: [PATCH 02/10] Fixed some bugs --- easybuild/tools/systemtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index eceb432da2..2a59cf117e 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -118,9 +118,9 @@ def get_total_memory(): for memline in meminfo: memline_sub = re.sub(r'^MemTotal:\s*(\d+)\s*kB$', r'\1', memline) if memline_sub != memline: - memtotal = memline_sub / 1024 + memtotal = int(memline_sub) / 1024 - if memtotal = 0: + if memtotal == 0: raise SystemToolsException('Can not determine total memory on this system') else: return memtotal From d7c7b6f8b66ab62c373714e25cc616e873a847a5 Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Fri, 19 Feb 2016 09:54:14 +1300 Subject: [PATCH 03/10] Replace initial value of 0 with None --- easybuild/tools/systemtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 64b330092a..b74f9d14a5 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -110,7 +110,7 @@ def get_total_memory(): # Return total memory as an integer, a number of megabytes. - memtotal = 0 + memtotal = None os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_MEMINFO_FP): @@ -120,7 +120,7 @@ def get_total_memory(): if memline_sub != memline: memtotal = int(memline_sub) / 1024 - if memtotal == 0: + if memtotal is None: raise SystemToolsException('Can not determine total memory on this system') else: return memtotal From d37488c3ae33e831601c5139705fd47118c721f8 Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Wed, 24 Feb 2016 11:54:56 +1300 Subject: [PATCH 04/10] Minor changes to get_total_memory and a test for same --- easybuild/tools/systemtools.py | 15 +++++++++------ test/framework/systemtools.py | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index b74f9d14a5..d75f1a4727 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -106,19 +106,22 @@ def get_core_count(): """NO LONGER SUPPORTED: use get_avail_core_count() instead""" _log.nosupport("get_core_count() is replaced by get_avail_core_count()", '2.0') -def get_total_memory(): - - # Return total memory as an integer, a number of megabytes. +def get_total_memory(): + """ + Try to ascertain this node's total memory + + @return: total memory as an integer, specifically a number of megabytes + """ memtotal = None os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_MEMINFO_FP): with open(PROC_MEMINFO_FP, 'r') as meminfo: for memline in meminfo: - memline_sub = re.sub(r'^MemTotal:\s*(\d+)\s*kB$', r'\1', memline) - if memline_sub != memline: - memtotal = int(memline_sub) / 1024 + mem_mo = re.match(r'MemTotal:\s*(\d+)\s*kB$', memline) + if mem_mo: + memtotal = int(mem_mo.group(1)) / 1024 if memtotal is None: raise SystemToolsException('Can not determine total memory on this system') diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index be0b20d6be..cd1ea65949 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -417,6 +417,11 @@ def test_glibc_version_darwin(self): st.get_os_type = lambda: st.DARWIN self.assertEqual(get_glibc_version(), UNKNOWN) + def test_get_total_memory(self): + """Test the function that gets the total memory.""" + memtotal = get_total_memory() + self.assertTrue(isinstance(memtotal, int)) + def test_system_info(self): """Test getting system info.""" system_info = get_system_info() From ac8a8384a0169eebc5c8ac7d3f7de7e885ee8f14 Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Wed, 24 Feb 2016 12:52:23 +1300 Subject: [PATCH 05/10] Make sure get_total_memory is actually available for testing --- test/framework/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index cd1ea65949..346eaf6a8f 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -40,7 +40,7 @@ from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext -from easybuild.tools.systemtools import get_system_info, get_gcc_version +from easybuild.tools.systemtools import get_system_info, get_total_memory, get_gcc_version MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' From 4b68db6fd51b9b893b7824c8e8dcc14d999325da Mon Sep 17 00:00:00 2001 From: Benjamin Roberts Date: Wed, 24 Feb 2016 13:52:05 +1300 Subject: [PATCH 06/10] Add total memory entry to system_info dictionary --- easybuild/tools/systemtools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index d75f1a4727..2f2ce9ae6f 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -489,6 +489,7 @@ def get_system_info(): python_version = '; '.join(sys.version.split('\n')) return { 'core_count': get_avail_core_count(), + 'total_memory': get_total_memory(), 'cpu_model': get_cpu_model(), 'cpu_speed': get_cpu_speed(), 'cpu_vendor': get_cpu_vendor(), From f1f6909f92b118f254fc36f1b2cd344639902b75 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Feb 2016 18:50:19 +0100 Subject: [PATCH 07/10] don't raise SystemException, return UNKNOWN if memory could not be determined --- easybuild/tools/systemtools.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index 4e0c68288d..acfcfeb497 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -110,23 +110,23 @@ def get_core_count(): def get_total_memory(): """ Try to ascertain this node's total memory - + @return: total memory as an integer, specifically a number of megabytes """ memtotal = None os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_MEMINFO_FP): - with open(PROC_MEMINFO_FP, 'r') as meminfo: - for memline in meminfo: - mem_mo = re.match(r'MemTotal:\s*(\d+)\s*kB$', memline) - if mem_mo: - memtotal = int(mem_mo.group(1)) / 1024 + meminfo = read_file(PROC_MEMINFO_FP) + mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M) + if mem_mo: + memtotal = int(mem_mo.group(1)) / 1024 if memtotal is None: - raise SystemToolsException('Can not determine total memory on this system') - else: - return memtotal + memtotal = UNKNOWN + _log.warning("Failed to determine total memory, returning %s" % memtotal) + + return memtotal def get_cpu_vendor(): From 9efefd3469fe9c7e621c244c4b3024bf452ba511 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Feb 2016 18:50:47 +0100 Subject: [PATCH 08/10] break up test for get_total_memory in tests for Linux, Darwin and native --- test/framework/systemtools.py | 67 +++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 6 deletions(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index eeb51dfdd9..cf9a1f75f8 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -37,15 +37,13 @@ from easybuild.tools.filetools import read_file from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import CPU_FAMILIES, ARM, DARWIN, IBM, INTEL, LINUX, POWER, UNKNOWN, VENDORS +from easybuild.tools.systemtools import MAX_FREQ_FP, PROC_CPUINFO_FP, PROC_MEMINFO_FP from easybuild.tools.systemtools import det_parallelism, get_avail_core_count, get_cpu_family from easybuild.tools.systemtools import get_cpu_model, get_cpu_speed, get_cpu_vendor, get_glibc_version from easybuild.tools.systemtools import get_os_type, get_os_name, get_os_version, get_platform_name, get_shared_lib_ext from easybuild.tools.systemtools import get_system_info, get_total_memory, get_gcc_version -MAX_FREQ_FP = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq' -PROC_CPUINFO_FP = '/proc/cpuinfo' - PROC_CPUINFO_TXT = None PROC_CPUINFO_TXT_ARM = """processor : 0 model name : ARMv7 Processor rev 5 (v7l) @@ -134,6 +132,50 @@ address sizes : 46 bits physical, 48 bits virtual power management: """ +PROC_MEMINFO_TXT = """MemTotal: 66059108 kB +MemFree: 2639988 kB +Buffers: 236368 kB +Cached: 59396644 kB +SwapCached: 84 kB +Active: 3288736 kB +Inactive: 56906588 kB +Active(anon): 246284 kB +Inactive(anon): 348796 kB +Active(file): 3042452 kB +Inactive(file): 56557792 kB +Unevictable: 1048576 kB +Mlocked: 2048 kB +SwapTotal: 20971516 kB +SwapFree: 20969556 kB +Dirty: 76 kB +Writeback: 0 kB +AnonPages: 1610864 kB +Mapped: 118176 kB +Shmem: 32744 kB +Slab: 891272 kB +SReclaimable: 646764 kB +SUnreclaim: 244508 kB +KernelStack: 18960 kB +PageTables: 31528 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 54001068 kB +Committed_AS: 2331888 kB +VmallocTotal: 34359738367 kB +VmallocUsed: 492584 kB +VmallocChunk: 34325311012 kB +HardwareCorrupted: 0 kB +AnonHugePages: 1232896 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +DirectMap4k: 5056 kB +DirectMap2M: 2045952 kB +DirectMap1G: 65011712 kB +""" def mocked_read_file(fp): @@ -141,6 +183,7 @@ def mocked_read_file(fp): known_fps = { MAX_FREQ_FP: '2850000', PROC_CPUINFO_FP: PROC_CPUINFO_TXT, + PROC_MEMINFO_FP: PROC_MEMINFO_TXT, } if fp in known_fps: return known_fps[fp] @@ -387,7 +430,7 @@ def test_os_version(self): def test_gcc_version_native(self): """Test getting gcc version.""" gcc_version = get_gcc_version() - self.assertTrue(isinstance(gcc_version, basestring) or gcc_version == UNKNOWN) + self.assertTrue(isinstance(gcc_version, basestring) or gcc_version == None) def test_gcc_version_linux(self): """Test getting gcc version (mocked for Linux).""" @@ -417,10 +460,22 @@ def test_glibc_version_darwin(self): st.get_os_type = lambda: st.DARWIN self.assertEqual(get_glibc_version(), UNKNOWN) - def test_get_total_memory(self): + def test_get_total_memory_linux(self): + """Test the function that gets the total memory.""" + st.get_os_type = lambda: st.LINUX + st.read_file = mocked_read_file + st.os.path.exists = lambda fp: mocked_os_path_exists(PROC_MEMINFO_FP, fp) + self.assertEqual(get_total_memory(), 64510) + + def test_get_total_memory_darwin(self): + """Test the function that gets the total memory.""" + st.get_os_type = lambda: st.DARWIN + self.assertEqual(get_total_memory(), UNKNOWN) + + def test_get_total_memory_native(self): """Test the function that gets the total memory.""" memtotal = get_total_memory() - self.assertTrue(isinstance(memtotal, int)) + self.assertTrue(isinstance(memtotal, int) or memtotal == UNKNOWN) def test_system_info(self): """Test getting system info.""" From 5ebe8daa1f6d84a5506ba73cb31e3efd695068a8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Feb 2016 19:17:11 +0100 Subject: [PATCH 09/10] fix implementation of get_total_memory on Darwin --- easybuild/tools/systemtools.py | 10 +++++++++- test/framework/systemtools.py | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index acfcfeb497..608ad1eece 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -117,14 +117,22 @@ def get_total_memory(): os_type = get_os_type() if os_type == LINUX and os.path.exists(PROC_MEMINFO_FP): + _log.debug("Trying to determine total memory size on Linux via %s", PROC_MEMINFO_FP) meminfo = read_file(PROC_MEMINFO_FP) mem_mo = re.match(r'^MemTotal:\s*(\d+)\s*kB', meminfo, re.M) if mem_mo: memtotal = int(mem_mo.group(1)) / 1024 + elif os_type == DARWIN: + cmd = "sysctl -n hw.memsize" + _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) + out, ec = run_cmd(cmd, force_in_dry_run=True) + if ec == 0: + memtotal = int(out.strip()) / (1024**2) + if memtotal is None: memtotal = UNKNOWN - _log.warning("Failed to determine total memory, returning %s" % memtotal) + _log.warning("Failed to determine total memory, returning %s", memtotal) return memtotal diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index cf9a1f75f8..ffa7fcb067 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -203,6 +203,7 @@ def mocked_run_cmd(cmd, **kwargs): "ldd --version": "ldd (GNU libc) 2.12", "sysctl -n hw.cpufrequency_max": "2400000000", "sysctl -n hw.ncpu": '10', + "sysctl -n hw.meminfo": '8589934592', "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz", "sysctl -n machdep.cpu.vendor": 'GenuineIntel', "ulimit -u": '40', @@ -470,12 +471,13 @@ def test_get_total_memory_linux(self): def test_get_total_memory_darwin(self): """Test the function that gets the total memory.""" st.get_os_type = lambda: st.DARWIN - self.assertEqual(get_total_memory(), UNKNOWN) + st.run_cmd = mocked_run_cmd + self.assertEqual(get_total_memory(), 8192) def test_get_total_memory_native(self): """Test the function that gets the total memory.""" memtotal = get_total_memory() - self.assertTrue(isinstance(memtotal, int) or memtotal == UNKNOWN) + self.assertTrue(isinstance(memtotal, int)) def test_system_info(self): """Test getting system info.""" From f4f34df155c0e0729bf9face3eb75cb9f5a7bc9c Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 30 Mar 2016 11:24:22 +0200 Subject: [PATCH 10/10] fix mocking of 'sysctl -n hw.memsize' in systemtools tests --- test/framework/systemtools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index ffa7fcb067..891ef56ed0 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -203,7 +203,7 @@ def mocked_run_cmd(cmd, **kwargs): "ldd --version": "ldd (GNU libc) 2.12", "sysctl -n hw.cpufrequency_max": "2400000000", "sysctl -n hw.ncpu": '10', - "sysctl -n hw.meminfo": '8589934592', + "sysctl -n hw.memsize": '8589934592', "sysctl -n machdep.cpu.brand_string": "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz", "sysctl -n machdep.cpu.vendor": 'GenuineIntel', "ulimit -u": '40',