From f55c514fd071bb987216dfcbc7cb4438f80810d9 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Fri, 14 Jun 2024 17:15:26 +0000 Subject: [PATCH 1/4] add hostlist from python-hostlist --- R/makeSlurmCluster.R | 34 +++- inst/hostlist | 238 +++++++++++++++++++++++ inst/hostlist.py | 441 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 707 insertions(+), 6 deletions(-) create mode 100755 inst/hostlist create mode 100644 inst/hostlist.py diff --git a/R/makeSlurmCluster.R b/R/makeSlurmCluster.R index fce66ac..d785e8b 100644 --- a/R/makeSlurmCluster.R +++ b/R/makeSlurmCluster.R @@ -14,13 +14,28 @@ get_hosts <- function( fn <- tempfile("slurmr-job-", tmpdir = opts_slurmR$get_tmp_path()) jn <- basename(fn) out <- sprintf("%s/%s.out", tmp_path, jn) + hostfile <- sprintf("%s/%s.hostfile", tmp_path, jn) # Writing the script dots <- c(list(ntasks = ntasks, output = out), list(...)) + + hostlist_script<-system.file(package="slurmR","hostlist") + hostlist_args<-"--append-slurm-tasks=$SLURM_TASKS_PER_NODE -d -e -a \\: $SLURM_NODELIST | \ +while IFS=':' read -ra array + do + hostname=(\"${array[0]}\") + count=(\"${array[1]}\") + for i in `seq 1 $count` + do + echo $hostname + done + done" dat <- paste( - "#!/bin/sh", + "#!/bin/bash", paste0("#SBATCH ", parse_flags(dots), collapse = "\n"), paste(opts_slurmR$get_preamble(), collapse = "\n"), + paste(hostlist_script,"\\",collapse = "\n"), + paste(hostlist_args, ">", hostfile), "echo ==start-hostnames==", "srun hostname", "sleep infinity", @@ -32,19 +47,26 @@ get_hosts <- function( # Submitting the job jobid <- sbatch(fn, wait = FALSE, submit = TRUE) + # # Returning + # while (!file.exists(hostfile)) { + # Sys.sleep(1) + # } + # + # hosts <- readLines(hostfile) + # Returning hosts <- function() { tryCatch({ - hostnames <- suppressWarnings(readLines(out)) - hostnames_start <- which(grepl("^==start-hostnames==$", hostnames)) + 1L + hostnames <- suppressWarnings(readLines(hostfile)) + hostnames_start <- 1L ans <- hostnames[hostnames_start:(hostnames_start + ntasks - 1L)] if (any(is.na(ans))) - stop("Still reading...", call. = FALSE) + stop("Still reading...", call. = FALSE) ans }, error = function(e) e) } - - clean <- function() suppressWarnings(file.remove(out)) + + clean <- function() suppressWarnings(file.remove(c(out,hostfile))) structure( list( diff --git a/inst/hostlist b/inst/hostlist new file mode 100755 index 0000000..22b2946 --- /dev/null +++ b/inst/hostlist @@ -0,0 +1,238 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Hostlist utility + +from __future__ import print_function + +__version__ = "1.23.0" + +# Copyright (C) 2008-2018 +# Kent Engström , +# Thomas Bellman , +# Pär Lindfors and +# Torbjörn Lönnemark , +# National Supercomputer Centre +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +import sys +import os +import optparse +import operator +import re + +from hostlist import expand_hostlist, collect_hostlist, numerically_sorted, parse_slurm_tasks_per_node, BadHostlist, __version__ as library_version + +# Python 3 compatibility +try: + from functools import reduce +except ImportError: + pass # earlier Python versions have this in __builtin__ + +# Helper functions +def flatten(list_of_lists): + res = [] + for l in list_of_lists: + res.extend(l) + return res + +# Operators + +def func_union(args): + return reduce(operator.or_, args) + +def func_intersection(args): + return reduce(operator.and_, args) + +def func_difference(args): + return reduce(operator.sub, args) + +def func_xor(args): + return reduce(operator.xor, args) + +op = optparse.OptionParser(usage="usage: %prog [OPTION]... [HOSTLIST]...") +op.add_option("-u", "--union", + action="store_const", dest="func", const=func_union, + default=func_union, + help="compute the union of the hostlist arguments (default)") +op.add_option("-i", "--intersection", + action="store_const", dest="func", const=func_intersection, + help="compute the intersection of the hostlist arguments") +op.add_option("-d", "--difference", + action="store_const", dest="func", const=func_difference, + help="compute the difference between the first hostlist argument and the rest") +op.add_option("-x", "--symmetric-difference", + action="store_const", dest="func", const=func_xor, + help="compute the symmetric difference between the first hostlist argument and the rest") +op.add_option("-c", "--collapse", + action="store_false", dest="expand", + help="output the result as a hostlist expression (default)") +op.add_option("-o", "--offset", + action="store", type="int", + help="skip OFFSET hosts from the beginning (default: skip nothing)") +op.add_option("-l", "--limit", + action="store", type="int", + help="limit result to the LIMIT first hosts (default: no limit)") +op.add_option("-n", "--count", + action="store_true", + help="output the number of hosts instead of a hostlist") +op.add_option("-e", "--expand", + action="store_true", + help="output the result as an expanded list of hostnames") +op.add_option("-w", + action="store_true", + dest="expand_deprecated", + help="DEPRECATED version of -e/--expand") +op.add_option("-q", "--quiet", + action="store_true", + help="output nothing (useful with --non-empty)") +op.add_option("-0", "--non-empty", + action="store_true", + help="return success only if the resulting hostlist is non-empty") +op.add_option("-s", "--separator", + action="store", type="string", default="\n", + help="separator to use between hostnames when outputting an expanded list (default is newline)") +op.add_option("-p", "--prepend", + action="store", type="string", default="", + help="string to prepend to each hostname when outputting an expanded list") +op.add_option("-a", "--append", + action="store", type="string", default="", + help="string to append to each hostname when outputting an expanded list") +op.add_option("-S", "--substitute", + action="store", type="string", + help="regular expression substitution ('from,to') to apply to each hostname") +op.add_option("--append-slurm-tasks", + action="store", type="string", + help="append task count based on the passed SLURM_TASKS_PER_NODE string") +op.add_option("--repeat-slurm-tasks", + action="store", type="string", + help="repeat hostnames based on the passed SLURM_TASKS_PER_NODE string") +op.add_option("--chop", + action="store", type="int", + help="chop into chunks of this size when doing --collapse (default: all in one chunk)") +op.add_option("--version", + action="store_true", + help="show version") +(opts, args) = op.parse_args() + +if opts.version: + print("Version %s (library version %s)" % (__version__, library_version)) + sys.exit() + +func_args = [] +try: + for a in args: + input_string = a + if a == "-": + input_string = sys.stdin.read() + + func_args.append(set()) + for e in input_string.split(): + func_args[-1] |= set(expand_hostlist(e)) + +except BadHostlist as e: + sys.stderr.write("Bad hostlist ``%s'' encountered: %s\n" + % ((a,) + e.args)) + sys.exit(os.EX_DATAERR) + +if not func_args: + op.print_help() + sys.exit(os.EX_USAGE) + +if opts.expand_deprecated: + sys.stderr.write("WARNING: Option -w is deprecated. Use -e or --expand instead!\n") + +# Set up initial hostlist from the arguments and the function +res = opts.func(func_args) + +# Handle --substitute +if opts.substitute: + try: + from_re, to = opts.substitute.split(",", 1) + res = [re.sub(from_re, to, host) for host in res] + res = set(res) # remove duplicates that may have been created + except (ValueError, re.error): + sys.stderr.write("Bad --substitute option: '%s'\n" % opts.substitute) + sys.exit(os.EX_DATAERR) + +# Sort numerically +res = numerically_sorted(res) # res can be list or set before this line + +# Handle --offset +if opts.offset is not None: + res = res[opts.offset:] + +# Handle --limit +if opts.limit is not None: + res = res[:opts.limit] + +# Handle options using SLURM task lists +if opts.append_slurm_tasks and opts.repeat_slurm_tasks: + sys.stderr.write("You cannot use --append-slurm-tasks and --repeat--slurm-tasks at the same time.\n") + sys.exit(os.EX_DATAERR) +elif opts.append_slurm_tasks or opts.repeat_slurm_tasks: + if opts.append_slurm_tasks: + task_list = opts.append_slurm_tasks + else: + task_list = opts.repeat_slurm_tasks + + try: + task_list = parse_slurm_tasks_per_node(task_list) + except BadHostlist as e: + sys.stderr.write("Bad task list encountered: %s\n" % e.args) + sys.exit(os.EX_DATAERR) + if len(task_list) != len(res): + sys.stderr.write("Length of tasks list != number of hostnames\n") + sys.exit(os.EX_DATAERR) + +# Output in the right way +if opts.quiet: + pass +elif opts.count: + print(len(res)) +elif opts.expand or opts.expand_deprecated: + if opts.append_slurm_tasks: + print(opts.separator.join([opts.prepend + host + opts.append + str(tasks) + for host, tasks in zip(res, task_list)])) + elif opts.repeat_slurm_tasks: + repeated_hosts = flatten([[host]*tasks for host, tasks in zip(res, task_list)]) + print(opts.separator.join([opts.prepend + host + opts.append + for host in repeated_hosts])) + else: + print(opts.separator.join([opts.prepend + host + opts.append + for host in res])) +else: + # --collapse + if opts.chop and opts.chop > 0: + chunk_size = opts.chop + else: + chunk_size = len(res) + i=0 + while i < len(res): + try: + if i > 0: + sys.stdout.write(opts.separator) + sys.stdout.write(opts.prepend + collect_hostlist(res[i:i+chunk_size]) + opts.append) + i += chunk_size + except BadHostlist as e: + sys.stderr.write("Bad hostname encountered: %s\n" % e.args) + sys.exit(os.EX_DATAERR) + sys.stdout.write("\n") + +# Exit +if opts.non_empty and len(res) == 0: + sys.exit(os.EX_NOINPUT) diff --git a/inst/hostlist.py b/inst/hostlist.py new file mode 100644 index 0000000..ba52b14 --- /dev/null +++ b/inst/hostlist.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Hostlist library +# +# Copyright (C) 2008-2018 +# Kent Engström , +# Thomas Bellman , +# Pär Lindfors and +# Torbjörn Lönnemark , +# National Supercomputer Centre +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +"""Handle hostlist expressions. + +This module provides operations to expand and collect hostlist +expressions. + +The hostlist expression syntax is the same as in several programs +developed at LLNL (https://computing.llnl.gov/linux/). However in +corner cases the behaviour of this module have not been compared for +compatibility with pdsh/dshbak/SLURM et al. +""" + +__version__ = "1.23.0" + +import re +import itertools + +# Replace range with xrange on Python 2, do nothing on Python 3 (where xrange +# does not exist, and range returns an iterator) +try: + range = xrange +except: + pass + +# Exception used for error reporting to the caller +class BadHostlist(Exception): pass + +# Configuration to guard against ridiculously long expanded lists +MAX_SIZE = 100000 + +# Hostlist expansion + +def expand_hostlist(hostlist, allow_duplicates=False, sort=False): + """Expand a hostlist expression string to a Python list. + + Example: expand_hostlist("n[9-11],d[01-02]") ==> + ['n9', 'n10', 'n11', 'd01', 'd02'] + + Unless allow_duplicates is true, duplicates will be purged + from the results. If sort is true, the output will be sorted. + """ + + results = [] + bracket_level = 0 + part = "" + + for c in hostlist + ",": + if c == "," and bracket_level == 0: + # Comma at top level, split! + if part: results.extend(expand_part(part)) + part = "" + bad_part = False + else: + part += c + + if c == "[": bracket_level += 1 + elif c == "]": bracket_level -= 1 + + if bracket_level > 1: + raise BadHostlist("nested brackets") + elif bracket_level < 0: + raise BadHostlist("unbalanced brackets") + + if bracket_level > 0: + raise BadHostlist("unbalanced brackets") + + if not allow_duplicates: + results = remove_duplicates(results) + if sort: + results = numerically_sorted(results) + return results + +def expand_part(s): + """Expand a part (e.g. "x[1-2]y[1-3][1-3]") (no outer level commas).""" + + # Base case: the empty part expand to the singleton list of "" + if s == "": + return [""] + + # Split into: + # 1) prefix string (may be empty) + # 2) rangelist in brackets (may be missing) + # 3) the rest + + m = re.match(r'([^,\[]*)(\[[^\]]*\])?(.*)', s) + (prefix, rangelist, rest) = m.group(1,2,3) + + # Expand the rest first (here is where we recurse!) + rest_expanded = expand_part(rest) + + # Expand our own part + if not rangelist: + # If there is no rangelist, our own contribution is the prefix only + us_expanded = [prefix] + else: + # Otherwise expand the rangelist (adding the prefix before) + us_expanded = expand_rangelist(prefix, rangelist[1:-1]) + + # Combine our list with the list from the expansion of the rest + # (but guard against too large results first) + if len(us_expanded) * len(rest_expanded) > MAX_SIZE: + raise BadHostlist("results too large") + + return [us_part + rest_part + for us_part in us_expanded + for rest_part in rest_expanded] + +def expand_rangelist(prefix, rangelist): + """ Expand a rangelist (e.g. "1-10,14"), putting a prefix before.""" + + # Split at commas and expand each range separately + results = [] + for range_ in rangelist.split(","): + results.extend(expand_range(prefix, range_)) + return results + +def expand_range(prefix, range_): + """ Expand a range (e.g. 1-10 or 14), putting a prefix before.""" + + # Check for a single number first + m = re.match(r'^[0-9]+$', range_) + if m: + return ["%s%s" % (prefix, range_)] + + # Otherwise split low-high + m = re.match(r'^([0-9]+)-([0-9]+)$', range_) + if not m: + raise BadHostlist("bad range") + + (s_low, s_high) = m.group(1,2) + low = int(s_low) + high = int(s_high) + width = len(s_low) + + if high < low: + raise BadHostlist("start > stop") + elif high - low > MAX_SIZE: + raise BadHostlist("range too large") + + results = [] + for i in range(low, high+1): + results.append("%s%0*d" % (prefix, width, i)) + return results + +def remove_duplicates(l): + """Remove duplicates from a list (but keep the order).""" + seen = set() + results = [] + for e in l: + if e not in seen: + results.append(e) + seen.add(e) + return results + +# Hostlist collection + +def collect_hostlist(hosts, silently_discard_bad = False): + """Collect a hostlist string from a Python list of hosts. + + We start grouping from the rightmost numerical part. + Duplicates are removed. + + A bad hostname raises an exception (unless silently_discard_bad + is true causing the bad hostname to be silently discarded instead). + """ + + # Split hostlist into a list of (host, "") for the iterative part. + # (Also check for bad node names now) + # The idea is to move already collected numerical parts from the + # left side (seen by each loop) to the right side (just copied). + + left_right = [] + for host in hosts: + # We remove leading and trailing whitespace first, and skip empty lines + host = host.strip() + if host == "": continue + + # We cannot accept a host containing any of the three special + # characters in the hostlist syntax (comma and flat brackets) + if re.search(r'[][,]', host): + if silently_discard_bad: + continue + else: + raise BadHostlist("forbidden character") + + left_right.append((host, "")) + + # Call the iterative function until it says it's done + looping = True + while looping: + left_right, looping = collect_hostlist_1(left_right) + return ",".join([left + right for left, right in left_right]) + +def collect_hostlist_1(left_right): + """Collect a hostlist string from a list of hosts (left+right). + + The input is a list of tuples (left, right). The left part + is analyzed, while the right part is just passed along + (it can contain already collected range expressions). + """ + + # Scan the list of hosts (left+right) and build two things: + # *) a set of all hosts seen (used later) + # *) a list where each host entry is preprocessed for correct sorting + + sortlist = [] + remaining = set() + for left, right in left_right: + host = left + right + remaining.add(host) + + # Match the left part into parts + m = re.match(r'^(.*?)([0-9]+)?([^0-9]*)$', left) + (prefix, num_str, suffix) = m.group(1,2,3) + + # Add the right part unprocessed to the suffix. + # This ensures than an already computed range expression + # in the right part is not analyzed again. + suffix = suffix + right + + if num_str is None: + # A left part with no numeric part at all gets special treatment! + # The regexp matches with the whole string as the suffix, + # with nothing in the prefix or numeric parts. + # We do not want that, so we move it to the prefix and put + # None as a special marker where the suffix should be. + assert prefix == "" + sortlist.append(((host, None), None, None, host)) + else: + # A left part with at least an numeric part + # (we care about the rightmost numeric part) + num_int = int(num_str) + num_width = len(num_str) # This width includes leading zeroes + sortlist.append(((prefix, suffix), num_int, num_width, host)) + + # Sort lexicographically, first on prefix, then on suffix, then on + # num_int (numerically), then... + # This determines the order of the final result. + + def sort_key(entry): + """ + Sort sortlist entries without causing TypeError in Python 3. + + Prefix is always present and a string. + + If suffix is None, use the empty string. The empty string will sort + before all other strings (as None did in Python 2). + + If num_int is None, use -1 instead. -1 sorts before all *possible* + numbers (as None did in Python 2). Negative numbers cannot be present + in a hostlist. + + If num_width is None, use -1 instead. As before, -1 sorts before all + possible widths. An actual num_width of 0 should not be possible, so we + could technically use 0, but using -1 doesn't hurt, so we might as well + use that instead. + """ + ((prefix, suffix), num_int, num_width, host) = entry + return ( + (prefix, '' if suffix is None else suffix), + -1 if num_int is None else num_int, + -1 if num_width is None else num_width, host + ) + + + sortlist.sort(key=sort_key) + + # We are ready to collect the result parts as a list of new (left, + # right) tuples. + + results = [] + needs_another_loop = False + + # Now group entries with the same prefix+suffix combination (the + # key is the first element in the sortlist) to loop over them and + # then to loop over the list of hosts sharing the same + # prefix+suffix combination. + + for ((prefix, suffix), group) in itertools.groupby(sortlist, + key=lambda x:x[0]): + + if suffix is None: + # Special case: a host with no numeric part + results.append(("", prefix)) # Move everything to the right part + remaining.remove(prefix) + else: + # The general case. We prepare to collect a list of + # ranges expressed as (low, high, width) for later + # formatting. + range_list = [] + + for ((prefix2, suffix2), num_int, num_width, host) in group: + if host not in remaining: + # Below, we will loop internally to enumate a whole range + # at a time. We then remove the covered hosts from the set. + # Therefore, skip the host here if it is gone from the set. + continue + assert num_int is not None + + # Scan for a range starting at the current host + low = num_int + while True: + host = "%s%0*d%s" % (prefix, num_width, num_int, suffix) + if host in remaining: + remaining.remove(host) + num_int += 1 + else: + break + high = num_int - 1 + assert high >= low + range_list.append((low, high, num_width)) + + # We have a list of ranges to format. We make sure + # we move our handled numerical part to the right to + # stop it from being processed again. + needs_another_loop = True + if len(range_list) == 1 and range_list[0][0] == range_list[0][1]: + # Special case to make sure that n1 is not shown as n[1] etc + results.append((prefix, + "%0*d%s" % + (range_list[0][2], range_list[0][0], suffix))) + else: + # General case where high > low + results.append((prefix, "[" + \ + ",".join([format_range(l, h, w) + for l, h, w in range_list]) + \ + "]" + suffix)) + + # At this point, the set of remaining hosts should be empty and we + # are ready to return the result, together with the flag that says + # if we need to loop again (we do if we have added something to a + # left part). + assert not remaining + return results, needs_another_loop + +def format_range(low, high, width): + """Format a range from low to high inclusively, with a certain width.""" + + if low == high: + return "%0*d" % (width, low) + else: + return "%0*d-%0*d" % (width, low, width, high) + +# Sort a list of hosts numerically + +def numerically_sorted(l): + """Sort a list of hosts numerically. + + E.g. sorted order should be n1, n2, n10; not n1, n10, n2. + """ + + return sorted(l, key=numeric_sort_key) + +numeric_sort_key_regexp = re.compile("([0-9]+)|([^0-9]+)") +def numeric_sort_key(x): + """Compose a sorting key to compare strings "numerically": + + We split numerical (integer) and non-numerical parts into a list, + making sure that the numerical parts are converted to Python ints, + and then sort on the lists. Thus, if we sort x10y and x9z8, we will + compare ["x", 10, "y"] with ["x", 9, "x", "8"] and return x9z8 + before x10y". + + Python 3 complication: We cannot compare int and str, so while we can + compare x10y and x9z8, we cannot compare x10y and 9z8. Kludge: insert + a blank string first if the list would otherwise start with an integer. + This will give the same ordering as before, as integers seem to compare + smaller than strings in Python 2. + """ + + keylist = [int(i_ni[0]) if i_ni[0] else i_ni[1] + for i_ni in numeric_sort_key_regexp.findall(x)] + if keylist and isinstance(keylist[0], int): + keylist.insert(0, "") + return keylist + +# Parse SLURM_TASKS_PER_NODE into a list of task numbers +# +# Description from the SLURM sbatch man page: +# Number of tasks to be initiated on each node. Values +# are comma separated and in the same order as +# SLURM_NODELIST. If two or more consecutive nodes are +# to have the same task count, that count is followed by +# "(x#)" where "#" is the repetition count. For example, +# "SLURM_TASKS_PER_NODE=2(x3),1" indicates that the first +# three nodes will each execute three tasks and the +# fourth node will execute one task. + +def parse_slurm_tasks_per_node(s): + res = [] + for part in s.split(","): + m = re.match(r'^([0-9]+)(\(x([0-9]+)\))?$', part) + if m: + tasks = int(m.group(1)) + repetitions = m.group(3) + if repetitions is None: + repetitions = 1 + else: + repetitions = int(repetitions) + if repetitions > MAX_SIZE: + raise BadHostlist("task list repetitions too large") + for i in range(repetitions): + res.append(tasks) + else: + raise BadHostlist("bad task list syntax") + return res + +# +# Keep this part to tell users where the command line interface went +# + +if __name__ == '__main__': + import os, sys + sys.stderr.write("The command line utility has been moved to a separate 'hostlist' program.\n") + sys.exit(os.EX_USAGE) From bfd1866c59fdbc853e8cb215cea66c40ceed5d93 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Mon, 17 Jun 2024 07:45:51 +0000 Subject: [PATCH 2/4] update NEWS.md --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 8e57da3..2fb3894 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# slurmR 0.5-5 + +* Using [python-hostlist 1.23](https://www.nsc.liu.se/~kent/python-hostlist/) to no longer rely on SLURM output for extracting hostnames. + # slurmR 0.5-4 * Adressing [`roxygen2` issue #1491](https://github.com/r-lib/roxygen2/issues/1491). From 7fbb934268de47238a431b0a39d056928b2cb13f Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Mon, 17 Jun 2024 12:23:09 +0000 Subject: [PATCH 3/4] fix a couple of codacity findings --- inst/hostlist.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/inst/hostlist.py b/inst/hostlist.py index ba52b14..f93e77c 100644 --- a/inst/hostlist.py +++ b/inst/hostlist.py @@ -9,7 +9,7 @@ # Pär Lindfors and # Torbjörn Lönnemark , # National Supercomputer Centre -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -69,13 +69,12 @@ def expand_hostlist(hostlist, allow_duplicates=False, sort=False): results = [] bracket_level = 0 part = "" - + for c in hostlist + ",": if c == "," and bracket_level == 0: # Comma at top level, split! if part: results.extend(expand_part(part)) part = "" - bad_part = False else: part += c @@ -242,7 +241,7 @@ def collect_hostlist_1(left_right): # Add the right part unprocessed to the suffix. # This ensures than an already computed range expression # in the right part is not analyzed again. - suffix = suffix + right + suffix = suffix + right if num_str is None: # A left part with no numeric part at all gets special treatment! @@ -295,7 +294,7 @@ def sort_key(entry): # right) tuples. results = [] - needs_another_loop = False + needs_another_loop = False # Now group entries with the same prefix+suffix combination (the # key is the first element in the sortlist) to loop over them and @@ -315,7 +314,7 @@ def sort_key(entry): # formatting. range_list = [] - for ((prefix2, suffix2), num_int, num_width, host) in group: + for ((suffix2), num_int, num_width, host) in group: if host not in remaining: # Below, we will loop internally to enumate a whole range # at a time. We then remove the covered hosts from the set. @@ -425,7 +424,7 @@ def parse_slurm_tasks_per_node(s): repetitions = int(repetitions) if repetitions > MAX_SIZE: raise BadHostlist("task list repetitions too large") - for i in range(repetitions): + for _ in range(repetitions): res.append(tasks) else: raise BadHostlist("bad task list syntax") From 4e803056b6d3caa9643090f5b4a0196efca92331 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Mon, 17 Jun 2024 12:30:00 +0000 Subject: [PATCH 4/4] 3 more fixes --- inst/hostlist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inst/hostlist.py b/inst/hostlist.py index f93e77c..7ae1f68 100644 --- a/inst/hostlist.py +++ b/inst/hostlist.py @@ -14,7 +14,7 @@ # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU @@ -69,7 +69,7 @@ def expand_hostlist(hostlist, allow_duplicates=False, sort=False): results = [] bracket_level = 0 part = "" - + for c in hostlist + ",": if c == "," and bracket_level == 0: # Comma at top level, split! @@ -314,7 +314,7 @@ def sort_key(entry): # formatting. range_list = [] - for ((suffix2), num_int, num_width, host) in group: + for (num_int, num_width, host) in group: if host not in remaining: # Below, we will loop internally to enumate a whole range # at a time. We then remove the covered hosts from the set.