diff --git a/bits_helpers/build.py b/bits_helpers/build.py index fb37da66..d72251a4 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -11,7 +11,7 @@ from bits_helpers.utilities import getPackageList, asList from bits_helpers.utilities import validateDefaults from bits_helpers.utilities import Hasher -from bits_helpers.utilities import resolve_tag, resolve_version, short_commit_hash +from bits_helpers.utilities import resolve_tag, resolve_version, short_commit_hash, resolve_spec_data from bits_helpers.git import Git, git from bits_helpers.sl import Sapling from bits_helpers.scm import SCMError @@ -218,6 +218,15 @@ def h_all(data): # pylint: disable=function-redefined hasher(spec.get("source", "none")) if "source" in spec: hasher(tag) + if "sources" in spec: + for src in spec["sources"]: + h_all(src) + if "patches" in spec: + for patch in spec["patches"]: + h_all(patch) + with open(os.path.join(spec["pkgdir"], "patches", patch)) as ref: + patch_content = "".join(ref.readlines()) + h_all(patch_content) dh = Hasher() for dep in spec.get("requires", []): @@ -386,7 +395,7 @@ def generate_initdotsh(package, specs, architecture, post_build=False): # Set "env" variables. # We only put the values in double-quotes, so that they can refer to other # shell variables or do command substitution (e.g. $(brew --prefix ...)). - lines.extend('export {}="{}"'.format(key, value) + lines.extend('export {}="{}"'.format(key, resolve_spec_data(spec, value, "")) for key, value in spec.get("env", {}).items() if key != "DYLD_LIBRARY_PATH") @@ -398,16 +407,17 @@ def generate_initdotsh(package, specs, architecture, post_build=False): if key != "DYLD_LIBRARY_PATH") # First convert all values to list, so that we can use .setdefault().insert() below. - prepend_path = {key: asList(value) + prepend_path = {key: [resolve_spec_data(spec, dir, "") for dir in asList(value)] for key, value in spec.get("prepend_path", {}).items()} # By default we add the .../bin directory to PATH and .../lib to LD_LIBRARY_PATH. # Prepend to these paths, so that our packages win against system ones. - for key, value in (("PATH", "bin"), ("LD_LIBRARY_PATH", "lib")): + for key, value in (("PATH", "bin"), ("LD_LIBRARY_PATH", "lib"), ("LD_LIBRARY_PATH", "lib64")): prepend_path.setdefault(key, []).insert(0, "${}_ROOT/{}".format(bigpackage, value)) - lines.extend('export {key}="{value}${{{key}+:${key}}}"' - .format(key=key, value=":".join(value)) + lines.extend('[ ! -d "{value}" ] || export {key}="{value}${{{key}+:${key}}}"' + .format(key=key, value=dir) for key, value in prepend_path.items() - if key != "DYLD_LIBRARY_PATH") + if key != "DYLD_LIBRARY_PATH" + for dir in value) # Return string without a trailing newline, since we expect call sites to # append that (and the obvious way to inesrt it into the build template is by @@ -467,9 +477,9 @@ def doBuild(args, parser): buildTargetsDone = [] dieOnError(not exists(args.configDir), - 'Cannot find recipes under directory "%s".\n' - 'Maybe you need to "cd" to the right directory or ' - 'you forgot to run "bits init"?' % args.configDir) + 'Cannot find recipes under directory "%s".\n' + 'Maybe you need to "cd" to the right directory or ' + 'you forgot to run "bits init"?' % args.configDir) _, value = git(("symbolic-ref", "-q", "HEAD"), directory=args.configDir, check=False) branch_basename = re.sub("refs/heads/", "", value) @@ -481,7 +491,7 @@ def doBuild(args, parser): defaultsReader = lambda : readDefaults(args.configDir, args.defaults, parser.error, args.architecture) (err, overrides, taps) = parseDefaults(args.disable, - defaultsReader, debug) + defaultsReader, debug) dieOnError(err, err) makedirs(join(workDir, "SPECS"), exist_ok=True) @@ -623,6 +633,9 @@ def doBuild(args, parser): "source code from this directory, add a 'source:' key to " "alidist/{recipe}.sh instead." .format(package=p, recipe=p.lower())) + + if "tag" not in spec: + spec["tag"] = spec["version"] if "source" in spec: # Tag may contain date params like %(year)s, %(month)s, %(day)s, %(hour). spec["tag"] = resolve_tag(spec) @@ -649,10 +662,44 @@ def doBuild(args, parser): spec["tag"] = args.develPrefix if "develPrefix" in args else develPackageBranch spec["commit_hash"] = "0" + if "sources" in spec: + spec["commit_hash"] = spec["tag"] # Version may contain date params like tag, plus %(commit_hash)s, # %(short_hash)s and %(tag)s. spec["version"] = resolve_version(spec, args.defaults, branch_basename, branch_stream) + spec.setdefault("variables", OrderedDict(spec.get("variables", {}))) + variables = spec["variables"] + if "Python" in spec.get("requires", []): + # Find the Python package spec safely + python_version_str = "" + py_spec = specs.get("Python") + if isinstance(py_spec, dict): + python_version_str = (py_spec.get("version", "") or "").replace("v", "") + python_version = python_version_str.split(".") if python_version_str else [] + + # Safely extract major, minor, patch versions + major = python_version[0] if len(python_version) > 0 else "0" + minor = python_version[1] if len(python_version) > 1 else "0" + patch = python_version[2] if len(python_version) > 2 else "0" + + # Populate variables dictionary + variables.update({ + "python_major_version": major, + "python_minor_version": minor, + "python_patch_version": patch, + "python_major_minor": f"{major}.{minor}", + "python_major_minor_str": f"{major}{minor}", + }) + for k, v in variables.items(): + variables[k] = resolve_spec_data(spec, v, args.defaults, branch_basename, branch_stream) + if "source" in spec: + spec["source"] = resolve_spec_data(spec, spec["source"], args.defaults, branch_basename, branch_stream) + if "sources" in spec: + spec["sources"] = [resolve_spec_data(spec, src, args.defaults, branch_basename, branch_stream) for src in spec["sources"]] + if variables or spec.get("expand_recipe", False): + spec["recipe"] = resolve_spec_data(spec, spec["recipe"], args.defaults, branch_basename, branch_stream) + if spec["is_devel_pkg"] and "develPrefix" in args and args.develPrefix != "ali-master": spec["version"] = args.develPrefix @@ -1065,6 +1112,18 @@ def doBuild(args, parser): ("FULL_BUILD_REQUIRES", " ".join(spec["full_build_requires"])), ("FULL_REQUIRES", " ".join(spec["full_requires"])), ] + if "sources" in spec: + for idx, src in enumerate(spec["sources"]): + buildEnvironment.append(("SOURCE%s" % idx, basename(src))) + buildEnvironment.append(("SOURCE_COUNT", str(len(spec["sources"])))) + else: + buildEnvironment.append(("SOURCE_COUNT", "0")) + if "patches" in spec: + for idx, src in enumerate(spec["patches"]): + buildEnvironment.append(("PATCH%s" % idx, basename(src))) + buildEnvironment.append(("PATCH_COUNT", str(len(spec["patches"])))) + else: + buildEnvironment.append(("PATCH_COUNT", "0")) # Add the extra environment as passed from the command line. buildEnvironment += [e.partition('=')[::2] for e in args.environment] @@ -1259,3 +1318,4 @@ def doBuild(args, parser): banner("Untracked files in the following directories resulted in a rebuild of " "the associated package and its dependencies:\n%s\n\nPlease commit or remove them to avoid useless rebuilds.", "\n".join(untrackedFilesDirectories)) debug("Everything done") + diff --git a/bits_helpers/download.py b/bits_helpers/download.py new file mode 100644 index 00000000..a33a4dce --- /dev/null +++ b/bits_helpers/download.py @@ -0,0 +1,343 @@ +try: + from md5 import new as md5adder +except ImportError: + from hashlib import md5 as md5adder +from os.path import abspath, join, exists, dirname, basename +from os import rename, unlink +import re +from tempfile import mkdtemp +from subprocess import getstatusoutput +from urllib.request import urlopen, Request +from urllib.error import URLError +import base64 +from time import time +from types import SimpleNamespace +from bits_helpers.log import error, warning, debug +import json + +urlRe = re.compile(r".*:.*/.*") +urlAuthRe = re.compile(r'^(http(s|)://)([^:]+:[^@]+)@(.+)$') + +def format(s, **kwds): + return s % kwds + +def executeWithErrorCheck(command, errorMessage): + debug(command) + error, output = getstatusoutput(command) + if error: + warning(errorMessage + ":") + warning("") + warning(command) + warning("") + warning("resulted in:") + warning(output) + return False + debug(output) + return True + + +def packCheckout(tempdir, dest, *exports): + """ Use this helper method when download protocol is like cvs/svn/git + where the code is checked out in a temporary directory and then tarred + up. + """ + export = " ".join(['"%s"' % x for x in exports]) + packCommand = """cd %(tempdir)s; tar -zcf "%(dest)s" %(export)s """ + packCommand = packCommand % locals() + errorMessage = "Error while creating a tar archive for checked out area" + return executeWithErrorCheck(packCommand, errorMessage) + +# We have our own version rather than using the one from os +# because the latter does not work seem to be thread safe. +def makedirs(path): + returncode, out = getstatusoutput("mkdir -p %s" % (path,)) + if returncode != 0: + raise OSError("makedirs() failed (return: %s):\n%s" % (returncode, out)) + +def downloadUrllib2(source, destDir, work_dir, dest_filename=None): + try: + dest = "/".join([destDir.rstrip("/"), dest_filename if dest_filename else basename(source)]) + headers={"Cache-Control": "no-cache"} + m = urlAuthRe.match(source) + if m: + source = m.group(1)+m.group(4) + headers['Authorization'] = "Basic %s" % base64.b64encode(m.group(3)) + req = Request(source, headers=headers) + s = urlopen(req) + tmpfile = "%s.%f.tmp" % (dest, time()) + f = open(tmpfile, "wb") + # Read in blocks to avoid using too much memory. + block_sz = 8192 * 16 + while True: + buffer = s.read(block_sz) + if not buffer: + break + f.write(buffer) + f.close() + if not exists(dest): + rename(tmpfile, dest) + else: + unlink(tmpfile) + except URLError as e: + debug("Error while downloading %s: %s" % (source, e)) + return False + except Exception as e: + debug("Error while downloading %s: %s" % (source, e)) + return False + return True + +# Download a files from a git url. We do not clone the remote reposiotory, but +# we simply pull the branch we are interested in and then we drop all the git +# information while creating a tarball. The syntax to define a repository is +# the following: +# +# git:/local/repository?obj=BRANCH/TAG +# git://remote-repository?obj=BRANCH/TAG +# git+https://remote-repository-over-http/foo.git?obj=BRANCH/TAG +# +# If "obj" does not contain a "/", it's value will be considered a branch and TAG will be "HEAD". +# If "obj" is not specified, it will be "master/HEAD" by default. +# By default export is the -TAG unless it is HEAD, +# in which case it will be -BRANCH. +# One can specify an additional parameter +# +# filter= +# +# which will be used to pack only a subset of the checkout. + +def downloadGit(source, dest, work_dir): + protocol, gitroot, args = parseGitUrl(source) + tempdir = createTempDir(work_dir, "tmp") + + exportpath = join(tempdir, args["export"]) + if protocol=="git": protocol="https" + if protocol: + protocol += "://" + if not protocol and not gitroot.endswith(".git"): + gitroot = join(gitroot, ".git") + + dest = join(dest, args["output"].lstrip("/")) + args.update({"protocol": protocol, "tempdir": tempdir, + "gitroot": gitroot, "dest": dest, + "exportpath": exportpath}) + makedirs(exportpath) + if "submodules" in args: + args["submodules"] = " git submodule update --recursive --init &&" + else: + args["submodules"] = "" + command = format("cd %(exportpath)s &&" + "git init &&" + "git pull --tags %(protocol)s%(gitroot)s refs/heads/%(branch)s &&" + "git remote add origin %(protocol)s%(gitroot)s &&" + "git reset --hard %(tag)s && %(submodules)s" + "find . ! -path '%(filter)s' -delete &&" + "rm -rf .git .gitattributes .gitignore", **args) + error, output = getstatusoutput(command % args) + if error: + warning("Error while downloading sources from %s using git.\n\n" + "%s\n\n" + "resulted in:\n%s" % (gitroot, command % args, output)) + return False + return packCheckout(args["tempdir"], args["dest"], args["export"]) + + +def parseGitUrl(url): + protocol, gitroot, args = parseUrl(url, requestedKind="git", + defaults={"obj": "master/HEAD"}) + parts = args["obj"].rsplit("/", 1) + if len(parts) != 2: + parts += ["HEAD"] + args["branch"], args["tag"] = parts + + if not "export" in args: + args["export"] = basename(re.sub(r"\.git$", "", re.sub(r"[?].*", "", gitroot))) + if args["tag"] != "HEAD": + args["export"] += args["tag"] + else: + args["export"] += args["branch"] + + if not "output" in args: + args["output"] = args["export"] + ".tar.gz" + args["gitroot"] = gitroot + if not "filter" in args: + args["filter"] = "*" + args["filter"] = sanitize(args["filter"]) + return protocol, gitroot, args + + + + + +def createTempDir(workDir, subDir): + tempdir = join(workDir, subDir) + if not exists(tempdir): + makedirs(tempdir) + tempdir = mkdtemp(dir=tempdir) + return tempdir + +# Minimal string sanitization. +def sanitize(s): + return re.sub(r"[^a-zA-Z_0-9*./-]", "", s) + +def getUrlChecksum(s): + m = md5adder(fixUrl(s).encode()) + return m.hexdigest() + +def parseUrl(url, requestedKind=None, defaults={}, required=[]): + match = re.match("([^+:]*)([^:]*)://([^?]*)(.*)", url) + if not match: + raise MalformedUrl(url) + parts = match.groups() + protocol, deliveryProtocol, server, arguments = match.groups() + arguments = arguments.strip("?") + # In case of urls of the kind: + # git+https://some.web.git.repository.net + # we consider "https" the actual protocol and + # "git" merely the request kind. + if requestedKind and not protocol == requestedKind: + raise MalformedUrl(url) + if deliveryProtocol: + protocol = deliveryProtocol.strip("+") + arguments.replace("&", "&") + args = list(defaults.items()) + parsedArgs = re.split("&", arguments) + parsedArgs = [x.split("=") for x in parsedArgs] + parsedArgs = [(len(x) != 2 and [x[0], True]) or x for x in parsedArgs] + args.extend(parsedArgs) + argsDict = dict(args) + missingArgs = [arg for arg in required if arg not in argsDict] + if missingArgs: + raise MalformedUrl(url, missingArgs) + return protocol, server, argsDict + +def fixUrl(s): + for x in ['no-cmssdt-cache=1', 'cmdist-generated=1']: + if x in s: + s = s.replace(x,'').replace("&&","&").replace("?&","?") + if s.endswith('&'): s=s[:-1] + if s.endswith('?'): s=s[:-1] + return s + +def downloadPip(source, dest, work_dir): + # Valid PIP URL formats are + # pip://package/version?[pip_options=downloadOptions&][pip=pip_command&][pip_package=package&]output=/tarbalname + # pip://package/version/tarbalname + url_parts = source.split("pip://", 1)[1].split("?", 1) + filename = source.rsplit("/", 1)[1] + opts = [] + if len(url_parts) > 1: opts = url_parts[1].split("&") + pkg = url_parts[0].split("/") + pack = pkg[0].strip() + tar_names = [pack.replace("-", "_"), pack] if '-' in pack else [pack] + for tar_name in tar_names: + pypi_file = '%s-%s.tar.gz' % (tar_name, pkg[1].strip()) + pypi_url = 'https://pypi.io/packages/source/%s/%s/%s' % (pack[0], pack, pypi_file) + if downloadUrllib2(pypi_url, dest, work_dir, dest_filename=filename): + return + pack = pack + '==' + pkg[1].strip() + pip_opts = "--no-deps --no-binary=:all:" + pip="pip" + isSourceDownload=True + + for opt in opts: + if opt.startswith("pip="): pip=opt.split('=',1)[-1] + elif opt.startswith("pip_options="): + pip_optsT = opt[12:].replace("+", " ").replace("%20", " ").replace("%3D", "=") + # hack here a alternative source location + pip_opts = '' + spSrc = pip_optsT.split() + for i in range(len(spSrc)): + if 'ALTSRC' in spSrc[i]: + pack = spSrc[i + 1] + i = i + 1 + else: + if ("no-binary" in spSrc[i]) or ("only-binary" in spSrc[i]): + spSrc[i] = re.sub(',arch=[a-z0-9_]+','',spSrc[i]) + pip_opts = pip_opts + ' ' + spSrc[i] + if "only-binary=:all:" in spSrc[i]: + isSourceDownload=False + elif "no-binary" in spSrc[i] and "all" not in spSrc[i]: + isSourceDownload=False #not totally robust - but basically use pip if source is overridden + + if isSourceDownload: + debug("Looking for sources at https://pypi.org/pypi/"+pack.split('=')[0]+"/json") + fj=urlopen("https://pypi.org/pypi/"+pack.split('=')[0]+"/json") + data=json.load(fj) + url=None + if "releases" in data and pack.split('=')[2] in data["releases"]: + for file in data["releases"][pack.split('=')[2]]: + if file["packagetype"] == "sdist": + url=file["url"] + if url is not None: + debug("Found source on pypi - downloading") + return downloadUrllib2(url, dest, work_dir, dest_filename=filename) + + if not '--no-deps' in pip_opts: pip_opts = '--no-deps ' + pip_opts + if not '--no-cache-dir' in pip_opts: pip_opts = '--no-cache-dir ' + pip_opts + comm = 'cd ' + dest + ";" + pip + ' download ' + pip_opts + ' --disable-pip-version-check -q -d . %s; [ -e %s ] || mv *.* %s; ls -l' % (pack, filename, filename) + error, output = getstatusoutput(comm) + return not error + + +downloadHandlers = { + 'http': downloadUrllib2, + 'https': downloadUrllib2, + 'ftp': downloadUrllib2, + 'ftps': downloadUrllib2, + 'git': downloadGit, + 'pip': downloadPip} + +def download(source, dest, work_dir): + noCmssdtCache = True if 'no-cmssdt-cache=1' in source else False + isCmsdistGenerated = True if 'cmdist-generated=1' in source else False + source = fixUrl(source) + checksum = getUrlChecksum(source) + + # Syntactic sugar to allow the following urls for tag collector: + # + # cmstc:[base.]release[.tagset[.tagset[...]]]/src.tar.gz + # + # in place of: + # + # cmstc://?tag=release&baserel=base&extratag=tagset1,tagset2,..&module=CMSSW&export=src&output=/src.tar.gz + if source.startswith("cmstc:") and not source.startswith("cmstc://"): + url = source.split(":", 1)[1] + desc, output = url.rsplit("/", 1) + parts = desc.split(".") + releases = [x for x in parts if not x.isdigit()] + extratags = [x for x in parts if x.isdigit()] + if extratags: + extratags = "&extratags=" + ",".join(extratags) + if len(releases) == 1: + baserel = "" + release = "tag=" + releases[0] + elif len(releases) == 2: + baserel = "&baserel=" + releases[0] + release = releases[1] + else: + raise MalformedUrl(source) + source = "cmstc://?%s%s%s&module=CMSSW&export=src&output=/%s" % (release, baserel, extratags, output) + + cacheDir = abspath(join(work_dir, "SOURCES/cache")) + urlTypeRe = re.compile(r"([^:+]*)([^:]*)://.*") + match = urlTypeRe.match(source) + if not urlTypeRe.match(source): + raise MalformedUrl(source) + downloadHandler = downloadHandlers[match.group(1)] + filename = source.rsplit("/", 1)[1] + downloadDir = join(cacheDir, checksum[0:2], checksum) + try: + makedirs(downloadDir) + except OSError as e: + if not exists(downloadDir): + raise downloadDir + + realFile = join(downloadDir, filename) + if not exists(realFile): + debug ("Trying to fetch source file: %s", source) + downloadHandler(source, downloadDir, work_dir) + if exists(realFile): + executeWithErrorCheck("mkdir -p {dest}; cp {src} {dest}/".format(dest=dest, src=realFile), "Failed to move source") + else: + raise downloadDir + return diff --git a/bits_helpers/utilities.py b/bits_helpers/utilities.py index 02eb6a10..9742c965 100644 --- a/bits_helpers/utilities.py +++ b/bits_helpers/utilities.py @@ -124,9 +124,10 @@ def short_commit_hash(spec): "day": str(now.day).zfill(2), "hour": str(now.hour).zfill(2) } -def resolve_version(spec, defaults, branch_basename, branch_stream): - """Expand the version replacing the following keywords: +def resolve_spec_data(spec, data, defaults, branch_basename="", branch_stream=""): + """Expand the data replacing the following keywords: + - %(package)s - %(commit_hash)s - %(short_hash)s - %(tag)s @@ -134,6 +135,8 @@ def resolve_version(spec, defaults, branch_basename, branch_stream): - %(branch_stream)s - %(tag_basename)s - %(defaults_upper)s + - %(version)s + - %(root_dir)s - %(year)s - %(month)s - %(day)s @@ -144,7 +147,10 @@ def resolve_version(spec, defaults, branch_basename, branch_stream): defaults_upper = defaults != "release" and "_" + defaults.upper().replace("-", "_") or "" commit_hash = spec.get("commit_hash", "hash_unknown") tag = str(spec.get("tag", "tag_unknown")) - return spec["version"] % { + package = spec.get("package") + all_vars = { + "package": package, + "root_dir": "${%s_ROOT}" % package.upper().replace("-","_"), "commit_hash": commit_hash, "short_hash": commit_hash[0:10], "tag": tag, @@ -152,8 +158,28 @@ def resolve_version(spec, defaults, branch_basename, branch_stream): "branch_stream": branch_stream or tag, "tag_basename": basename(tag), "defaults_upper": defaults_upper, + "version": str(spec.get("version", "version_unknown")), + "platform_machine": platform.machine(), + "sys_platform": sys.platform, + "os_name": os.name, **nowKwds, } + for k, v in spec.get("variables",{}).items(): + all_vars[k] = v + + # Support for indirect variable expansion e.g. with + # variables: + # v1: foo + # foo_key: bar + # final: %%(%(v1)s_key)s + # "final" will have the value "bar" (first expanded to "%(foo_key)s" and + # then to value of "foo_key" i.e. "bar") + while re.search("\%\([a-zA-Z][a-zA-Z0-9_]*\)s", data): + data = data % all_vars + return data + +def resolve_version(spec, defaults, branch_basename, branch_stream): + return resolve_spec_data(spec, spec["version"], defaults, branch_basename, branch_stream) def resolve_tag(spec): """Expand the tag, replacing the following keywords: @@ -331,13 +357,22 @@ def readDefaults(configDir, defaults, error, architecture): return (defaultsMeta, defaultsBody) -def getRecipeReader(url:str , dist=None): - m = re.search(r'^dist:(.*)@([^@]+)$', url) - if m and dist: +def getRecipeReader(url:str , dist=None, genPackages={}): + m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url) + if m and m.group(1) == "generate": + return GeneratedPackage(genPackages[m.group(2)]["command"]) + elif m and dist: return GitReader(url, dist) else: return FileReader(url) +# Generate a recipe of package +class GeneratedPackage(object): + def __init__(self, command) -> None: + self.command = command + def __call__(self): + return getoutput(self.command).strip() + # Read a recipe from a file class FileReader(object): def __init__(self, url) -> None: @@ -440,16 +475,21 @@ def checkForFilename(taps, pkg, d): filename = taps.get(pkg, "%s/%s/latest" % (d, pkg)) return filename -def resolveFilename(taps, pkg, configDir): +def getConfigPaths(configDir): configPath = os.environ.get("BITS_PATH") - cfgDir = configDir - pkgDirs = [cfgDir] + pkgDirs = [configDir] if configPath: - for d in configPath.split(","): - pkgDirs.append(cfgDir + "/" + d + ".bits") + for d in [join(configDir, "%s.bits" % r) for r in configPath.split(",") if r]: + if exists(d): + pkgDirs.append(d) + return pkgDirs - for d in pkgDirs: +def resolveFilename(taps, pkg, configDir, genPackages): + if pkg in genPackages: + return ("generate:%s@%s" % (pkg, genPackages[pkg]["version"]), genPackages[pkg]["pkgdir"]) + + for d in getConfigPaths(configDir): filename = checkForFilename(taps,pkg,d) if exists(filename): return(filename,os.path.abspath(d)) @@ -489,6 +529,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, requirementsCache = {} trackingEnvCache = {} packages = packages[:] + generatedPackages = getGeneratedPackages(configDir) validDefaults = [] # empty list: all OK; None: no valid default; non-empty list: list of valid ones while packages: @@ -505,12 +546,12 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, # and all dependencies' names go into a package's hash. pkg_filename = ("defaults-" + defaults) if p == "defaults-release" else p.lower() - filename,pkgdir = resolveFilename(taps, pkg_filename, configDir) + filename,pkgdir = resolveFilename(taps, pkg_filename, configDir, generatedPackages) dieOnError(not filename, "Package %s not found in %s" % (p, configDir)) assert(filename is not None) - err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir)) + err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages)) dieOnError(err, err) # Unless there was an error, both spec and recipe should be valid. # otherwise the error should have been caught above. @@ -675,6 +716,17 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, packages += spec["requires"] return (systemPackages, ownPackages, failedRequirements, validDefaults) +def getGeneratedPackages(configDir): + pkgs = {} + pkgDirs = getConfigPaths(configDir) + for pkgdir in pkgDirs: + for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir,"*","packages.py"))]: + sys.path.insert(0,join(pkgdir, vp)) + pkg = __import__("packages") + pkg.getPackages(pkgs, pkgdir) + sys.modules.pop('packages') + x=sys.path.pop(0) + return pkgs class Hasher: def __init__(self) -> None: diff --git a/bits_helpers/workarea.py b/bits_helpers/workarea.py index 257fe48a..2ab04e17 100644 --- a/bits_helpers/workarea.py +++ b/bits_helpers/workarea.py @@ -7,6 +7,7 @@ from collections import OrderedDict from bits_helpers.log import dieOnError, debug, error +from bits_helpers.download import download from bits_helpers.utilities import call_ignoring_oserrors, symlink, short_commit_hash FETCH_LOG_NAME = "fetch-log.txt" @@ -150,7 +151,14 @@ def scm_exec(command, directory=".", check=True): if spec["commit_hash"] != spec["tag"]: symlink(spec["commit_hash"], os.path.join(source_parent_dir, spec["tag"].replace("/", "_"))) - if "source" not in spec: + if "patches" in spec: + os.makedirs(source_dir, exist_ok=True) + for patch in spec["patches"]: + shutil.copyfile(os.path.join(spec["pkgdir"], 'patches', patch),os.path.join(source_dir, patch)) + if "sources" in spec: + for s in spec["sources"]: + download(s,source_dir, work_dir) + elif "source" not in spec: # There are no sources, so just create an empty SOURCEDIR. os.makedirs(source_dir, exist_ok=True) elif spec["is_devel_pkg"]: