diff --git a/bits b/bits index fbb8d5dd..268740bb 100755 --- a/bits +++ b/bits @@ -195,21 +195,22 @@ function configBits() { config_file="" -for ((i=0;i<$ARGC;i++)); -do +for ((i=0;i<$ARGC;i++)); do arg=${ARGV[$i]} opt=${arg%%=*} val=${arg##*=} case $opt in --config) config_file=$val - ;; + ;; *) + new_args+=("$arg") ;; esac done -configBits $config_file +configBits "$config_file" +set -- "${new_args[@]}" for arg in $@ do diff --git a/bits_helpers/build.py b/bits_helpers/build.py index 2c006aba..62b1551c 100644 --- a/bits_helpers/build.py +++ b/bits_helpers/build.py @@ -547,7 +547,9 @@ def performPreferCheckWithTempDir(pkg, cmd): "The following packages are system requirements and could not be found:\n\n- %s\n\n" "Please run:\n\n\tbitsDoctor --defaults %s %s\n\nto get a full diagnosis." % ("\n- ".join(sorted(failed)), args.defaults, " ".join(args.pkgname))) - + + banner("Configured directory:\n%s", os.path.abspath(args.configDir)) + banner("Package Recipe will be searched in the following order \n%s", os.environ.get("BITS_PATH")) for x in specs.values(): x["requires"] = [r for r in x["requires"] if r not in args.disable] x["build_requires"] = [r for r in x["build_requires"] if r not in args.disable] diff --git a/bits_helpers/utilities.py b/bits_helpers/utilities.py index 752a9171..14a6a3f5 100644 --- a/bits_helpers/utilities.py +++ b/bits_helpers/utilities.py @@ -193,7 +193,7 @@ def resolve_tag(spec): - %(day)s - %(hour)s """ - return spec["tag"] % nowKwds + return spec["tag"] % {**nowKwds, **spec} def normalise_multiple_options(option, sep=","): @@ -282,7 +282,7 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl # possibly compatible linux distributions, but tries to get right the # common one, obvious one. If you use a Unknownbuntu which is compatible # with Ubuntu 15.10 you will still have to give an explicit platform -# string. +# string. # # FIXME: we should have a fallback for lsb_release, since platform.dist # is going away. @@ -395,11 +395,14 @@ def readDefaults(configDir, defaults, error, architecture, xdefaults): return (defaultsMeta, defaultsBody) - -def getRecipeReader(url:str , dist=None, genPackages={}): +def getRecipeReader(url: str, dist=None, genPackages={}): m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url) if m and m.group(1) == "generate": - return GeneratedPackage(genPackages[m.group(2)]["command"]) + pkg, version = m.group(2), m.group(3) + # search across all generated dirs + if pkg in genPackages and genPackages[pkg]["version"] == version: + return GeneratedPackage(genPackages[pkg]) + raise ValueError(f"Generated package {pkg}@{version} not found") elif m and dist: return GitReader(url, dist) else: @@ -407,8 +410,9 @@ def getRecipeReader(url:str , dist=None, genPackages={}): # Generate a recipe of package class GeneratedPackage(object): - def __init__(self, command) -> None: - self.command = command + def __init__(self, obj) -> None: + self.command = obj["command"] + self.url = obj["url"] def __call__(self): return getoutput(self.command).strip() @@ -481,13 +485,28 @@ def represent_ordereddict(dumper, data): YamlOrderedDumper.add_representer(OrderedDict, represent_ordereddict) return yaml.dump(s, Dumper=YamlOrderedDumper) -def parseRecipe(reader): +def parseRecipe(reader, generatePackages=None, visited=None): assert(reader.__call__) err, spec, recipe = (None, None, None) try: d = reader() header,recipe = d.split("---", 1) spec = yamlLoad(header) + if spec and "from" in spec: + basename = os.path.basename(getattr(reader, "url", "") or "") + filename = basename[:-3] if basename.endswith(".sh") else basename + repoDir = os.environ.get("BITS_REPO_DIR") + if visited is None: + visited = [] + if spec["from"] in visited: + raise RuntimeError(f" Cyclic Dependency: {' -> '.join(list(visited) + [spec['from']])}") + visited.append(spec["from"]) + parent_dir = os.path.join(repoDir, spec["from"]) + base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages) + base_reader = getRecipeReader(base_filename, repoDir, generatePackages[parent_dir]) + err, base_spec, base_recipe = parseRecipe(base_reader, generatePackages, visited) + spec, recipe_append = handleMergePolicy(spec, base_spec) + recipe = recipe + base_recipe if recipe_append else recipe validateSpec(spec) except RuntimeError as e: err = str(e) @@ -501,6 +520,8 @@ def parseRecipe(reader): err = "Unable to parse %s\n%s" % (reader.url, str(e)) except ValueError: err = "Unable to parse %s. Header missing." % reader.url + except Exception as e: + err = "Unknown Exception in parseRecipe %s.\n%s" % (reader.url, e) return err, spec, recipe @@ -602,24 +623,22 @@ def checkForFilename(taps, pkg, d): def getConfigPaths(configDir): configPath = os.environ.get("BITS_PATH") pkgDirs = [configDir] - if configPath: for d in [join(configDir, "%s.bits" % r) for r in configPath.split(",") if r]: - if exists(d): - pkgDirs.append(d) + if exists(d): + pkgDirs.append(d) return pkgDirs -def resolveFilename(taps, pkg, configDir, genPackages): - if pkg in genPackages: - return ("generate:%s@%s" % (pkg, genPackages[pkg]["version"]), genPackages[pkg]["pkgdir"]) - +def resolveFilename(taps, pkg, configDir, generatedPackages): for d in getConfigPaths(configDir): - filename = checkForFilename(taps,pkg,d) + if d in generatedPackages and pkg in generatedPackages[d]: + meta = generatedPackages[d][pkg] + return ("generate:%s@%s" % (pkg, meta["version"]), meta["pkgdir"]) + filename = checkForFilename(taps, pkg, d) if exists(filename): - return(filename,os.path.abspath(d)) - + return (filename, d) dieOnError(True, "Package %s not found in %s" % (pkg, configDir)) - + def resolveDefaultsFilename(defaults, configDir): configPath = os.environ.get("BITS_PATH") cfgDir = configDir @@ -627,7 +646,7 @@ def resolveDefaultsFilename(defaults, configDir): if configPath: for d in configPath.split(","): - pkgDirs.append(cfgDir + "/" + d + ".bits") + pkgDirs.append(cfgDir + "/" + d + ".bits") for d in pkgDirs: filename = "%s/defaults-%s.sh" % (d, defaults) @@ -642,7 +661,7 @@ def resolveDefaultsFilename(defaults, configDir): "\n".join("- " + basename(x).replace("defaults-", "").replace(".sh", "") for x in glob(join(configDir, "defaults-*.sh"))))) ''' - + def getPackageList(packages, specs, configDir, preferSystem, noSystem, architecture, disable, defaults, performPreferCheck, performRequirementCheck, performValidateDefaults, overrides, taps, log, force_rebuild=()): @@ -675,7 +694,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, dieOnError(not filename, "Package %s not found in %s" % (p, configDir)) assert(filename is not None) - err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages)) + err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages[pkgdir]), generatedPackages) dieOnError(err, err) # Unless there was an error, both spec and recipe should be valid. # otherwise the error should have been caught above. @@ -843,16 +862,66 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem, return (systemPackages, ownPackages, failedRequirements, validDefaults) def getGeneratedPackages(configDir): - pkgs = {} + all_pkgs = {} pkgDirs = getConfigPaths(configDir) for pkgdir in pkgDirs: - for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir,"*","packages.py"))]: - sys.path.insert(0,join(pkgdir, vp)) + dir_pkgs = {} + for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir, "*", "packages.py"))]: + sys.path.insert(0, join(pkgdir, vp)) pkg = __import__("packages") - pkg.getPackages(pkgs, pkgdir) - sys.modules.pop('packages') - x=sys.path.pop(0) - return pkgs + pkg.getPackages(dir_pkgs, pkgdir) + sys.modules.pop("packages") + sys.path.pop(0) + all_pkgs[pkgdir] = dir_pkgs + return all_pkgs + + +def handleMergePolicy(override_spec, final_base): + mergePolicy = override_spec.get("merge_policy", {}) + remove_keys = mergePolicy.get("remove", []) + force_inherit = mergePolicy.get("inherit", []) + if isinstance(remove_keys, str): + remove_keys = remove_keys.replace(" ", "").split(",") + recipe_append = "recipe" not in remove_keys + for k in remove_keys: + if k in final_base: + final_base.pop(k, None) + if isinstance(force_inherit, str): + force_inherit = force_inherit.replace(" ", "").split(",") + for key in force_inherit: + if key in final_base: + override_spec[key] = final_base[key] + merge_keys = mergePolicy.get("merge", []) + if isinstance(merge_keys, str): + merge_keys = merge_keys.replace(" ", "").split(",") + override_spec.pop("merge_policy", None) + override_spec.pop("from", None) + for key in merge_keys: + if key not in override_spec: + raise ValueError(f"Merge key {key} not found in override spec") + if key not in final_base: + final_base[key] = override_spec[key] + else: + if isinstance(final_base[key], OrderedDict) and isinstance( + override_spec[key], OrderedDict + ): + merged = final_base[key].copy() + merged.update(override_spec[key]) + final_base[key] = merged + elif isinstance(final_base[key], list) and isinstance( + override_spec[key], list + ): + for x in override_spec[key]: + if x not in final_base[key]: + final_base[key].append(x) + else: + raise ValueError( + f"Merge key not allowed for {key} as it's of type {type(final_base.get(key, 'unknown'))}" + ) + override_spec.pop(key) + for k, v in override_spec.items(): + final_base[k] = override_spec[k] + return final_base, recipe_append class Hasher: def __init__(self) -> None: