support for inheriting package recepies and improved generated packages

bits

@@ -195,21 +195,22 @@ function configBits() {
 
 config_file=""
 
-for ((i=0;i<$ARGC;i++));
-do
+for ((i=0;i<$ARGC;i++)); do
     arg=${ARGV[$i]}
     opt=${arg%%=*}
     val=${arg##*=}
     case $opt in
        --config)
           config_file=$val
-       ;;   
+       ;;
        *)
+         new_args+=("$arg")
          ;;
    esac
 done
 
-configBits $config_file
+configBits "$config_file"
+set -- "${new_args[@]}"
 
 for arg in $@
 do

bits_helpers/build.py

@@ -547,7 +547,9 @@ def performPreferCheckWithTempDir(pkg, cmd):
              "The following packages are system requirements and could not be found:\n\n- %s\n\n"
              "Please run:\n\n\tbitsDoctor --defaults %s %s\n\nto get a full diagnosis." %
              ("\n- ".join(sorted(failed)), args.defaults, " ".join(args.pkgname)))
-
+
+  banner("Configured directory:\n%s", os.path.abspath(args.configDir))
+  banner("Package Recipe will be searched in the following order \n%s", os.environ.get("BITS_PATH"))
   for x in specs.values():
     x["requires"] = [r for r in x["requires"] if r not in args.disable]
     x["build_requires"] = [r for r in x["build_requires"] if r not in args.disable]

bits_helpers/utilities.py

@@ -193,7 +193,7 @@ def resolve_tag(spec):
   - %(day)s
   - %(hour)s
   """
-  return spec["tag"] % nowKwds
+  return spec["tag"] % {**nowKwds, **spec}
 
 
 def normalise_multiple_options(option, sep=","):
@@ -282,7 +282,7 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl
 # possibly compatible linux distributions, but tries to get right the
 # common one, obvious one. If you use a Unknownbuntu which is compatible
 # with Ubuntu 15.10 you will still have to give an explicit platform
-# string. 
+# string.
 #
 # FIXME: we should have a fallback for lsb_release, since platform.dist
 # is going away.
@@ -395,20 +395,24 @@ def readDefaults(configDir, defaults, error, architecture, xdefaults):
 
   return (defaultsMeta, defaultsBody)
 
-
-def getRecipeReader(url:str , dist=None, genPackages={}):
+def getRecipeReader(url: str, dist=None, genPackages={}):
   m = re.search(r'^(dist|generate):(.*)@([^@]+)$', url)
   if m and m.group(1) == "generate":
-    return GeneratedPackage(genPackages[m.group(2)]["command"])
+    pkg, version = m.group(2), m.group(3)
+    # search across all generated dirs
+    if pkg in genPackages and genPackages[pkg]["version"] == version:
+      return GeneratedPackage(genPackages[pkg])
+    raise ValueError(f"Generated package {pkg}@{version} not found")
   elif m and dist:
     return GitReader(url, dist)
   else:
     return FileReader(url)
 
 # Generate a recipe of package
 class GeneratedPackage(object):
-  def __init__(self, command) -> None:
-    self.command = command
+  def __init__(self, obj) -> None:
+    self.command = obj["command"]
+    self.url = obj["url"]
   def __call__(self):
     return  getoutput(self.command).strip()
 
@@ -481,13 +485,28 @@ def represent_ordereddict(dumper, data):
   YamlOrderedDumper.add_representer(OrderedDict, represent_ordereddict)
   return yaml.dump(s, Dumper=YamlOrderedDumper)
 
-def parseRecipe(reader):
+def parseRecipe(reader, generatePackages=None, visited=None):
   assert(reader.__call__)
   err, spec, recipe = (None, None, None)
   try:
     d = reader()
     header,recipe = d.split("---", 1)
     spec = yamlLoad(header)
+    if spec and "from" in spec:
+      basename = os.path.basename(getattr(reader, "url", "") or "")
+      filename = basename[:-3] if basename.endswith(".sh") else basename
+      repoDir = os.environ.get("BITS_REPO_DIR")
+      if visited is None:
+        visited = []
+      if spec["from"] in visited:
+        raise RuntimeError(f" Cyclic Dependency: {' -> '.join(list(visited) + [spec['from']])}")
+      visited.append(spec["from"])
+      parent_dir = os.path.join(repoDir, spec["from"])
+      base_filename, pkgdir = resolveFilename({}, filename, parent_dir, generatePackages)
+      base_reader = getRecipeReader(base_filename, repoDir, generatePackages[parent_dir])
+      err, base_spec, base_recipe = parseRecipe(base_reader, generatePackages, visited)
+      spec, recipe_append = handleMergePolicy(spec, base_spec)
+      recipe = recipe + base_recipe if recipe_append else recipe
     validateSpec(spec)
   except RuntimeError as e:
     err = str(e)
@@ -501,6 +520,8 @@ def parseRecipe(reader):
     err = "Unable to parse %s\n%s" % (reader.url, str(e))
   except ValueError:
     err = "Unable to parse %s. Header missing." % reader.url
+  except Exception as e:
+    err = "Unknown Exception in parseRecipe %s.\n%s" % (reader.url, e)
   return err, spec, recipe
 
 
@@ -602,32 +623,30 @@ def checkForFilename(taps, pkg, d):
 def getConfigPaths(configDir):
   configPath = os.environ.get("BITS_PATH")
   pkgDirs = [configDir]
-
   if configPath:
     for d in [join(configDir, "%s.bits" % r) for r in configPath.split(",") if r]:
-       if exists(d):
-         pkgDirs.append(d)
+      if exists(d):
+        pkgDirs.append(d)
   return pkgDirs
 
-def resolveFilename(taps, pkg, configDir, genPackages):
-  if pkg in genPackages:
-    return ("generate:%s@%s" % (pkg, genPackages[pkg]["version"]), genPackages[pkg]["pkgdir"])
-
+def resolveFilename(taps, pkg, configDir, generatedPackages):
   for d in getConfigPaths(configDir):
-    filename = checkForFilename(taps,pkg,d)
+    if d in generatedPackages and pkg in generatedPackages[d]:
+      meta = generatedPackages[d][pkg]
+      return ("generate:%s@%s" % (pkg, meta["version"]), meta["pkgdir"])
+    filename = checkForFilename(taps, pkg, d)
     if exists(filename):
-      return(filename,os.path.abspath(d))
-
+      return (filename, d)
   dieOnError(True, "Package %s not found in %s" % (pkg, configDir))
-    
+
 def resolveDefaultsFilename(defaults, configDir):
   configPath = os.environ.get("BITS_PATH")
   cfgDir = configDir
   pkgDirs = [cfgDir]
 
   if configPath:
     for d in configPath.split(","):
-       pkgDirs.append(cfgDir + "/" + d + ".bits")
+      pkgDirs.append(cfgDir + "/" + d + ".bits")
 
   for d in pkgDirs:
     filename = "%s/defaults-%s.sh" % (d, defaults)
@@ -642,7 +661,7 @@ def resolveDefaultsFilename(defaults, configDir):
            "\n".join("- " + basename(x).replace("defaults-", "").replace(".sh", "")
                      for x in glob(join(configDir, "defaults-*.sh")))))
   '''
-  
+
 def getPackageList(packages, specs, configDir, preferSystem, noSystem,
                    architecture, disable, defaults, performPreferCheck, performRequirementCheck,
                    performValidateDefaults, overrides, taps, log, force_rebuild=()):
@@ -675,7 +694,7 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
     dieOnError(not filename, "Package %s not found in %s" % (p, configDir))
     assert(filename is not None)
 
-    err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages))
+    err, spec, recipe = parseRecipe(getRecipeReader(filename, configDir, generatedPackages[pkgdir]), generatedPackages)
     dieOnError(err, err)
     # Unless there was an error, both spec and recipe should be valid.
     # otherwise the error should have been caught above.
@@ -843,16 +862,66 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
   return (systemPackages, ownPackages, failedRequirements, validDefaults)
 
 def getGeneratedPackages(configDir):
-  pkgs = {}
+  all_pkgs = {}
   pkgDirs = getConfigPaths(configDir)
   for pkgdir in pkgDirs:
-    for vp in [x.split(os.sep)[-2] for x in  glob(join(pkgdir,"*","packages.py"))]:
-      sys.path.insert(0,join(pkgdir, vp))
+    dir_pkgs = {}
+    for vp in [x.split(os.sep)[-2] for x in glob(join(pkgdir, "*", "packages.py"))]:
+      sys.path.insert(0, join(pkgdir, vp))
       pkg = __import__("packages")
-      pkg.getPackages(pkgs, pkgdir)
-      sys.modules.pop('packages')
-      x=sys.path.pop(0)
-  return pkgs
+      pkg.getPackages(dir_pkgs, pkgdir)
+      sys.modules.pop("packages")
+      sys.path.pop(0)
+    all_pkgs[pkgdir] = dir_pkgs
+  return all_pkgs
+
+
+def handleMergePolicy(override_spec, final_base):
+  mergePolicy = override_spec.get("merge_policy", {})
+  remove_keys = mergePolicy.get("remove", [])
+  force_inherit = mergePolicy.get("inherit", [])
+  if isinstance(remove_keys, str):
+    remove_keys = remove_keys.replace(" ", "").split(",")
+  recipe_append = "recipe" not in remove_keys
+  for k in remove_keys:
+    if k in final_base:
+      final_base.pop(k, None)
+  if isinstance(force_inherit, str):
+    force_inherit = force_inherit.replace(" ", "").split(",")
+  for key in force_inherit:
+    if key in final_base:
+      override_spec[key] = final_base[key]
+  merge_keys = mergePolicy.get("merge", [])
+  if isinstance(merge_keys, str):
+    merge_keys = merge_keys.replace(" ", "").split(",")
+  override_spec.pop("merge_policy", None)
+  override_spec.pop("from", None)
+  for key in merge_keys:
+    if key not in override_spec:
+      raise ValueError(f"Merge key {key} not found in override spec")
+    if key not in final_base:
+      final_base[key] = override_spec[key]
+    else:
+      if isinstance(final_base[key], OrderedDict) and isinstance(
+        override_spec[key], OrderedDict
+      ):
+        merged = final_base[key].copy()
+        merged.update(override_spec[key])
+        final_base[key] = merged
+      elif isinstance(final_base[key], list) and isinstance(
+        override_spec[key], list
+      ):
+        for x in override_spec[key]:
+          if x not in final_base[key]:
+            final_base[key].append(x)
+      else:
+        raise ValueError(
+          f"Merge key not allowed for {key} as it's of type {type(final_base.get(key, 'unknown'))}"
+        )
+    override_spec.pop(key)
+  for k, v in override_spec.items():
+    final_base[k] = override_spec[k]
+  return final_base, recipe_append
 
 class Hasher:
   def __init__(self) -> None: