diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index e9e6ddfb9..1f4295223 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1687,3 +1687,124 @@ def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) + + +############################################################# +# Buildtree Commands # +############################################################# +@cli.group(short_help="Manipulate cached buildtree.") +def buildtree(): + """Manipulate cached buildtree""" + + +##################################################################### +# Buildtree Checkout Command # +##################################################################### +@buildtree.command(name="checkout", short_help="Checkout contents of an buildtree") +@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.") +@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") +@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible") +@click.option( + "--tar", + default=None, + metavar="LOCATION", + type=click.Path(), + help="Create a tarball from the artifact contents instead " + "of a file tree. If LOCATION is '-', the tarball " + "will be dumped to the standard output.", +) +@click.option( + "--compression", + default=None, + type=click.Choice(["gz", "xz", "bz2"]), + help="The compression option of the tarball created.", +) +@click.option( + "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to" +) +@click.option( + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading artifacts", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", +) +@click.argument("target", required=False, type=click.Path(readable=False)) +@click.pass_obj +def buildtree_checkout( + app, + buildroot, + force, + hardlinks, + tar, + compression, + directory, + artifact_remotes, + ignore_project_artifact_remotes, + target, +): + """Checkout buildtree + + When this command is executed from a workspace directory, the default + is to checkout the artifact of the workspace element. + """ + from .. import utils + + if hardlinks and tar: + click.echo("ERROR: options --hardlinks and --tar conflict", err=True) + sys.exit(-1) + + if tar and directory: + click.echo("ERROR: options --directory and --tar conflict", err=True) + sys.exit(-1) + + if not tar: + if compression: + click.echo("ERROR: --compression can only be provided if --tar is provided", err=True) + sys.exit(-1) + else: + location = tar + try: + inferred_compression = utils._get_compression(tar) + except UtilError as e: + click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True) + sys.exit(-1) + if compression and inferred_compression != "" and inferred_compression != compression: + click.echo( + "WARNING: File extension and compression differ." + "File extension has been overridden by --compression", + err=True, + ) + if not compression: + compression = inferred_compression + + with app.initialized(): + if not target: + target = app.stream.get_default_target() + if not target: + raise AppError('Missing argument "ELEMENT".') + + if not tar: + if directory is None: + location = os.path.abspath(os.path.join(os.getcwd(), target)) + if location[-4:] == ".bst": + location = location[:-4] + else: + location = directory + + app.stream.buildtree_checkout( + target, + location=location, + buildroot=buildroot, + force=force, + hardlinks=hardlinks, + compression=compression, + tar=bool(tar), + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index f945bf568..6dd915294 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -333,23 +333,8 @@ def shell( reason="shell-missing-deps", ) - # Check if we require a pull queue attempt, with given artifact state and context if usebuildtree: - if not element._cached_buildroot(): - if not element._cached(): - message = "Artifact not cached locally or in available remotes" - reason = "missing-buildtree-artifact-not-cached" - elif element._buildroot_exists(): - message = "Buildtree is not cached locally or in available remotes" - reason = "missing-buildtree-artifact-buildtree-not-cached" - else: - message = "Artifact was created without buildtree" - reason = "missing-buildtree-artifact-created-without-buildtree" - raise StreamError(message, reason=reason) - - # Raise warning if the element is cached in a failed state - if element._cached_failure(): - self._context.messenger.warn("using a buildtree from a failed build.") + self._check_buildtree(element) # Ensure we have our sources if we are launching a build shell if scope == _Scope.BUILD and not usebuildtree: @@ -916,6 +901,74 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE): if not ref_removed: self._context.messenger.info("No artifacts were removed") + + # buildtree_checkout() + # + # Checkout target buildtree artifact to the specified location + # + # Args: + # target: Target to checkout + # location: Location to checkout the artifact to + # force: Whether files can be overwritten if necessary + # hardlinks: Whether checking out files hardlinked to + # their artifacts is acceptable + # tar: If true, a tarball from the artifact contents will + # be created, otherwise the file tree of the artifact + # will be placed at the given location. If true and + # location is '-', the tarball will be dumped on the + # standard output. + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # + def buildtree_checkout( + self, + target: str, + *, + location: Optional[str] = None, + buildroot: bool = False, + force: bool = False, + hardlinks: bool = False, + compression: str = "", + tar: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ): + + elements = self._load( + (target,), + selection=_PipelineSelection.NONE, + load_artifacts=True, + attempt_artifact_metadata=True, + connect_artifact_cache=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) + + # self.targets contains a list of the loaded target objects + # if we specify --deps build, Stream._load() will return a list + # of build dependency objects, however, we need to prepare a sandbox + # with the target (which has had its appropriate dependencies loaded) + element: Element = self.targets[0] + + self._check_location_writable(location, force=force, tar=tar) + + # Check whether the required elements are cached, and then + # try to pull them if they are not already cached. + # + self.query_cache(elements) + self._pull_missing_artifacts(elements) + + self._check_buildtree(element) + + try: + artifact = element._get_artifact() + virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree() + self._export_artifact(tar, location, compression, element, hardlinks, virdir) + except BstError as e: + raise StreamError( + "Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason + ) from e + # source_checkout() # # Checkout sources of the target element to the specified location @@ -1907,6 +1960,33 @@ def _check_location_writable(self, location, force=False, tar=False): if not force and os.path.exists(location): raise StreamError("Output file '{}' already exists".format(location)) + # _check_buildtree() + # + # Check if we require a pull queue attempt, with given artifact state and context. + # + # Args: + # element (Element): Destination path + # + # Raises: + # (StreamError): If the no buildroot found + # + def _check_buildtree(self, element): + if not element._cached_buildroot(): + if not element._cached(): + message = "Artifact not cached locally or in available remotes" + reason = "missing-buildtree-artifact-not-cached" + elif element._buildroot_exists(): + message = "Buildtree is not cached locally or in available remotes" + reason = "missing-buildtree-artifact-buildtree-not-cached" + else: + message = "Artifact was created without buildtree" + reason = "missing-buildtree-artifact-created-without-buildtree" + raise StreamError(message, reason=reason) + + # Raise warning if the element is cached in a failed state + if element._cached_failure(): + self._context.messenger.warn("using a buildtree from a failed build.") + # Helper function for source_checkout() def _source_checkout( self, diff --git a/tests/integration/buildtree.py b/tests/integration/buildtree.py new file mode 100644 index 000000000..dc84112d6 --- /dev/null +++ b/tests/integration/buildtree.py @@ -0,0 +1,133 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os +import tarfile +import shutil + +import pytest + +from buildstream._testing import cli, cli_integration, Cli # pylint: disable=unused-import +from buildstream.exceptions import ErrorDomain +from buildstream._testing._utils.site import HAVE_SANDBOX + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") + +# +# Verify fail cases when checkout buildtree +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_buildtree_checkout_fail(cli, datafiles): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(cli.directory, "checkout") + tar = os.path.join(cli.directory, "source-checkout.tar") + + res = cli.run(project=project, args=["--cache-buildtrees", "never", "build", element_name]) + res.assert_success() + + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-artifact-created-without-buildtree") + + res = cli.run(project=project, args=["buildtree", "checkout", "--compression", "gz", "--directory", checkout, element_name]) + assert res.exit_code != 0 + assert "ERROR: --compression can only be provided if --tar is provided" in res.stderr + + + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--directory", checkout, element_name]) + assert res.exit_code != 0 + assert "ERROR: options --directory and --tar conflict" in res.stderr + + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--hardlinks", element_name]) + assert res.exit_code != 0 + assert "ERROR: options --hardlinks and --tar conflict" in res.stderr + +# +# Verify checkout buildtree +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_buildtree_checkout(cli, datafiles): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(cli.directory, "checkout") + tar = os.path.join(cli.directory, "source-checkout.tar") + + # Build only once with cached buildtree + res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + # verify checkout buildtree only + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_success() + + expect_buildtree = ['test'] + assert expect_buildtree == os.listdir(checkout) + shutil.rmtree(checkout) + + # verify checkout buildtree only tar + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, element_name]) + res.assert_success() + + assert os.path.exists(tar) + with tarfile.open(tar) as tf: + expected_content = [os.path.join(".", expect) for expect in expect_buildtree] + tar_members = [f.name for f in tf] + assert tar_members == expected_content + os.remove(tar) + + # verify checkout buildtree only tar in different compression formats + for compression in ["gz", "xz", "bz2"]: + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", "-", "--compression", compression, element_name], binary_capture=True) + res.assert_success() + + with open(tar, "wb") as f: + f.write(res.output) + + with tarfile.open(tar, "r:" + compression) as tf: + expected_content = [os.path.join(".", expect) for expect in expect_buildtree] + tar_members = [f.name for f in tf] + assert tar_members == expected_content + + os.remove(tar) + + # verify checkout whole buildroot + res = cli.run(project=project, args=["buildtree", "checkout", "--buildroot", "--directory", checkout, element_name]) + res.assert_success() + + expect_buildroot = ['lib', 'media', 'proc', 'usr', 'home', 'buildstream-install', 'dev', 'var', 'sys', 'bin', 'run', 'buildstream', 'tmp', 'sbin', 'etc', 'mnt', 'srv', 'root'] + assert expect_buildroot == os.listdir(checkout) + + # verify checkout whole buildroot with force flag + res = cli.run(project=project, args=["buildtree", "checkout", "--force", "--directory", checkout, element_name]) + res.assert_success() + + assert sorted([*expect_buildtree, *expect_buildroot]) == sorted(os.listdir(checkout)) + + shutil.rmtree(checkout) + + # verify checkout whole buildroot with hardlinks + res = cli.run(project=project, args=["buildtree", "checkout", "--buildroot", "--hardlinks", "--directory", checkout, element_name]) + res.assert_success() + + expect_buildroot = ['lib', 'media', 'proc', 'usr', 'home', 'buildstream-install', 'dev', 'var', 'sys', 'bin', 'run', 'buildstream', 'tmp', 'sbin', 'etc', 'mnt', 'srv', 'root'] + assert expect_buildroot == os.listdir(checkout) + shutil.rmtree(checkout)