cli: add buildtree checkout command

src/buildstream/_frontend/cli.py

@@ -1687,3 +1687,124 @@ def artifact_delete(app, artifacts, deps):
     """Remove artifacts from the local cache"""
     with app.initialized():
         app.stream.artifact_delete(artifacts, selection=deps)
+
+
+#############################################################
+#                    Buildtree Commands                     #
+#############################################################
+@cli.group(short_help="Manipulate cached buildtree.")
+def buildtree():
+    """Manipulate cached buildtree"""
+
+
+#####################################################################
+#                    Buildtree Checkout Command                     #
+#####################################################################
+@buildtree.command(name="checkout", short_help="Checkout contents of an buildtree")
+@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.")
+@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten")
+@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible")
+@click.option(
+    "--tar",
+    default=None,
+    metavar="LOCATION",
+    type=click.Path(),
+    help="Create a tarball from the artifact contents instead "
+    "of a file tree. If LOCATION is '-', the tarball "
+    "will be dumped to the standard output.",
+)
+@click.option(
+    "--compression",
+    default=None,
+    type=click.Choice(["gz", "xz", "bz2"]),
+    help="The compression option of the tarball created.",
+)
+@click.option(
+    "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to"
+)
+@click.option(
+    "--artifact-remote",
+    "artifact_remotes",
+    type=RemoteSpecType(RemoteSpecPurpose.PULL),
+    multiple=True,
+    help="A remote for downloading artifacts",
+)
+@click.option(
+    "--ignore-project-artifact-remotes",
+    is_flag=True,
+    help="Ignore remote artifact cache servers recommended by projects",
+)
+@click.argument("target", required=False, type=click.Path(readable=False))
+@click.pass_obj
+def buildtree_checkout(
+    app,
+    buildroot,
+    force,
+    hardlinks,
+    tar,
+    compression,
+    directory,
+    artifact_remotes,
+    ignore_project_artifact_remotes,
+    target,
+):
+    """Checkout buildtree
+
+    When this command is executed from a workspace directory, the default
+    is to checkout the artifact of the workspace element.
+    """
+    from .. import utils
+
+    if hardlinks and tar:
+        click.echo("ERROR: options --hardlinks and --tar conflict", err=True)
+        sys.exit(-1)
+
+    if tar and directory:
+        click.echo("ERROR: options --directory and --tar conflict", err=True)
+        sys.exit(-1)
+
+    if not tar:
+        if compression:
+            click.echo("ERROR: --compression can only be provided if --tar is provided", err=True)
+            sys.exit(-1)
+    else:
+        location = tar
+        try:
+            inferred_compression = utils._get_compression(tar)
+        except UtilError as e:
+            click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True)
+            sys.exit(-1)
+        if compression and inferred_compression != "" and inferred_compression != compression:
+            click.echo(
+                "WARNING: File extension and compression differ."
+                "File extension has been overridden by --compression",
+                err=True,
+            )
+        if not compression:
+            compression = inferred_compression
+
+    with app.initialized():
+        if not target:
+            target = app.stream.get_default_target()
+            if not target:
+                raise AppError('Missing argument "ELEMENT".')
+
+        if not tar:
+            if directory is None:
+                location = os.path.abspath(os.path.join(os.getcwd(), target))
+                if location[-4:] == ".bst":
+                    location = location[:-4]
+            else:
+                location = directory
+
+        app.stream.buildtree_checkout(
+            target,
+            location=location,
+            buildroot=buildroot,
+            force=force,
+            hardlinks=hardlinks,
+            compression=compression,
+            tar=bool(tar),
+            artifact_remotes=artifact_remotes,
+            ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+        )

src/buildstream/_stream.py

@@ -333,23 +333,8 @@ def shell(
                     reason="shell-missing-deps",
                 )
 
-        # Check if we require a pull queue attempt, with given artifact state and context
         if usebuildtree:
-            if not element._cached_buildroot():
-                if not element._cached():
-                    message = "Artifact not cached locally or in available remotes"
-                    reason = "missing-buildtree-artifact-not-cached"
-                elif element._buildroot_exists():
-                    message = "Buildtree is not cached locally or in available remotes"
-                    reason = "missing-buildtree-artifact-buildtree-not-cached"
-                else:
-                    message = "Artifact was created without buildtree"
-                    reason = "missing-buildtree-artifact-created-without-buildtree"
-                raise StreamError(message, reason=reason)
-
-            # Raise warning if the element is cached in a failed state
-            if element._cached_failure():
-                self._context.messenger.warn("using a buildtree from a failed build.")
+            self._check_buildtree(element)
 
         # Ensure we have our sources if we are launching a build shell
         if scope == _Scope.BUILD and not usebuildtree:
@@ -916,6 +901,74 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE):
         if not ref_removed:
             self._context.messenger.info("No artifacts were removed")
 
+
+    # buildtree_checkout()
+    #
+    # Checkout target buildtree artifact to the specified location
+    #
+    # Args:
+    #    target: Target to checkout
+    #    location: Location to checkout the artifact to
+    #    force: Whether files can be overwritten if necessary
+    #    hardlinks: Whether checking out files hardlinked to
+    #               their artifacts is acceptable
+    #    tar: If true, a tarball from the artifact contents will
+    #         be created, otherwise the file tree of the artifact
+    #         will be placed at the given location. If true and
+    #         location is '-', the tarball will be dumped on the
+    #         standard output.
+    #    artifact_remotes: Artifact cache remotes specified on the commmand line
+    #    ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects
+    #
+    def buildtree_checkout(
+        self,
+        target: str,
+        *,
+        location: Optional[str] = None,
+        buildroot: bool = False,
+        force: bool = False,
+        hardlinks: bool = False,
+        compression: str = "",
+        tar: bool = False,
+        artifact_remotes: Iterable[RemoteSpec] = (),
+        ignore_project_artifact_remotes: bool = False,
+    ):
+
+        elements = self._load(
+            (target,),
+            selection=_PipelineSelection.NONE,
+            load_artifacts=True,
+            attempt_artifact_metadata=True,
+            connect_artifact_cache=True,
+            artifact_remotes=artifact_remotes,
+            ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+        )
+
+        # self.targets contains a list of the loaded target objects
+        # if we specify --deps build, Stream._load() will return a list
+        # of build dependency objects, however, we need to prepare a sandbox
+        # with the target (which has had its appropriate dependencies loaded)
+        element: Element = self.targets[0]
+
+        self._check_location_writable(location, force=force, tar=tar)
+
+        # Check whether the required elements are cached, and then
+        # try to pull them if they are not already cached.
+        #
+        self.query_cache(elements)
+        self._pull_missing_artifacts(elements)
+
+        self._check_buildtree(element)
+
+        try:
+            artifact = element._get_artifact()
+            virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree()
+            self._export_artifact(tar, location, compression, element, hardlinks, virdir)
+        except BstError as e:
+            raise StreamError(
+                "Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason
+            ) from e
+
     # source_checkout()
     #
     # Checkout sources of the target element to the specified location
@@ -1907,6 +1960,33 @@ def _check_location_writable(self, location, force=False, tar=False):
             if not force and os.path.exists(location):
                 raise StreamError("Output file '{}' already exists".format(location))
 
+    # _check_buildtree()
+    #
+    # Check if we require a pull queue attempt, with given artifact state and context.
+    #
+    # Args:
+    #    element (Element): Destination path
+    #
+    # Raises:
+    #    (StreamError): If the no buildroot found
+    #
+    def _check_buildtree(self, element):
+        if not element._cached_buildroot():
+            if not element._cached():
+                message = "Artifact not cached locally or in available remotes"
+                reason = "missing-buildtree-artifact-not-cached"
+            elif element._buildroot_exists():
+                message = "Buildtree is not cached locally or in available remotes"
+                reason = "missing-buildtree-artifact-buildtree-not-cached"
+            else:
+                message = "Artifact was created without buildtree"
+                reason = "missing-buildtree-artifact-created-without-buildtree"
+            raise StreamError(message, reason=reason)
+
+        # Raise warning if the element is cached in a failed state
+        if element._cached_failure():
+            self._context.messenger.warn("using a buildtree from a failed build.")
+
     # Helper function for source_checkout()
     def _source_checkout(
         self,

tests/integration/buildtree.py

@@ -0,0 +1,133 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+import tarfile
+import shutil
+
+import pytest
+
+from buildstream._testing import cli, cli_integration, Cli  # pylint: disable=unused-import
+from buildstream.exceptions import ErrorDomain
+from buildstream._testing._utils.site import HAVE_SANDBOX
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
+
+#
+# Verify fail cases when checkout buildtree
+#
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_buildtree_checkout_fail(cli, datafiles):
+    project = str(datafiles)
+    element_name = "build-shell/buildtree.bst"
+    checkout = os.path.join(cli.directory, "checkout")
+    tar = os.path.join(cli.directory, "source-checkout.tar")
+
+    res = cli.run(project=project, args=["--cache-buildtrees", "never", "build", element_name])
+    res.assert_success()
+
+    res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name])
+    res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-artifact-created-without-buildtree")
+
+    res = cli.run(project=project, args=["buildtree", "checkout", "--compression", "gz", "--directory", checkout, element_name])
+    assert res.exit_code != 0
+    assert "ERROR: --compression can only be provided if --tar is provided" in res.stderr
+
+
+    res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--directory", checkout, element_name])
+    assert res.exit_code != 0
+    assert "ERROR: options --directory and --tar conflict" in res.stderr
+
+    res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--hardlinks", element_name])
+    assert res.exit_code != 0
+    assert "ERROR: options --hardlinks and --tar conflict" in res.stderr
+
+#
+# Verify checkout buildtree
+#
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_buildtree_checkout(cli, datafiles):
+    project = str(datafiles)
+    element_name = "build-shell/buildtree.bst"
+    checkout = os.path.join(cli.directory, "checkout")
+    tar = os.path.join(cli.directory, "source-checkout.tar")
+
+    # Build only once with cached buildtree
+    res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name])
+    res.assert_success()
+
+    # verify checkout buildtree only
+    res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name])
+    res.assert_success()
+
+    expect_buildtree = ['test']
+    assert expect_buildtree == os.listdir(checkout)
+    shutil.rmtree(checkout)
+
+    # verify checkout buildtree only tar
+    res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, element_name])
+    res.assert_success()
+
+    assert os.path.exists(tar)
+    with tarfile.open(tar) as tf:
+        expected_content = [os.path.join(".", expect) for expect in expect_buildtree]
+        tar_members = [f.name for f in tf]
+        assert tar_members == expected_content
+    os.remove(tar)
+
+    # verify checkout buildtree only tar in different compression formats
+    for compression in ["gz", "xz", "bz2"]:
+        res = cli.run(project=project, args=["buildtree", "checkout", "--tar", "-", "--compression", compression, element_name], binary_capture=True)
+        res.assert_success()
+
+        with open(tar, "wb") as f:
+            f.write(res.output)
+
+        with tarfile.open(tar, "r:" + compression) as tf:
+            expected_content = [os.path.join(".", expect) for expect in expect_buildtree]
+            tar_members = [f.name for f in tf]
+            assert tar_members == expected_content
+
+        os.remove(tar)
+
+    # verify checkout whole buildroot
+    res = cli.run(project=project, args=["buildtree", "checkout", "--buildroot", "--directory", checkout, element_name])
+    res.assert_success()
+
+    expect_buildroot = ['lib', 'media', 'proc', 'usr', 'home', 'buildstream-install', 'dev', 'var', 'sys', 'bin', 'run', 'buildstream', 'tmp', 'sbin', 'etc', 'mnt', 'srv', 'root']
+    assert expect_buildroot == os.listdir(checkout)
+
+    # verify checkout whole buildroot with force flag
+    res = cli.run(project=project, args=["buildtree", "checkout", "--force", "--directory", checkout, element_name])
+    res.assert_success()
+
+    assert sorted([*expect_buildtree, *expect_buildroot]) == sorted(os.listdir(checkout))
+
+    shutil.rmtree(checkout)
+
+    # verify checkout whole buildroot with hardlinks
+    res = cli.run(project=project, args=["buildtree", "checkout", "--buildroot", "--hardlinks", "--directory", checkout, element_name])
+    res.assert_success()
+
+    expect_buildroot = ['lib', 'media', 'proc', 'usr', 'home', 'buildstream-install', 'dev', 'var', 'sys', 'bin', 'run', 'buildstream', 'tmp', 'sbin', 'etc', 'mnt', 'srv', 'root']
+    assert expect_buildroot == os.listdir(checkout)
+    shutil.rmtree(checkout)