diff --git a/README.md b/README.md index 12af7125..b3f17e6d 100644 --- a/README.md +++ b/README.md @@ -322,12 +322,13 @@ tasks: ### Includes -The `includes` key is used to import tasks from either local or remote task files. This is useful for sharing common tasks across multiple task files. When importing a task from a local task file, the path is relative to the file you are currently in. When running a task, the tasks in the task file as well as the `includes` get processed to ensure there are no infinite loop references. +The `includes` key is used to import tasks from local, remote, or OCI task files. This is useful for sharing common tasks across multiple task files. When importing a task from a local task file, the path is relative to the file you are currently in. When running a task, the tasks in the task file as well as the `includes` get processed to ensure there are no infinite loop references. ```yaml includes: - local: ./path/to/tasks-to-import.yaml - remote: https://raw.githubusercontent.com/defenseunicorns/maru-runner/main/src/test/tasks/remote-import-tasks.yaml + - common: oci://ghcr.io/myorg/tasks:latest tasks: - name: import-local @@ -336,6 +337,9 @@ tasks: - name: import-remote actions: - task: remote:echo-var + - name: import-oci + actions: + - task: common:hello-world ``` Note that included task files can also include other task files, with the following restriction: @@ -352,6 +356,47 @@ run import-local run local:some-local-task ``` +#### OCI Task Files + +Maru supports using OCI artifacts as task files. This allows you to store your tasks in container registries and version them using tags. + +**Pushing Tasks to OCI Registries** + +You can push your task files to OCI registries using the built-in `push` command: + +```bash +# Login to your registry first +gh auth token | maru auth login ghcr.io --token-stdin + +# Push the task file +maru push hello.yaml ghcr.io/myorg/maru-tasks:v1.0.0 + +# Push the task file (to an insecure registry) +maru push --insecure hello.yaml ghcr.io/myorg/maru-tasks:v1.0.0 +``` + +**Using OCI Tasks** + +To use an OCI task file, use the `oci://` prefix in your includes: + +```yaml +includes: + - common: oci://ghcr.io/myorg/tasks:v1.0.0 + +tasks: + - name: use-oci-task + actions: + - task: common:setup-env +``` + +Authentication to private OCI registries works the same way as for remote HTTPS task files, using the `maru auth login` command with the registry hostname. + +If the registry is insecure, you can use the `--insecure` flag when pushing the task file: + +```bash +maru run --insecure common:setup-env +``` + #### Authenticated Includes Some included remote task files may require authentication to access - to access these you can use the `maru auth login` command to add a personal access token (bearer auth) to your computer keychain. diff --git a/go.mod b/go.mod index e407c076..fc8e5d95 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,9 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/otiai10/copy v1.14.1 // indirect github.com/otiai10/mint v1.6.3 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect diff --git a/go.sum b/go.sum index 41429574..4147f583 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,12 @@ github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUt github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/otiai10/copy v1.14.1 h1:5/7E6qsUMBaH5AnQ0sSLzzTg1oTECmcCmT6lvF45Na8= github.com/otiai10/copy v1.14.1/go.mod h1:oQwrEDDOci3IM8dJF0d8+jnbfPDllW6vUjNc3DoZm9I= github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= diff --git a/src/cmd/push.go b/src/cmd/push.go new file mode 100644 index 00000000..0d180b66 --- /dev/null +++ b/src/cmd/push.go @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2023-Present the Maru Authors + +// Package cmd contains the CLI commands for maru. +package cmd + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/maru-runner/src/config" + "github.com/defenseunicorns/maru-runner/src/message" + v1 "github.com/opencontainers/image-spec/specs-go/v1" // ocispec + "github.com/spf13/cobra" + keyring "github.com/zalando/go-keyring" + oras "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" +) + +// OCI artifact media types +const ( + yamlMediaType = "application/yaml" + emptyConfigType = "application/vnd.oci.empty.v1+json" + defaultYamlArtifact = "application/vnd.oci.image.manifest.v1+json" +) + +var pushCmd = &cobra.Command{ + Use: "push TASK_FILE OCI_REFERENCE", + Short: "Push a task file to an OCI registry", + Long: `Push a Maru task file to an OCI registry. + +Examples: + # Push a task file to GitHub Container Registry + maru push tasks.yaml ghcr.io/myorg/maru-tasks:latest + + # Push a task file with a specific tag + maru push tasks.yaml ghcr.io/myorg/maru-tasks:v1.0.0 +`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + taskFile := args[0] + reference := args[1] + + return pushTaskFile(taskFile, reference) + }, +} + +var pushInsecure bool + +func init() { + initViper() + rootCmd.AddCommand(pushCmd) + pushCmd.Flags().BoolVar(&pushInsecure, "insecure", false, "Allow interaction with OCI registries that are not using HTTPS") +} + +// Parse an OCI reference into registry, repository, and tag +func parseOCIReference(reference string) (registry string, repository string, tag string, err error) { + // Verify reference starts with oci:// prefix + if !strings.HasPrefix(reference, "oci://") { + return "", "", "", fmt.Errorf("reference must start with 'oci://', got: %s", reference) + } + + // Remove oci:// prefix + reference = strings.TrimPrefix(reference, "oci://") + + // Format expected: registry/repo/path:tag + parts := strings.SplitN(reference, "/", 2) + if len(parts) < 2 { + return "", "", "", fmt.Errorf("invalid reference format: %s", reference) + } + + registry = parts[0] // e.g., ghcr.io + remainder := parts[1] // e.g., myorg/maru-tasks:0.0.1 + + // Split the remainder at the colon to get repo path and tag + repoAndTag := strings.SplitN(remainder, ":", 2) + repository = repoAndTag[0] // e.g., myorg/maru-tasks + tag = "latest" + if len(repoAndTag) > 1 { + tag = repoAndTag[1] // e.g., 0.0.1 + } + + return registry, repository, tag, nil +} + +// Push a task file to an OCI registry +func pushTaskFile(taskFilePath, reference string) error { + ctx := context.Background() + + // Verify the task file exists + if _, err := os.Stat(taskFilePath); os.IsNotExist(err) { + return fmt.Errorf("task file not found: %s", taskFilePath) + } + + // Create a temporary directory for the file store + tmpDir, err := os.MkdirTemp("", "maru-push-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Create a file store + fs, err := file.New(tmpDir) + if err != nil { + return fmt.Errorf("failed to create file store: %w", err) + } + defer fs.Close() + + // Parse the OCI reference + registry, repoPath, tag, err := parseOCIReference(reference) + if err != nil { + return err + } + + // Full repository reference + fullRepo := fmt.Sprintf("%s/%s", registry, repoPath) + message.SLog.Info(fmt.Sprintf("Pushing %s to %s:%s", taskFilePath, fullRepo, tag)) + + // Get absolute path of the task file (for reading) + taskFileAbs, err := filepath.Abs(taskFilePath) + if err != nil { + return fmt.Errorf("failed to get absolute path: %w", err) + } + // Use only the base name to avoid absolute path issues in the artifact. + taskFileName := filepath.Base(taskFilePath) + + // Create the empty config file (empty content to mimic /dev/null) + configFileName := "config.json" + configFileFull := filepath.Join(tmpDir, configFileName) + if err := os.WriteFile(configFileFull, []byte(""), 0644); err != nil { + return fmt.Errorf("failed to create config file: %w", err) + } + + // Add the empty config to the file store using a relative name. + configDesc, err := fs.Add(ctx, configFileName, emptyConfigType, configFileFull) + if err != nil { + return fmt.Errorf("failed to add config file: %w", err) + } + + // Add the task file to the file store with a relative name. + taskDesc, err := fs.Add(ctx, taskFileName, yamlMediaType, taskFileAbs) + if err != nil { + return fmt.Errorf("failed to add task file: %w", err) + } + + // Pack the files into a manifest + layers := []v1.Descriptor{taskDesc} + manifestDesc, err := oras.PackManifest(ctx, fs, oras.PackManifestVersion1_1, defaultYamlArtifact, oras.PackManifestOptions{ + Layers: layers, + ConfigDescriptor: &configDesc, + }) + if err != nil { + return fmt.Errorf("failed to pack manifest: %w", err) + } + + // Tag the manifest + err = fs.Tag(ctx, manifestDesc, tag) + if err != nil { + return fmt.Errorf("failed to tag manifest: %w", err) + } + + // Create a new repository client + repo, err := remote.NewRepository(fullRepo) + if err != nil { + return fmt.Errorf("failed to create repository client: %w", err) + } + + // Configure insecure mode if requested + if pushInsecure { + message.SLog.Info(fmt.Sprintf("Using insecure mode for %s", fullRepo)) + repo.PlainHTTP = true + } + + // Try to get token from keyring for the registry + token, err := keyring.Get(config.KeyringService, registry) + if err == nil && token != "" { + // Configure authentication + authClient := &auth.Client{ + Client: http.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(registry, auth.Credential{ + Username: "token", + Password: token, + }), + } + repo.Client = authClient + } else { + message.SLog.Debug(fmt.Sprintf("No authentication token found for %s", registry)) + message.SLog.Info(fmt.Sprintf("You may need to authenticate using 'maru auth login %s --token '", registry)) + } + + // Copy from the file store to the remote repository + _, err = oras.Copy(ctx, fs, tag, repo, tag, oras.DefaultCopyOptions) + if err != nil { + return fmt.Errorf("failed to push OCI artifact: %w", err) + } + + message.SLog.Info(fmt.Sprintf("Successfully pushed %s to %s", taskFilePath, reference)) + return nil +} diff --git a/src/cmd/run.go b/src/cmd/run.go index ebbe8e7d..844e3d6d 100644 --- a/src/cmd/run.go +++ b/src/cmd/run.go @@ -55,6 +55,9 @@ func (i *listFlag) Set(value string) error { // dryRun is a flag to only load / validate tasks without running commands var dryRun bool +// insecure is a flag to allow interaction with OCI registries that are not using HTTPS +var insecure bool + // setRunnerVariables provides a map of set variables from the command line var setRunnerVariables map[string]string @@ -96,6 +99,11 @@ var runCmd = &cobra.Command{ auth := v.GetStringMapString(V_AUTH) + // Pass insecure flag to the auth map for OCI operations + if insecure { + auth["insecure"] = "true" + } + listFormat := listTasks if listAllTasks != listOff { listFormat = listAllTasks @@ -202,6 +210,7 @@ func init() { runFlags := runCmd.Flags() runFlags.StringVarP(&config.TaskFileLocation, "file", "f", config.TasksYAML, lang.CmdRunFlag) runFlags.BoolVar(&dryRun, "dry-run", false, lang.CmdRunDryRun) + runFlags.BoolVar(&insecure, "insecure", false, "Allow interaction with OCI registries that are not using HTTPS") // Setup the --list flag flag.Var(&listTasks, "list", lang.CmdRunList) diff --git a/src/pkg/runner/runner.go b/src/pkg/runner/runner.go index 1bd6b8dd..69aeae0a 100644 --- a/src/pkg/runner/runner.go +++ b/src/pkg/runner/runner.go @@ -239,10 +239,24 @@ func includeTaskAbsLocation(currentFileLocation, includeFileLocation string) (st return absIncludeFileLocation, nil } -// LoadIncludeTask loads an included task file either from a remote or local file +// isOCIReference checks if a string is an OCI reference +func isOCIReference(reference string) bool { + return strings.HasPrefix(reference, "oci://") +} + +// LoadIncludeTask loads an included task file from a remote, OCI, or local file func LoadIncludeTask(currentFileLocation, includeFileLocation string, auth map[string]string) (string, types.TasksFile, error) { var includedTasksFile types.TasksFile + // Check if this is an OCI reference + if isOCIReference(includeFileLocation) { + // For OCI references, we use the full reference as the location identifier + ociReference := strings.TrimPrefix(includeFileLocation, "oci://") + err := utils.ReadOCIYaml(ociReference, &includedTasksFile, auth) + return includeFileLocation, includedTasksFile, err + } + + // Handle normal file or URL references absIncludeFileLocation, err := includeTaskAbsLocation(currentFileLocation, includeFileLocation) if err != nil { return absIncludeFileLocation, includedTasksFile, err diff --git a/src/pkg/utils/utils.go b/src/pkg/utils/utils.go index 9f786221..55a23c8a 100644 --- a/src/pkg/utils/utils.go +++ b/src/pkg/utils/utils.go @@ -5,12 +5,14 @@ package utils import ( + "context" "fmt" "io" "net/http" "net/url" "os" "path" + "path/filepath" "regexp" "strings" @@ -19,7 +21,11 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" goyaml "github.com/goccy/go-yaml" "github.com/pterm/pterm" - "github.com/zalando/go-keyring" + keyring "github.com/zalando/go-keyring" + oras "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" ) const ( @@ -203,3 +209,141 @@ func ReadRemoteYaml(location string, destConfig any, auth map[string]string) (er return nil } + +// ReadOCIYaml fetches a YAML file from an OCI registry and unmarshals it into the provided destination +func ReadOCIYaml(reference string, destConfig any, authMap map[string]string) error { + ctx := context.Background() + + // Create a temporary directory to store the YAML files + tmpDir, err := os.MkdirTemp("", "maru-oci-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer func() { + _ = os.RemoveAll(tmpDir) + }() + + message.SLog.Debug(fmt.Sprintf("Fetching OCI artifact: %s", reference)) + + // Remove the oci:// prefix if present + reference = strings.TrimPrefix(reference, "oci://") + + // Parse the reference to extract the registry, repository and tag + // Format expected: registry/repo/path:tag + + // First, get registry and the rest by splitting at first slash + parts := strings.SplitN(reference, "/", 2) + if len(parts) < 2 { + return fmt.Errorf("invalid reference format: %s", reference) + } + + registry := parts[0] // e.g., ghcr.io + remainder := parts[1] // e.g., myorg/maru-tasks/hello:0.0.1 + + // Split the remainder at the colon to get repo path and tag + repoAndTag := strings.SplitN(remainder, ":", 2) + repoPath := repoAndTag[0] // e.g., myorg/maru-tasks/hello + tag := "latest" + if len(repoAndTag) > 1 { + tag = repoAndTag[1] // e.g., 0.0.1 + } + + // Full repository reference that includes the registry + fullRepo := fmt.Sprintf("%s/%s", registry, repoPath) + + // Create a new repository client + remoteRepo, err := remote.NewRepository(fullRepo) + if err != nil { + return fmt.Errorf("failed to create repository client: %w", err) + } + + // Configure insecure mode if requested + if insecureStr, ok := authMap["insecure"]; ok && insecureStr == "true" { + message.SLog.Info(fmt.Sprintf("Using insecure mode for %s", fullRepo)) + remoteRepo.PlainHTTP = true + } + + // Add authentication if available in the auth map + if token, ok := authMap[registry]; ok { + authClient := &auth.Client{ + Client: http.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(registry, auth.Credential{ + Username: "token", + Password: token, + }), + } + remoteRepo.Client = authClient + } else { + // Try to get token from keyring + token, err := keyring.Get(config.KeyringService, registry) + if err == nil { + authClient := &auth.Client{ + Client: http.DefaultClient, + Cache: auth.NewCache(), + Credential: auth.StaticCredential(registry, auth.Credential{ + Username: "token", + Password: token, + }), + } + remoteRepo.Client = authClient + } + } + + // Create a file store for the output + fs, err := file.New(tmpDir) + if err != nil { + return fmt.Errorf("failed to create file store: %w", err) + } + defer func() { + _ = fs.Close() + }() + + // Log what we're about to pull + message.SLog.Debug(fmt.Sprintf("Pulling OCI artifact from repository %s with tag %s", fullRepo, tag)) + + // Copy the artifact to the file store + _, err = oras.Copy(ctx, remoteRepo, tag, fs, "", oras.DefaultCopyOptions) + if err != nil { + return fmt.Errorf("failed to pull OCI artifact: %w", err) + } + + message.SLog.Debug(fmt.Sprintf("Successfully fetched OCI artifact: %s", reference)) + + // Find the YAML file in the output directory + var yamlFiles []string + err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + yamlFiles = append(yamlFiles, path) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to find YAML files in output directory: %w", err) + } + + if len(yamlFiles) == 0 { + return fmt.Errorf("no YAML files found in the OCI artifact") + } + + // Use the first YAML file found + yamlFilePath := yamlFiles[0] + message.SLog.Debug(fmt.Sprintf("Using YAML file: %s", yamlFilePath)) + + // Read the YAML file + yamlData, err := os.ReadFile(yamlFilePath) + if err != nil { + return fmt.Errorf("failed to read YAML file: %w", err) + } + + // Unmarshal the YAML data + err = goyaml.Unmarshal(yamlData, destConfig) + if err != nil { + return fmt.Errorf("failed to unmarshal YAML data: %w", err) + } + + return nil +}