diff --git a/architecture/05-build-system.md b/architecture/05-build-system.md index 8a79198d75..0b8f0b9627 100644 --- a/architecture/05-build-system.md +++ b/architecture/05-build-system.md @@ -217,7 +217,7 @@ Resolution follows a 3-tier priority for each wheel: | 2. Auto-detect `dist/coglet-*.whl` | Dev builds only | | 3. Default | Install from PyPI | -Local wheel files are copied into `.cog/build/` and referenced via the `cog_build` named build context, then `COPY --from=cog_build`'d and `pip install`'d in the Dockerfile. +Local wheel files are copied into `.cog/build/` and referenced via the `cog_build` named build context, then `COPY --from=cog_build`'d and `pip install`'d in the Dockerfile. User-provided local wheel and source archive requirements use the same staging boundary: Cog copies only the referenced artifacts into `.cog/build/` before the requirements install, preserving the later `COPY . /src` source layer. --- diff --git a/docs/llms.txt b/docs/llms.txt index e62311b0d6..20aca719ce 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -3470,6 +3470,20 @@ Your `cog.yaml` file can set either `python_packages` or `python_requirements`, This follows the standard [requirements.txt](https://pip.pypa.io/en/stable/reference/requirements-file-format/) format. +Requirements files can also reference local Python package artifacts, such as wheels and source archives: + +`requirements.txt`: + +``` +./dist/mylib-0.1.0-py3-none-any.whl +./vendor/helperlib.zip +./packages/localpkg.tar.gz +``` + +Local artifact paths are resolved relative to the requirements file, and the referenced files must be inside your project directory. Cog stages these artifacts before installing requirements, so this is the supported way to install a local package artifact during `cog build`. + +Local package directories, `name @ file:...` requirements, local `--find-links` directories, recursive local artifact includes, and inline hashes or options on local artifact lines are not supported. + To install Git-hosted Python packages, add `git` to the `system_packages` list, then use the `git+https://` syntax to specify the package name. For example: `cog.yaml`: @@ -3547,7 +3561,7 @@ build: - cd cowsay-3.7.0 && make install ``` -Your code is _not_ available to commands in `run`. This is so we can build your image efficiently when running locally. +Your code is _not_ available to commands in `run`. This is so we can build your image efficiently when running locally. To install a local wheel or source archive, list it in your `python_requirements` file instead of running `pip install ./artifact.zip` from `run`. Each command in `run` can be either a string or a dictionary in the following format: diff --git a/docs/yaml.md b/docs/yaml.md index e7d98b80e1..f8eb942165 100644 --- a/docs/yaml.md +++ b/docs/yaml.md @@ -59,6 +59,20 @@ Your `cog.yaml` file can set either `python_packages` or `python_requirements`, This follows the standard [requirements.txt](https://pip.pypa.io/en/stable/reference/requirements-file-format/) format. +Requirements files can also reference local Python package artifacts, such as wheels and source archives: + +`requirements.txt`: + +``` +./dist/mylib-0.1.0-py3-none-any.whl +./vendor/helperlib.zip +./packages/localpkg.tar.gz +``` + +Local artifact paths are resolved relative to the requirements file, and the referenced files must be inside your project directory. Cog stages these artifacts before installing requirements, so this is the supported way to install a local package artifact during `cog build`. + +Local package directories, `name @ file:...` requirements, local `--find-links` directories, recursive local artifact includes, and inline hashes or options on local artifact lines are not supported. + To install Git-hosted Python packages, add `git` to the `system_packages` list, then use the `git+https://` syntax to specify the package name. For example: `cog.yaml`: @@ -136,7 +150,7 @@ build: - cd cowsay-3.7.0 && make install ``` -Your code is _not_ available to commands in `run`. This is so we can build your image efficiently when running locally. +Your code is _not_ available to commands in `run`. This is so we can build your image efficiently when running locally. To install a local wheel or source archive, list it in your `python_requirements` file instead of running `pip install ./artifact.zip` from `run`. Each command in `run` can be either a string or a dictionary in the following format: diff --git a/integration-tests/tests/local_python_requirement_artifact.txtar b/integration-tests/tests/local_python_requirement_artifact.txtar new file mode 100644 index 0000000000..089d5083f9 --- /dev/null +++ b/integration-tests/tests/local_python_requirement_artifact.txtar @@ -0,0 +1,34 @@ +# Local package artifacts listed in requirements.txt are available during build. + +mkdir dist +exec python3 -c 'import zipfile; z = zipfile.ZipFile("dist/local_pkg-0.1.0.zip", "w"); z.write("setup.py"); z.write("local_pkg/__init__.py"); z.close()' + +cog build -t $TEST_IMAGE +cog predict $TEST_IMAGE +stdout 'hello from local package' + +-- cog.yaml -- +build: + python_version: "3.12" + python_requirements: requirements.txt +predict: predict.py:Predictor + +-- requirements.txt -- +./dist/local_pkg-0.1.0.zip + +-- setup.py -- +from setuptools import setup + +setup(name="local-pkg", version="0.1.0", packages=["local_pkg"]) + +-- local_pkg/__init__.py -- +MESSAGE = "hello from local package" + +-- predict.py -- +from cog import BasePredictor +from local_pkg import MESSAGE + + +class Predictor(BasePredictor): + def predict(self) -> str: + return MESSAGE diff --git a/pkg/config/config.go b/pkg/config/config.go index 621b4bf49f..db62a9482e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,8 +1,11 @@ package config import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" + "os" "path/filepath" "regexp" "slices" @@ -60,6 +63,14 @@ type Build struct { SDKVersion string `json:"sdk_version,omitempty" yaml:"sdk_version,omitempty"` pythonRequirementsContent []string + localPackageArtifacts []LocalPackageArtifact +} + +type LocalPackageArtifact struct { + Requirement string + SourcePath string + StagedDir string + Filename string } type Concurrency struct { @@ -277,6 +288,9 @@ func (c *Config) Complete(projectDir string) error { return fmt.Errorf("failed to open python_requirements file: %w", err) } c.Build.pythonRequirementsContent = reqs + if err := c.loadLocalPackageArtifacts(projectDir, requirementsFilePath); err != nil { + return err + } } else if len(c.Build.PythonPackages) > 0 { // Backwards compatibility: if using deprecated python_packages, populate requirements content c.Build.pythonRequirementsContent = c.Build.PythonPackages @@ -297,6 +311,80 @@ func (c *Config) Complete(projectDir string) error { return nil } +func (c *Config) loadLocalPackageArtifacts(projectDir string, requirementsFilePath string) error { + projectRoot, err := filepath.Abs(projectDir) + if err != nil { + return fmt.Errorf("failed to resolve project directory: %w", err) + } + projectRoot, err = filepath.EvalSymlinks(projectRoot) + if err != nil { + return fmt.Errorf("failed to resolve project directory symlinks: %w", err) + } + + requirementsDir := filepath.Dir(requirementsFilePath) + seen := map[string]LocalPackageArtifact{} + artifacts := []LocalPackageArtifact{} + for _, line := range c.Build.pythonRequirementsContent { + artifactPath, ok, err := requirements.ParseLocalArtifactRequirement(line) + if err != nil { + return err + } + if !ok { + continue + } + + resolvedPath := artifactPath + if !filepath.IsAbs(resolvedPath) { + resolvedPath = filepath.Join(requirementsDir, resolvedPath) + } + absPath, err := filepath.Abs(resolvedPath) + if err != nil { + return fmt.Errorf("failed to resolve local Python package artifact %q: %w", artifactPath, err) + } + canonicalPath, err := filepath.EvalSymlinks(absPath) + if err != nil { + return fmt.Errorf("local Python package artifact %q not found: %w", artifactPath, err) + } + info, err := os.Stat(canonicalPath) + if err != nil { + return fmt.Errorf("failed to inspect local Python package artifact %q: %w", artifactPath, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("local Python package artifact %q must be a regular file", artifactPath) + } + if !pathWithin(projectRoot, canonicalPath) { + return fmt.Errorf("local Python package artifact %q must be inside the project directory", artifactPath) + } + + if artifact, ok := seen[canonicalPath]; ok { + artifact.Requirement = line + artifacts = append(artifacts, artifact) + continue + } + + relPath, err := filepath.Rel(projectRoot, canonicalPath) + if err != nil { + return fmt.Errorf("failed to resolve local Python package artifact %q relative to project directory: %w", artifactPath, err) + } + hash := sha256.Sum256([]byte(filepath.ToSlash(relPath))) + artifact := LocalPackageArtifact{ + Requirement: line, + SourcePath: canonicalPath, + StagedDir: hex.EncodeToString(hash[:])[:16], + Filename: filepath.Base(absPath), + } + seen[canonicalPath] = artifact + artifacts = append(artifacts, artifact) + } + c.Build.localPackageArtifacts = artifacts + return nil +} + +func pathWithin(root string, target string) bool { + rel, err := filepath.Rel(root, target) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + // PythonRequirementsForArch returns a requirements.txt file with all the GPU packages resolved for given OS and architecture. func (c *Config) PythonRequirementsForArch(goos string, goarch string, includePackages []string) (string, error) { packages := []string{} @@ -546,6 +634,13 @@ func (c *Config) RequirementsFile(projectDir string) string { return filepath.Join(projectDir, c.Build.PythonRequirements) } +func (c *Config) LocalPackageArtifacts() []LocalPackageArtifact { + if c.Build == nil { + return nil + } + return slices.Clone(c.Build.localPackageArtifacts) +} + func (c *Config) ParsedEnvironment() map[string]string { return c.parsedEnvironment } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 49164e087e..0494cf3595 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -196,6 +196,152 @@ flask>0.4 } +func TestPythonRequirementsLocalPackageArtifacts(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.MkdirAll(path.Join(tmpDir, "requirements", "dist"), 0o755)) + wheelPath := path.Join(tmpDir, "requirements", "dist", "local_pkg-0.1.0-py3-none-any.whl") + archivePath := path.Join(tmpDir, "requirements", "mylibpackage.zip") + require.NoError(t, os.WriteFile(wheelPath, []byte("wheel"), 0o644)) + require.NoError(t, os.WriteFile(archivePath, []byte("zip"), 0o644)) + require.NoError(t, os.WriteFile(path.Join(tmpDir, "requirements", "requirements.txt"), []byte(`torch==1.13.1 +./dist/local_pkg-0.1.0-py3-none-any.whl +mylibpackage.zip`), 0o644)) + + config := &Config{ + Build: &Build{ + GPU: true, + PythonVersion: "3.10", + PythonRequirements: "requirements/requirements.txt", + }, + } + require.NoError(t, config.Complete(tmpDir)) + + artifacts := config.LocalPackageArtifacts() + canonicalWheelPath, err := filepath.EvalSymlinks(wheelPath) + require.NoError(t, err) + canonicalArchivePath, err := filepath.EvalSymlinks(archivePath) + require.NoError(t, err) + require.Len(t, artifacts, 2) + require.Equal(t, "./dist/local_pkg-0.1.0-py3-none-any.whl", artifacts[0].Requirement) + require.Equal(t, canonicalWheelPath, artifacts[0].SourcePath) + require.Equal(t, "local_pkg-0.1.0-py3-none-any.whl", artifacts[0].Filename) + require.Len(t, artifacts[0].StagedDir, 16) + require.Equal(t, "mylibpackage.zip", artifacts[1].Requirement) + require.Equal(t, canonicalArchivePath, artifacts[1].SourcePath) + + requirements, err := config.PythonRequirementsForArch("linux", "amd64", []string{}) + require.NoError(t, err) + require.Equal(t, `--extra-index-url https://download.pytorch.org/whl/cu117/ +torch==1.13.1 +./dist/local_pkg-0.1.0-py3-none-any.whl +mylibpackage.zip`, requirements) +} + +func TestPythonRequirementsLocalPackageArtifactDuplicateAliases(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.MkdirAll(path.Join(tmpDir, "dist"), 0o755)) + wheelPath := path.Join(tmpDir, "dist", "local_pkg-0.1.0-py3-none-any.whl") + require.NoError(t, os.WriteFile(wheelPath, []byte("wheel"), 0o644)) + require.NoError(t, os.WriteFile(path.Join(tmpDir, "requirements.txt"), []byte("./dist/local_pkg-0.1.0-py3-none-any.whl\ndist/local_pkg-0.1.0-py3-none-any.whl"), 0o644)) + + config := &Config{Build: &Build{PythonVersion: "3.10", PythonRequirements: "requirements.txt"}} + require.NoError(t, config.Complete(tmpDir)) + + artifacts := config.LocalPackageArtifacts() + require.Len(t, artifacts, 2) + require.Equal(t, "./dist/local_pkg-0.1.0-py3-none-any.whl", artifacts[0].Requirement) + require.Equal(t, "dist/local_pkg-0.1.0-py3-none-any.whl", artifacts[1].Requirement) + require.Equal(t, artifacts[0].SourcePath, artifacts[1].SourcePath) + require.Equal(t, artifacts[0].StagedDir, artifacts[1].StagedDir) + require.Equal(t, artifacts[0].Filename, artifacts[1].Filename) +} + +func TestPythonRequirementsLocalPackageArtifactUnsupportedLocalOptions(t *testing.T) { + for _, line := range []string{ + "--find-links ./wheels", + "--find-links=./wheels", + "-r requirements-local.txt", + "--requirement requirements-local.txt", + } { + t.Run(line, func(t *testing.T) { + projectDir := t.TempDir() + require.NoError(t, os.WriteFile(path.Join(projectDir, "requirements.txt"), []byte(line), 0o644)) + config := &Config{Build: &Build{PythonVersion: "3.10", PythonRequirements: "requirements.txt"}} + err := config.Complete(projectDir) + require.Error(t, err) + require.Contains(t, err.Error(), "local requirements option") + }) + } +} + +func TestPythonRequirementsLocalPackageArtifactValidation(t *testing.T) { + testCases := []struct { + name string + line string + setup func(t *testing.T, projectDir string) string + expectedErr string + }{ + { + name: "MissingArtifact", + line: "./missing.zip", + expectedErr: "not found", + }, + { + name: "DirectoryArtifact", + line: "./pkg.zip", + setup: func(t *testing.T, projectDir string) string { + require.NoError(t, os.Mkdir(path.Join(projectDir, "pkg.zip"), 0o755)) + return "" + }, + expectedErr: "regular file", + }, + { + name: "OutsideProject", + setup: func(t *testing.T, projectDir string) string { + outsideDir := t.TempDir() + outsidePath := path.Join(outsideDir, "pkg.zip") + require.NoError(t, os.WriteFile(outsidePath, []byte("zip"), 0o644)) + return outsidePath + }, + expectedErr: "inside the project directory", + }, + { + name: "SymlinkOutsideProject", + line: "./pkg.zip", + setup: func(t *testing.T, projectDir string) string { + outsideDir := t.TempDir() + outsidePath := path.Join(outsideDir, "pkg.zip") + require.NoError(t, os.WriteFile(outsidePath, []byte("zip"), 0o644)) + require.NoError(t, os.Symlink(outsidePath, path.Join(projectDir, "pkg.zip"))) + return "" + }, + expectedErr: "inside the project directory", + }, + { + name: "InlineHashUnsupported", + line: "./pkg.whl --hash=sha256:abc", + expectedErr: "inline options or hashes", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + projectDir := t.TempDir() + line := tc.line + if tc.setup != nil { + if setupLine := tc.setup(t, projectDir); setupLine != "" { + line = setupLine + } + } + require.NoError(t, os.WriteFile(path.Join(projectDir, "requirements.txt"), []byte(line), 0o644)) + config := &Config{Build: &Build{PythonVersion: "3.10", PythonRequirements: "requirements.txt"}} + err := config.Complete(projectDir) + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedErr) + }) + } +} + func TestValidateAndCompleteCUDAForAllTF(t *testing.T) { for _, compat := range TFCompatibilityMatrix { config := &Config{ diff --git a/pkg/config/data/config_schema_v1.0.json b/pkg/config/data/config_schema_v1.0.json index 27de32d113..9259c49b03 100644 --- a/pkg/config/data/config_schema_v1.0.json +++ b/pkg/config/data/config_schema_v1.0.json @@ -74,7 +74,7 @@ "python_requirements": { "$id": "#/properties/build/properties/python_requirements", "type": "string", - "description": "A pip requirements file specifying the Python packages to install." + "description": "A pip requirements file specifying the Python packages to install. Local wheel and source archive paths in the requirements file are supported when they point to files inside the project." }, "system_packages": { "$id": "#/properties/build/properties/system_packages", diff --git a/pkg/dockerfile/standard_generator.go b/pkg/dockerfile/standard_generator.go index 6282d42fb1..4613f8a6ab 100644 --- a/pkg/dockerfile/standard_generator.go +++ b/pkg/dockerfile/standard_generator.go @@ -14,6 +14,7 @@ import ( "github.com/replicate/cog/pkg/registry" "github.com/replicate/cog/pkg/requirements" "github.com/replicate/cog/pkg/util/console" + "github.com/replicate/cog/pkg/util/files" "github.com/replicate/cog/pkg/util/version" "github.com/replicate/cog/pkg/weightslegacy" "github.com/replicate/cog/pkg/wheels" @@ -902,6 +903,11 @@ func (g *StandardGenerator) pipInstalls() (string, error) { // Strip cog/coglet from user requirements — we always install them ourselves // via installCog(). Leaving them in would cause pip to overwrite our version. g.pythonRequirementsContents = g.filterManagedPackages(g.pythonRequirementsContents) + artifactCopyLines, err := g.stageLocalPackageArtifacts() + if err != nil { + return "", err + } + g.pythonRequirementsContents = g.rewriteLocalPackageArtifacts(g.pythonRequirementsContents) if strings.Trim(g.pythonRequirementsContents, "") == "" { return "", nil @@ -917,12 +923,55 @@ func (g *StandardGenerator) pipInstalls() (string, error) { if g.strip { pipInstallLine += " && " + StripDebugSymbolsCommand } - return strings.Join([]string{ + lines := []string{} + lines = append(lines, artifactCopyLines...) + lines = append(lines, copyLine[0], CFlags, pipInstallLine, "ENV CFLAGS=", - }, "\n"), nil + ) + return strings.Join(lines, "\n"), nil +} + +func (g *StandardGenerator) stageLocalPackageArtifacts() ([]string, error) { + artifacts := g.Config.LocalPackageArtifacts() + if len(artifacts) == 0 { + return nil, nil + } + staged := map[string]bool{} + for _, artifact := range artifacts { + dst := filepath.Join(g.tmpDir, "local_package_artifacts", artifact.StagedDir, artifact.Filename) + if staged[dst] { + continue + } + if err := files.Copy(artifact.SourcePath, dst); err != nil { + return nil, fmt.Errorf("failed to stage local Python package artifact %s: %w", artifact.SourcePath, err) + } + staged[dst] = true + } + return []string{"COPY --from=cog_build local_package_artifacts/ /tmp/local_package_artifacts/"}, nil +} + +func (g *StandardGenerator) rewriteLocalPackageArtifacts(reqContents string) string { + artifacts := map[string]string{} + for _, artifact := range g.Config.LocalPackageArtifacts() { + containerPath := path.Join("/tmp/local_package_artifacts", artifact.StagedDir, artifact.Filename) + artifacts[artifact.Requirement] = containerPath + } + if len(artifacts) == 0 { + return reqContents + } + + lines := []string{} + for line := range strings.SplitSeq(reqContents, "\n") { + if replacement, ok := artifacts[strings.TrimSpace(line)]; ok { + lines = append(lines, replacement) + } else { + lines = append(lines, line) + } + } + return strings.Join(lines, "\n") } func (g *StandardGenerator) runCommands() (string, error) { diff --git a/pkg/dockerfile/standard_generator_test.go b/pkg/dockerfile/standard_generator_test.go index dc7e8c8b82..d4f7082189 100644 --- a/pkg/dockerfile/standard_generator_test.go +++ b/pkg/dockerfile/standard_generator_test.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "strings" "testing" "time" @@ -338,6 +339,47 @@ build: require.Contains(t, actual, `uv run pip install --cache-dir /root/.cache/pip -r /tmp/requirements.txt`) } +func TestPythonRequirementsLocalPackageArtifact(t *testing.T) { + tmpDir := t.TempDir() + require.NoError(t, os.MkdirAll(path.Join(tmpDir, "requirements", "dist"), 0o755)) + artifactPath := path.Join(tmpDir, "requirements", "dist", "local_pkg-0.1.0-py3-none-any.whl") + require.NoError(t, os.WriteFile(artifactPath, []byte("wheel"), 0o644)) + require.NoError(t, os.WriteFile(path.Join(tmpDir, "requirements", "requirements.txt"), []byte("./dist/local_pkg-0.1.0-py3-none-any.whl\ndist/local_pkg-0.1.0-py3-none-any.whl"), 0o644)) + + conf, err := config.FromYAML([]byte(` +build: + python_version: "3.12" + python_requirements: "requirements/requirements.txt" +`)) + require.NoError(t, err) + require.NoError(t, conf.Complete(tmpDir)) + command := dockertest.NewMockCommand() + client := registrytest.NewMockRegistryClient() + buildDir := t.TempDir() + gen, err := NewStandardGenerator(conf, tmpDir, buildDir, "", command, client, true) + require.NoError(t, err) + gen.SetUseCogBaseImage(false) + pypiWheels(gen) + _, actual, _, err := gen.GenerateModelBaseWithSeparateWeights(t.Context(), "r8.im/replicate/cog-test") + require.NoError(t, err) + + artifact := conf.LocalPackageArtifacts()[0] + copyArtifacts := "COPY --from=cog_build local_package_artifacts/ /tmp/local_package_artifacts/" + copyRequirements := "COPY --from=cog_build requirements.txt /tmp/requirements.txt" + pipInstall := "uv run pip install --cache-dir /root/.cache/pip -r /tmp/requirements.txt" + require.Contains(t, actual, copyArtifacts) + require.Less(t, strings.Index(actual, copyArtifacts), strings.Index(actual, copyRequirements)) + require.Less(t, strings.Index(actual, copyRequirements), strings.Index(actual, pipInstall)) + + stagedArtifact, err := os.ReadFile(path.Join(buildDir, "local_package_artifacts", artifact.StagedDir, artifact.Filename)) + require.NoError(t, err) + require.Equal(t, []byte("wheel"), stagedArtifact) + + requirements, err := os.ReadFile(path.Join(buildDir, "requirements.txt")) + require.NoError(t, err) + require.Equal(t, "/tmp/local_package_artifacts/"+artifact.StagedDir+"/local_pkg-0.1.0-py3-none-any.whl\n/tmp/local_package_artifacts/"+artifact.StagedDir+"/local_pkg-0.1.0-py3-none-any.whl", string(requirements)) +} + // GPU builds on nvidia/cuda base images install Python via `uv python install` // (in installPythonCUDA), which marks it as externally managed (PEP 668). All // pip install lines must include --break-system-packages. diff --git a/pkg/requirements/local_artifact.go b/pkg/requirements/local_artifact.go new file mode 100644 index 0000000000..4da4388e03 --- /dev/null +++ b/pkg/requirements/local_artifact.go @@ -0,0 +1,109 @@ +package requirements + +import ( + "fmt" + "path/filepath" + "strings" +) + +var localArtifactSuffixes = []string{ + ".whl", + ".zip", + ".tar.gz", + ".tgz", + ".tar.bz2", + ".tar.xz", +} + +// ParseLocalArtifactRequirement identifies simple local wheel/source-archive +// requirement lines. It intentionally does not parse full pip requirement +// syntax; callers should reject unsupported local forms with clear errors. +func ParseLocalArtifactRequirement(line string) (string, bool, error) { + line = strings.TrimSpace(line) + if line == "" { + return "", false, nil + } + + if strings.HasPrefix(line, "file:") || strings.Contains(line, " @ file:") { + return "", false, fmt.Errorf("local file URL requirements are not supported: %s", line) + } + if option, ok := parseUnsupportedLocalOption(line); ok { + return "", false, fmt.Errorf("local requirements option %q is not supported: %s", option, line) + } + if strings.HasPrefix(line, "-") || isRemoteRequirement(line) { + return "", false, nil + } + + fields := strings.Fields(line) + if len(fields) > 1 { + if hasLocalArtifactSuffix(fields[0]) || isLocalPath(fields[0]) { + return "", false, fmt.Errorf("local package artifact requirements do not support inline options or hashes: %s", line) + } + return "", false, nil + } + + if !isLocalPath(line) && !hasLocalArtifactSuffix(line) { + return "", false, nil + } + if !hasLocalArtifactSuffix(line) { + return "", false, fmt.Errorf("local package requirement %q is not a supported wheel or source archive", line) + } + + return line, true, nil +} + +func parseUnsupportedLocalOption(line string) (string, bool) { + for _, option := range []string{"--find-links", "--requirement"} { + if value, ok := optionValue(line, option); ok && isLocalOptionValue(value) { + return option, true + } + } + for _, option := range []string{"-f", "-r"} { + if value, ok := shortOptionValue(line, option); ok && isLocalOptionValue(value) { + return option, true + } + } + return "", false +} + +func optionValue(line string, option string) (string, bool) { + if value, ok := strings.CutPrefix(line, option+"="); ok { + return strings.TrimSpace(value), true + } + if value, ok := strings.CutPrefix(line, option+" "); ok { + return strings.TrimSpace(value), true + } + return "", false +} + +func shortOptionValue(line string, option string) (string, bool) { + if value, ok := strings.CutPrefix(line, option+" "); ok { + return strings.TrimSpace(value), true + } + return "", false +} + +func isLocalOptionValue(value string) bool { + if value == "" || isRemoteRequirement(value) || strings.HasPrefix(value, "file:") { + return false + } + return true +} + +func isRemoteRequirement(line string) bool { + return strings.Contains(line, "://") || strings.HasPrefix(line, "git+") +} + +func isLocalPath(line string) bool { + return filepath.IsAbs(line) || strings.HasPrefix(line, "./") || strings.HasPrefix(line, "../") +} + +func hasLocalArtifactSuffix(path string) bool { + path = strings.ToLower(path) + for _, suffix := range localArtifactSuffixes { + if strings.HasSuffix(path, suffix) { + return true + } + } + return false +} diff --git a/pkg/requirements/requirements_test.go b/pkg/requirements/requirements_test.go index dabbfe9de3..17c0f200d9 100644 --- a/pkg/requirements/requirements_test.go +++ b/pkg/requirements/requirements_test.go @@ -20,6 +20,46 @@ func TestReadRequirements(t *testing.T) { require.Equal(t, []string{"torch==2.5.1"}, requirements) } +func TestParseLocalArtifactRequirement(t *testing.T) { + testCases := []struct { + name string + line string + expected string + expectedOK bool + expectsErr bool + }{ + {name: "Wheel", line: "./dist/pkg-0.1.0-py3-none-any.whl", expected: "./dist/pkg-0.1.0-py3-none-any.whl", expectedOK: true}, + {name: "Zip", line: "mylibpackage.zip", expected: "mylibpackage.zip", expectedOK: true}, + {name: "TarGz", line: "../pkg-0.1.0.tar.gz", expected: "../pkg-0.1.0.tar.gz", expectedOK: true}, + {name: "TarBz2", line: "/tmp/pkg-0.1.0.tar.bz2", expected: "/tmp/pkg-0.1.0.tar.bz2", expectedOK: true}, + {name: "Package", line: "torch==2.5.1"}, + {name: "URL", line: "https://example.com/pkg.zip"}, + {name: "VCS", line: "git+https://example.com/repo.git"}, + {name: "FileURL", line: "name @ file:./pkg.whl", expectsErr: true}, + {name: "DirectFileURL", line: "file:///tmp/pkg.whl", expectsErr: true}, + {name: "NamedFileURL", line: "name @ file:///tmp/pkg.whl", expectsErr: true}, + {name: "LocalFindLinks", line: "--find-links ./wheels", expectsErr: true}, + {name: "LocalFindLinksEquals", line: "--find-links=./wheels", expectsErr: true}, + {name: "RemoteFindLinks", line: "--find-links https://example.com/wheels"}, + {name: "LocalRecursiveRequirement", line: "-r requirements-local.txt", expectsErr: true}, + {name: "InlineHash", line: "./pkg.whl --hash=sha256:abc", expectsErr: true}, + {name: "LocalDirectory", line: "./pkg", expectsErr: true}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual, ok, err := ParseLocalArtifactRequirement(tc.line) + if tc.expectsErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.expectedOK, ok) + require.Equal(t, tc.expected, actual) + }) + } +} + func TestReadRequirementsLineContinuations(t *testing.T) { srcDir := t.TempDir() reqFile := path.Join(srcDir, "requirements.txt")