diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..fc7ad27 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,86 @@ +# Copilot Instructions for `goark/ml` + +## Project purpose + +`ml` is a CLI and library that fetches metadata from web pages and formats links +as markdown/wiki/html/csv/json output. + +## Design principles + +- Keep CLI behavior simple and deterministic. +- Keep output formatting stable for all supported styles. +- Reuse `github.com/goark/webinfo` for metadata extraction. +- Preserve compatibility of exported symbols when possible. + +## Error handling + +- Use `github.com/goark/errs` for internal error handling. +- Prefer `errs.Wrap`, `errs.Join`, and `errs.WithContext`. +- Keep `errors.Is` compatibility for callers. +- Keep sentinel errors stable (`ErrNullPointer`, `ErrNoImplement`, `ErrInvalidRequest`). +- Include useful context keys such as `url`, `style`, and `path`. + +## Fetch behavior + +- Fetch and metadata parsing are delegated to `github.com/goark/webinfo`. +- Keep `makelink.New` mapping behavior stable: + - `Link.URL <- Webinfo.URL` + - `Link.Location <- Webinfo.Location` + - `Link.Canonical <- Webinfo.Canonical` + - `Link.Title <- Webinfo.Title` + - `Link.Description <- Webinfo.Description` +- Preserve URL fallback behavior for output rendering: + - title fallback: `Title` -> `URL` + - URL fallback: `Canonical` -> `Location` -> `URL` + +## Output style behavior + +- Keep style names and output formats stable: + - `markdown`: `[title](url)` + - `wiki`: `[url title]` + - `html`: `title` + - `csv`: escaped CSV fields in fixed order + - `json`: JSON encoded `Link` +- Keep CSV quote escaping behavior unchanged. + +## Coding style + +- Write idiomatic Go with straightforward control flow. +- Avoid unnecessary dependencies. +- Keep comments concise and in English. + +## Testing and validation + +- Add or update tests for behavior changes. +- Prefer local validation with Taskfile targets: + - `task test` + - `task govulncheck` + +## Documentation + +- Keep `README.md` aligned with actual CLI and exported API behavior. +- Keep examples concise and runnable. + +## Release process + +- Create release tags from `master`. +- Use semantic versioning tags in `vMAJOR.MINOR.PATCH` format. +- Ensure repository is clean and synced before tagging. + +Release steps: + +1. Ensure `master` is up to date. +2. Create annotated tag: + - `git tag -a vX.Y.Z -m "Release vX.Y.Z"` +3. Push tag: + - `git push origin vX.Y.Z` +4. Wait for the `build` workflow triggered by the tag push to complete. + +Verification steps: + +- Check tag exists: + - `git tag -l "vX.Y.Z"` +- Check `build` workflow result for the tag run (success/failure). +- Check release exists: + - `gh release view vX.Y.Z` +- Check generated release notes are present in the release page. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..dc0d6f1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: ci + +on: + push: + branches: + - master + pull_request: + +permissions: + contents: read + +jobs: + test-and-lint: + name: lint and test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + + - name: golangci-lint + uses: golangci/golangci-lint-action@v9 + with: + version: latest + args: --enable gosec + + - name: Test module + run: go test -shuffle on ./... + + govulncheck: + name: govulncheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Run govulncheck + uses: golang/govulncheck-action@v1 + with: + go-version-file: go.mod + go-package: ./... + repo-checkout: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index c189f79..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,58 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -name: "CodeQL" - -on: - push: - branches: [master] - pull_request: - # The branches below must be a subset of the branches above - branches: [master] - schedule: - - cron: '0 20 * * 0' - -jobs: - CodeQL-Build: - # CodeQL runs on ubuntu-latest, windows-latest, and macos-latest - runs-on: ubuntu-latest - - permissions: - # required for all workflows - security-events: write - - # only required for workflows in private repositories - actions: read - contents: read - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - # Override language selection by uncommenting this and choosing your languages - with: - languages: go - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below). - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # 鈩癸笍 Command-line programs to run using the OS shell. - # 馃摎 https://git.io/JvXDl - - # 鉁忥笍 If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..94aa2bb --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,35 @@ +name: CodeQL + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 20 * * 0" + +permissions: + actions: read + contents: read + security-events: write + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: go + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v3 \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 084bb8a..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: lint -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - # Optional: allow read access to pull request. Use with `only-new-issues` option. - # pull-requests: read -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-go@v6 - with: - go-version-file: 'go.mod' - - name: golangci-lint - uses: golangci/golangci-lint-action@v9 - with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: latest - - # Optional: working directory, useful for monorepos - # working-directory: somedir - - # Optional: golangci-lint command line arguments. - args: --enable gosec - - # Optional: show only new issues if it's a pull request. The default value is `false`. - # only-new-issues: true - - # Optional: if set to true then the all caching functionality will be complete disabled, - # takes precedence over all other caching options. - # skip-cache: true - - # Optional: if set to true then the action don't cache or restore ~/go/pkg. - # skip-pkg-cache: true - - # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. - # skip-build-cache: true - - name: testing - run: go test -shuffle on ./... - - name: install govulncheck - run: go install golang.org/x/vuln/cmd/govulncheck@latest - - name: running govulncheck - run: govulncheck ./... diff --git a/.github/workflows/vulns.yml b/.github/workflows/vulns.yml deleted file mode 100644 index 0ce3aae..0000000 --- a/.github/workflows/vulns.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: vulns -on: - push: - branches: - - master - pull_request: -jobs: - vulns: - name: Vulnerability scanner - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - - name: install depm - run: go install github.com/goark/depm@latest - - name: WriteGoList - run: depm list --json > go.list - - name: Nancy - uses: sonatype-nexus-community/nancy-github-action@main diff --git a/README.md b/README.md index 3a3c833..5ef19f7 100644 --- a/README.md +++ b/README.md @@ -1,99 +1,124 @@ -# [ml] -- Make Link with Markdown Format +# [ml] -- Make links from web page metadata -[![check vulns](https://github.com/goark/ml/workflows/vulns/badge.svg)](https://github.com/goark/ml/actions) -[![lint status](https://github.com/goark/ml/workflows/lint/badge.svg)](https://github.com/goark/ml/actions) -[![lint status](https://github.com/goark/ml/workflows/build/badge.svg)](https://github.com/goark/ml/actions) +[![ci status](https://github.com/goark/ml/workflows/ci/badge.svg)](https://github.com/goark/ml/actions) +[![codeql status](https://github.com/goark/ml/workflows/CodeQL/badge.svg)](https://github.com/goark/ml/actions) +[![build status](https://github.com/goark/ml/workflows/build/badge.svg)](https://github.com/goark/ml/actions) [![GitHub license](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://raw.githubusercontent.com/goark/ml/master/LICENSE) -[![GitHub release](http://img.shields.io/github/release/goark/ml.svg)](https://github.com/goark/ml/releases/latest) +[![GitHub release](https://img.shields.io/github/release/goark/ml.svg)](https://github.com/goark/ml/releases/latest) +[![Go reference](https://pkg.go.dev/badge/github.com/goark/ml.svg)](https://pkg.go.dev/github.com/goark/ml) -This package is required Go 1.26.3 or later. +`ml` is a CLI and Go package to fetch web page metadata and render it as links +in multiple output styles (`markdown`, `wiki`, `html`, `csv`, `json`). -**Migrated repository to [github.com/goark/ml][ml]** +## Design goals -## Build and Install +- Keep CLI behavior simple for command-line and stdin workflows. +- Keep output style conversion deterministic and explicit. +- Reuse `github.com/goark/webinfo` for metadata extraction logic. +- Preserve compatibility of exported symbols when possible. +## Development + +### Requirements + +- Go 1.26.3 or later +- [Task](https://taskfile.dev/) command (local tool for this repository) + +### Local validation + +```text +task test +task govulncheck ``` -$ go install github.com/goark/ml@latest + +Run all maintenance tasks: + +```text +task ``` -## Binaries +## CI Workflows -See [latest release](https://github.com/goark/ml/releases/latest). +- `ci`: lint (`golangci-lint` with `gosec`), tests, and `govulncheck` +- `CodeQL`: scheduled and push/PR static analysis +- `build`: release build on `v*` tags via GoReleaser ## Usage -``` -$ ml -h -Usage: - ml [flags] [URL [URL]...] - -Flags: - --debug for debug - -h, --help help for ml - -i, --interactive interactive mode - -l, --log int history log size - -s, --style string link style [markdown|wiki|html|csv|json] (default "markdown") - -a, --user-agent string User-Agent string - -v, --version output version of ml -``` +### Install +```bash +go install github.com/goark/ml@latest ``` -$ ml https://git.io/vFR5M -[GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml) -``` +### CLI examples + +```bash +ml https://git.io/vFR5M ``` -$ echo https://git.io/vFR5M | ml -[GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml) + +```bash +echo https://git.io/vFR5M | ml ``` -### Support Other Styles +Use another style: +```bash +ml -s html https://git.io/vFR5M ``` -$ ml -s html https://git.io/vFR5M -GitHub - goark/ml: Make Link with Markdown Format 路 GitHub -``` - -Support Styles: `markdown`, `wiki`, `html`, `csv`, `json` -### Interactive Mode +Interactive mode: -``` -$ ml -i +```text +ml -i Input 'q' or 'quit' to stop ml> https://git.io/vFR5M -[GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml) +[GitHub - goark/ml: Make Link with Markdown Format](https://github.com/goark/ml) ml> ``` -## With Go Codes +### Public API + +- `makelink.New(ctx, urlStr, userAgent)` fetches metadata and builds a `Link`. +- `(*makelink.Link).Encode(style)` renders output in the given style. +- `makelink.GetStyle(name)` resolves style strings. + +### Use as a Go package ```go package main import ( - "context" - "fmt" - "io" - "os" + "context" + "fmt" + "io" + "os" - "github.com/goark/ml/makelink" + "github.com/goark/ml/makelink" ) func main() { - lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M", "") - if err != nil { - fmt.Fprintln(os.Stderr, err) - return - } - _, _ = io.Copy(os.Stdout, lnk.Encode(makelink.StyleMarkdown)) - // Output: - // [GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml) + lnk, err := makelink.New(context.Background(), "https://git.io/vFR5M", "") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + _, _ = io.Copy(os.Stdout, lnk.Encode(makelink.StyleMarkdown)) } ``` +## Behavior notes + +- Metadata extraction is delegated to `github.com/goark/webinfo`. +- `CanonicalURL()` falls back in this order: canonical -> location -> input URL. +- `TitleName()` falls back to URL when title is empty. + +## Error handling + +This project wraps internal errors with `github.com/goark/errs`. + ## Modules Requirement Graph [![dependency.png](./dependency.png)](./dependency.png) -[ml]: https://github.com/goark/ml "goark/ml: Make Link with Markdown Format" +[ml]: https://github.com/goark/ml "goark/ml" diff --git a/dependency.png b/dependency.png index d91f4c0..62b438a 100644 Binary files a/dependency.png and b/dependency.png differ diff --git a/facade/facade.go b/facade/facade.go index 81fcb5c..7a95272 100644 --- a/facade/facade.go +++ b/facade/facade.go @@ -17,9 +17,9 @@ import ( ) var ( - //Name is applicatin name + // Name is the application name. Name = "ml" - //Version is version for applicatin + // Version is the application version. Version = "dev-version" ) @@ -29,7 +29,7 @@ var ( debugFlag bool //debug flag ) -//newRootCmd returns cobra.Command instance for root command +// newRootCmd returns the root cobra command. func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command { rootCmd := &cobra.Command{ Use: Name + " [flags] [URL [URL]...]", @@ -62,7 +62,10 @@ func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command { if err := mkdirHistory(); err != nil { _ = ui.OutputErrln(err) } else if err := hist.Load(); err != nil { - _ = ui.OutputErrln(err) + // First run can have no history file; keep other load errors visible. + if !os.IsNotExist(err) { + _ = ui.OutputErrln(err) + } } } defer func() { @@ -136,7 +139,7 @@ func newRootCmd(ui *rwi.RWI, args []string) *cobra.Command { return rootCmd } -//Execute is called from main function +// Execute runs CLI processing and returns an exit code. func Execute(ui *rwi.RWI, args []string) (exit exitcode.ExitCode) { defer func() { //panic hundling @@ -161,7 +164,7 @@ func Execute(ui *rwi.RWI, args []string) (exit exitcode.ExitCode) { return } -/* Copyright 2017-2021 Spiegel +/* Copyright 2017-2026 Spiegel * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/facade/history/file.go b/facade/history/file.go index 25ff612..b08fb29 100644 --- a/facade/history/file.go +++ b/facade/history/file.go @@ -1,13 +1,14 @@ package history import ( + "errors" "os" "github.com/goark/errs" "github.com/nyaosorg/go-readline-ny" ) -// HistoryFile is a history file class. +// HistoryFile stores history entries backed by a file. type HistoryFile struct { *History path string @@ -15,18 +16,22 @@ type HistoryFile struct { var _ readline.IHistory = (*HistoryFile)(nil) -// NewFile function returns new HistoryFile instance. +// NewFile returns a new HistoryFile. func NewFile(size int, path string) *HistoryFile { return &HistoryFile{History: New(size), path: path} } -// Load method imports history data from file. +// Load imports history entries from the file. func (hf *HistoryFile) Load() (err error) { if hf == nil || hf.Size() == 0 || len(hf.path) == 0 { return nil } file, err := os.Open(hf.path) if err != nil { + // Missing history file on first run is expected. + if errors.Is(err, os.ErrNotExist) { + return nil + } return errs.Wrap(err) } defer func() { @@ -38,13 +43,13 @@ func (hf *HistoryFile) Load() (err error) { return } -// Save method exports history data to file. +// Save writes history entries to the file. func (hf *HistoryFile) Save() (err error) { if hf == nil || hf.Size() == 0 || len(hf.path) == 0 { return nil } - // file, err := os.Create(hf.path) - file, err := os.OpenFile(hf.path, os.O_RDWR|os.O_CREATE, 0600) + // Use OpenFile to preserve explicit 0600 permission while truncating stale data. + file, err := os.OpenFile(hf.path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return errs.Wrap(err) } @@ -57,7 +62,7 @@ func (hf *HistoryFile) Save() (err error) { return } -/* Copyright 2021-2025 Spiegel +/* Copyright 2021-2026 Spiegel * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/facade/history/history.go b/facade/history/history.go index 0d0ac8f..9de83e4 100644 --- a/facade/history/history.go +++ b/facade/history/history.go @@ -9,7 +9,7 @@ import ( "github.com/nyaosorg/go-readline-ny" ) -//History is a ring-buffer class for history (string) data. +// History is a ring buffer for command history strings. type History struct { head, tail int buffer []string @@ -17,6 +17,7 @@ type History struct { var _ readline.IHistory = (*History)(nil) +// New returns a History with the given size. func New(size int) *History { if size < 1 { return nil @@ -24,7 +25,7 @@ func New(size int) *History { return &History{head: 0, tail: -1, buffer: make([]string, size)} } -//Len method returns count of buffering strings. +// Len returns the number of entries currently buffered. func (hist *History) Len() int { if hist.Size() == 0 { return 0 @@ -44,7 +45,7 @@ func (hist *History) Len() int { return hist.tail - hist.head } -//At method returns .histroty (string) data at index. +// At returns the entry at index n. func (hist *History) At(n int) string { if hist.Size() == 0 || n >= hist.Len() { return "" @@ -53,7 +54,7 @@ func (hist *History) At(n int) string { return hist.buffer[i] } -//Size method returns size of history buffer. +// Size returns the capacity of the history buffer. func (hist *History) Size() int { if hist == nil { return 0 @@ -61,7 +62,7 @@ func (hist *History) Size() int { return len(hist.buffer) } -//Add method adds new string in history buffer. +// Add appends a new history entry. func (hist *History) Add(s string) { if hist.Size() == 0 || len(s) == 0 { return @@ -82,7 +83,7 @@ func (hist *History) Add(s string) { } } -//Import method imports history data from reader. +// Import loads history entries from r. func (hist *History) Import(r io.Reader) error { if hist.Size() == 0 { return nil @@ -94,7 +95,7 @@ func (hist *History) Import(r io.Reader) error { return errs.Wrap(scanner.Err()) } -//Export method exports history data to writer. +// Export writes all history entries to w. func (hist *History) Export(w io.Writer) error { for i := 0; i < hist.Len(); i++ { if _, err := fmt.Fprintln(w, hist.At(i)); err != nil { diff --git a/facade/interactive/interactive.go b/facade/interactive/interactive.go index 66026b2..1b0928e 100644 --- a/facade/interactive/interactive.go +++ b/facade/interactive/interactive.go @@ -14,6 +14,7 @@ import ( "github.com/nyaosorg/go-readline-ny" ) +// Do runs the interactive prompt loop. func Do(opts *options.Options) error { editor := &readline.Editor{ PromptWriter: func(w io.Writer) (int, error) { diff --git a/facade/options/options.go b/facade/options/options.go index f26db27..3849129 100644 --- a/facade/options/options.go +++ b/facade/options/options.go @@ -10,14 +10,14 @@ import ( "github.com/goark/ml/makelink" ) -//Options class is Options for making link +// Options holds settings for link generation. type Options struct { linkStyle makelink.Style hist *history.HistoryFile userAgent string } -//New returns new Options instance +// New returns a new Options instance. func New(s makelink.Style, hist *history.HistoryFile, userAgent string) *Options { if hist == nil { hist = history.NewFile(0, "") @@ -25,23 +25,23 @@ func New(s makelink.Style, hist *history.HistoryFile, userAgent string) *Options return &Options{linkStyle: s, hist: hist, userAgent: userAgent} } -//History method returns history.HistoryFile instance. +// History returns the history store. func (c *Options) History() *history.HistoryFile { return c.hist } -//MakeLink is making link +// MakeLink creates an encoded link from the given URL. func (c *Options) MakeLink(ctx context.Context, urlStr string) (io.Reader, error) { if c == nil { return nil, errs.Wrap(ecode.ErrNullPointer) } - c.History().Add(urlStr) lnk, err := makelink.New(ctx, urlStr, c.userAgent) if err != nil { return nil, errs.Wrap(err) } + c.History().Add(urlStr) return lnk.Encode(c.linkStyle), nil } -/* Copyright 2017-2021 Spiegel +/* Copyright 2017-2026 Spiegel * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/facade/options/options_test.go b/facade/options/options_test.go index 62dc2e4..d8747e2 100644 --- a/facade/options/options_test.go +++ b/facade/options/options_test.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "net/http" + "net/http/httptest" "os" "testing" @@ -13,8 +15,17 @@ import ( "github.com/goark/ml/makelink" ) +func newTestServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `Example Title`) + })) +} + func TestMakeLink(t *testing.T) { - urlStr := "https://git.io/vFR5M" + srv := newTestServer() + defer srv.Close() + + urlStr := srv.URL opt := options.New(makelink.StyleMarkdown, history.NewFile(1, ""), "") rRes, err := opt.MakeLink(context.Background(), urlStr) if err != nil { @@ -25,7 +36,7 @@ func TestMakeLink(t *testing.T) { t.Errorf("Error in io.Copy(): %+v", err) } - res := "[GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml)" + res := "[Example Title](https://example.com/canonical)" str := outBuf.String() if str != res { t.Errorf("Context.MakeLink() = \"%v\", want \"%v\".", str, res) @@ -37,7 +48,10 @@ func TestMakeLink(t *testing.T) { } func TestMakeLinkNil(t *testing.T) { - rRes, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), "https://git.io/vFR5M") + srv := newTestServer() + defer srv.Close() + + rRes, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), srv.URL) if err != nil { t.Errorf("Error in Context.MakeLink(): %+v", err) } @@ -46,7 +60,7 @@ func TestMakeLinkNil(t *testing.T) { t.Errorf("Error in io.Copy(): %+v", err) } - res := "[GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml)" + res := "[Example Title](https://example.com/canonical)" str := outBuf.String() if str != res { t.Errorf("Context.MakeLink() = \"%v\", want \"%v\".", str, res) @@ -54,7 +68,7 @@ func TestMakeLinkNil(t *testing.T) { } func TestMakeLinkErr(t *testing.T) { - _, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), "https://foo.bar") + _, err := options.New(makelink.StyleMarkdown, nil, "").MakeLink(context.Background(), "://bad-url") if err == nil { t.Error("Context.MakeLink() = nil error, not want nil error.") } else { @@ -62,7 +76,7 @@ func TestMakeLinkErr(t *testing.T) { } } -/* Copyright 2017-2021 Spiegel +/* Copyright 2017-2026 Spiegel * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/go.mod b/go.mod index 3fa6a56..29df76e 100644 --- a/go.mod +++ b/go.mod @@ -5,27 +5,29 @@ go 1.26 toolchain go1.26.3 require ( - github.com/PuerkitoBio/goquery v1.12.0 github.com/atotto/clipboard v0.1.4 - github.com/goark/errs v1.3.3 - github.com/goark/fetch v0.5.1 + github.com/goark/errs v1.3.4 github.com/goark/gocli v0.13.0 - github.com/mattn/go-encoding v0.0.2 + github.com/goark/webinfo v0.1.1 github.com/nyaosorg/go-readline-ny v1.15.1 github.com/spf13/cobra v1.10.2 - golang.org/x/net v0.53.0 ) require ( + github.com/PuerkitoBio/goquery v1.12.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect github.com/clipperhouse/uax29/v2 v2.6.0 // indirect + github.com/goark/fetch v0.5.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-encoding v0.0.2 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.23 // indirect github.com/mattn/go-tty v0.0.7 // indirect github.com/nyaosorg/go-ttyadapter v0.6.2 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/term v0.42.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/image v0.40.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/sys v0.44.0 // indirect + golang.org/x/term v0.43.0 // indirect + golang.org/x/text v0.37.0 // indirect ) diff --git a/go.sum b/go.sum index fd56d7c..f8b5ddb 100644 --- a/go.sum +++ b/go.sum @@ -7,12 +7,14 @@ github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn github.com/clipperhouse/uax29/v2 v2.6.0 h1:z0cDbUV+aPASdFb2/ndFnS9ts/WNXgTNNGFoKXuhpos= github.com/clipperhouse/uax29/v2 v2.6.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/goark/errs v1.3.3 h1:vqzm/1aqvh4Ha8JDqIqIU5ZoBtjaRezFcMC55B5ljAM= -github.com/goark/errs v1.3.3/go.mod h1:4xM7rorwYQlqh9kUhfKpC5P7VAJW2KfvuQpYnTaU0ek= -github.com/goark/fetch v0.5.1 h1:AZql1WTANG9umytvf4EzMzIyGHY4RP+N6Q5n0ALxRr0= -github.com/goark/fetch v0.5.1/go.mod h1:blLKmPKgSDw33ocut79ypHT4V0MLmfVY/MHKpq+OgD4= +github.com/goark/errs v1.3.4 h1:/+/xwF3UwXGxGGLurzBTaMMoryTBeaPfheJ1aW9cglA= +github.com/goark/errs v1.3.4/go.mod h1:4xM7rorwYQlqh9kUhfKpC5P7VAJW2KfvuQpYnTaU0ek= +github.com/goark/fetch v0.5.3 h1:ZwT5N04BSiPw2tF2gG5MXsmoSr+A/sxE52KJWy/aWzw= +github.com/goark/fetch v0.5.3/go.mod h1:jgu+bn1HN8AfEks+ENqiPJVF99Cvs7cb2dqSujhjsOE= github.com/goark/gocli v0.13.0 h1:hR/5E4JGMEcbQxkSqR7K/0XnYY2Hd6GDpuazXGC3jn4= github.com/goark/gocli v0.13.0/go.mod h1:pFYWXAXZ5G4QqPcXsDTSFbCuVg0qO40NYkp2XKthc18= +github.com/goark/webinfo v0.1.1 h1:GCeGZTcxXhQI4XLirEje30x9OaADuYhMG1xgxcikI+o= +github.com/goark/webinfo v0.1.1/go.mod h1:jPHA6OWQPNvLr6er7SwVMiOuvlNqeNRF0K56VFf66Sc= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -41,6 +43,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/image v0.40.0 h1:Tw4GyDXMo+daZN1znreBRC3VayR1aLFUyUEOLUdW1a8= +golang.org/x/image v0.40.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -55,8 +59,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -75,8 +79,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -86,8 +90,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -97,8 +101,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/makelink/makelink.go b/makelink/makelink.go index 0f3e4c3..f19c611 100644 --- a/makelink/makelink.go +++ b/makelink/makelink.go @@ -1,23 +1,18 @@ package makelink import ( - "bufio" "bytes" "context" "encoding/json" "fmt" "io" - "net/http" "strings" - "github.com/PuerkitoBio/goquery" "github.com/goark/errs" - "github.com/goark/fetch" - encoding "github.com/mattn/go-encoding" - "golang.org/x/net/html/charset" + "github.com/goark/webinfo" ) -// Link class is information of URL +// Link stores metadata of a URL. type Link struct { URL string `json:"url,omitempty"` Location string `json:"location,omitempty"` @@ -26,72 +21,21 @@ type Link struct { Description string `json:"description,omitempty"` } -// New returns new Link instance +// New fetches URL metadata and returns a Link instance. func New(ctx context.Context, urlStr, userAgent string) (link *Link, err error) { link = &Link{URL: urlStr} - u, err := fetch.URL(urlStr) + info, err := webinfo.Fetch(ctx, urlStr, userAgent) if err != nil { return link, errs.Wrap(err, errs.WithContext("url", urlStr)) } - if len(userAgent) == 0 { - userAgent = "goark/ml (+https://github.com/goark/ml)" //dummy user-agent string + link.URL = trimString(info.URL) + if len(link.URL) == 0 { + link.URL = urlStr } - resp, err := fetch.New(fetch.WithHTTPClient(&http.Client{})).GetWithContext( - ctx, - u, - fetch.WithRequestHeaderSet("User-Agent", userAgent), - ) - if err != nil { - return link, errs.Wrap(err, errs.WithContext("url", urlStr)) - } - defer func() { - if cerr := resp.Close(); cerr != nil { - err = errs.Join(err, cerr) - } - }() - - link.Location = resp.Request().URL.String() - - br := bufio.NewReader(resp.Body()) - var r io.Reader = br - if data, err2 := br.Peek(1024); err2 == nil { //next 1024 bytes without advancing the reader. - enc, name, _ := charset.DetermineEncoding(data, resp.Header().Get("content-type")) - if enc != nil { - r = enc.NewDecoder().Reader(br) - } else if len(name) > 0 { - if enc := encoding.GetEncoding(name); enc != nil { - r = enc.NewDecoder().Reader(br) - } - } - } - doc, err := goquery.NewDocumentFromReader(r) - if err != nil { - err = errs.Wrap(err) - return - } - - doc.Find("head").Each(func(_ int, s *goquery.Selection) { - s.Find("title").Each(func(_ int, s *goquery.Selection) { - t := s.Text() - if len(t) > 0 { - link.Title = trimString(t) - } - }) - s.Find("meta[name='description']").Each(func(_ int, s *goquery.Selection) { - if v, ok := s.Attr("content"); ok { - if len(v) > 0 { - link.Description = trimString(v) - } - } - }) - s.Find("link[rel='canonical']").Each(func(_ int, s *goquery.Selection) { - if v, ok := s.Attr("href"); ok { - if len(v) > 0 { - link.Canonical = trimString(v) - } - } - }) - }) + link.Location = trimString(info.Location) + link.Canonical = trimString(info.Canonical) + link.Title = trimString(info.Title) + link.Description = trimString(info.Description) return } @@ -105,7 +49,7 @@ func trimString(s string) string { return strings.TrimSpace(replacer.Replace(s)) } -// TitleName returns string of title name +// TitleName returns the display title. func (lnk *Link) TitleName() string { if lnk == nil { return "" @@ -116,7 +60,7 @@ func (lnk *Link) TitleName() string { return lnk.URL } -// CanonicalURL returns the canonical URL. +// CanonicalURL returns the canonical URL when available. func (lnk *Link) CanonicalURL() string { if lnk == nil { return "" @@ -130,7 +74,7 @@ func (lnk *Link) CanonicalURL() string { return lnk.URL } -// Encode returns string (io.Reader) with other style +// Encode encodes Link in the requested style. func (lnk *Link) Encode(t Style) io.Reader { if lnk == nil { return io.NopCloser(bytes.NewReader(nil)) @@ -154,6 +98,7 @@ func escapeQuoteCsv(s string) string { return strings.ReplaceAll(s, "\"", "\"\"") } +// String returns Link as JSON text. func (lnk *Link) String() string { if lnk == nil { return "" @@ -161,7 +106,7 @@ func (lnk *Link) String() string { return fmt.Sprint(lnk.Encode(StyleJSON)) } -/* Copyright 2017-2025 Spiegel +/* Copyright 2017-2026 Spiegel * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/makelink/makelink_test.go b/makelink/makelink_test.go index f4a27c8..62d23fa 100644 --- a/makelink/makelink_test.go +++ b/makelink/makelink_test.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "net/http" + "net/http/httptest" "os" "testing" ) @@ -66,7 +68,7 @@ func TestString(t *testing.T) { } func TestNewErr(t *testing.T) { - _, err := New(context.Background(), "https://foo.bar", "") + _, err := New(context.Background(), "://bad-url", "") if err == nil { t.Error("New() = nil error, not want nil error.") } else { @@ -74,6 +76,33 @@ func TestNewErr(t *testing.T) { } } +func TestNew(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `Example Title`) + })) + defer srv.Close() + + link, err := New(context.Background(), srv.URL, "") + if err != nil { + t.Fatalf("New() error = %v", err) + } + if link.URL != srv.URL { + t.Errorf("URL = %q, want %q", link.URL, srv.URL) + } + if link.Location != srv.URL { + t.Errorf("Location = %q, want %q", link.Location, srv.URL) + } + if link.Canonical != "https://example.com/canonical" { + t.Errorf("Canonical = %q, want %q", link.Canonical, "https://example.com/canonical") + } + if link.Title != "Example Title" { + t.Errorf("Title = %q, want %q", link.Title, "Example Title") + } + if link.Description != "Example Description" { + t.Errorf("Description = %q, want %q", link.Description, "Example Description") + } +} + func ExampleNew() { link, err := New(context.Background(), "https://git.io/vFR5M", "") if err != nil { @@ -81,8 +110,6 @@ func ExampleNew() { return } fmt.Println(link.Encode(StyleMarkdown)) - // Output: - // [GitHub - goark/ml: Make Link with Markdown Format 路 GitHub](https://github.com/goark/ml) } /* Copyright 2017-2026 Spiegel diff --git a/makelink/style.go b/makelink/style.go index 1c806b9..5649b27 100644 --- a/makelink/style.go +++ b/makelink/style.go @@ -7,21 +7,21 @@ import ( "github.com/goark/ml/ecode" ) -//Style as link style +// Style represents a link output style. type Style int const ( - //StyleUnknown is unknown link style + // StyleUnknown is an unknown link style. StyleUnknown Style = iota - //StyleMarkdown is unknown markdown style + // StyleMarkdown is markdown style. StyleMarkdown - //StyleWiki is unknown wiki style + // StyleWiki is wiki style. StyleWiki - //StyleHTML is unknown HTML anchor style + // StyleHTML is HTML anchor style. StyleHTML - //StyleCSV is CSV data format + // StyleCSV is CSV data format. StyleCSV - //StyleJSON is JSON format + // StyleJSON is JSON format. StyleJSON ) @@ -42,12 +42,12 @@ var ( } ) -//StyleList returns string Style list +// StyleList returns supported style names. func StyleList() string { return strings.Join(styleList, "|") } -//GetStyle returns Style from string +// GetStyle returns a Style from its string representation. func GetStyle(s string) (Style, error) { for t, v := range styleMap { if strings.EqualFold(v, s) {