From a9d7547df03775c1523620579c73ad3e559a7011 Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Wed, 11 Sep 2024 18:22:29 +0200 Subject: [PATCH 1/7] add beta command 'parse' --- .devcontainer/Dockerfile | 43 ++++++++++++ .devcontainer/devcontainer.json | 16 +++++ .github/dependabot.yml | 12 ++++ cmd/needle.go | 4 +- cmd/parse/parse.go | 84 ++++++++++++++++++++++++ cmd/parse/utils.go | 112 ++++++++++++++++++++++++++++++++ cmd/root.go | 7 ++ go.mod | 2 +- go.sum | 4 +- 9 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .github/dependabot.yml create mode 100644 cmd/parse/parse.go create mode 100644 cmd/parse/utils.go diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..fa289ac --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,43 @@ +ARG GO_VERSION=1.23 +ARG ALPINE_VERSION=3.20 + +FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=1000 + +# Setup user +RUN adduser $USERNAME -s /bin/sh -D -u $USER_UID $USER_GID && \ + mkdir -p /etc/sudoers.d && \ + echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \ + chmod 0440 /etc/sudoers.d/$USERNAME + +# Install packages +RUN apk add -q \ + --update \ + --progress \ + --no-cache \ + git \ + sudo \ + openssh-client \ + zsh \ + bind-tools \ + bash \ + make \ + jq \ + curl \ + vim + +# Install Go language server +# RUN go get -u -v golang.org/x/tools/cmd/gopls 2>&1 + +# Setup shell +USER $USERNAME +RUN sh -c "$(wget -O- https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)" "" --unattended &> /dev/null +ENV ENV="/home/$USERNAME/.ashrc" \ + ZSH=/home/$USERNAME/.oh-my-zsh \ + EDITOR=vi \ + LANG=en_US.UTF-8 +RUN printf 'ZSH_THEME="robbyrussell"\nENABLE_CORRECTION="false"\nplugins=(git copyfile extract colorize dotenv encode64 golang)\nsource $ZSH/oh-my-zsh.sh' > "/home/$USERNAME/.zshrc" +RUN echo "exec `which zsh`" > "/home/$USERNAME/.ashrc" +USER root diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..46b11e5 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,16 @@ +{ + "name": "Go", + "dockerFile": "Dockerfile", + "forwardPorts": [80, 443, 53], + "postCreateCommand": "go mod download", + "runArgs": [ + "-u", + "vscode", + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + // map SSH keys for Git + "-v", "${env:HOME}/.ssh:/home/vscode/.ssh:ro", + "--name", "needle-dev" +] +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f33a02c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for more information: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# https://containers.dev/guide/dependabot + +version: 2 +updates: + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: weekly diff --git a/cmd/needle.go b/cmd/needle.go index b33d8af..a2bf440 100644 --- a/cmd/needle.go +++ b/cmd/needle.go @@ -207,7 +207,7 @@ func start(_ *cobra.Command, _ []string) error { router := http.NewRouter(logger, routes...) - tlsSrv, err := server.NewServer( + tlsSrv, err := server.New( server.WithName("needle-tls"), server.WithLogger(logger), server.WithRouter(router), @@ -228,7 +228,7 @@ func start(_ *cobra.Command, _ []string) error { } }() - httpSrv, err := server.NewServer( + httpSrv, err := server.New( server.WithName("needle-http"), server.WithLogger(logger), server.WithRouter(router), diff --git a/cmd/parse/parse.go b/cmd/parse/parse.go new file mode 100644 index 0000000..f4da837 --- /dev/null +++ b/cmd/parse/parse.go @@ -0,0 +1,84 @@ +package parse + +import ( + "slices" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + logLevel string + inputFiles []string + outputFile string + publicIPv4 string + publicIPv6 string +) + +var parseHostCmd = &cobra.Command{ + Use: "parse", + Short: "Parse hosts file", + RunE: start, +} + +func bindFlag(flag string) error { + err := viper.BindPFlag(flag, parseHostCmd.PersistentFlags().Lookup(flag)) + if err != nil { + return err + } + + return nil +} + +func NewCmd() (*cobra.Command, error) { + parseHostCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error, fatal, panic)") + if err := bindFlag("log-level"); err != nil { + return nil, err + } + + parseHostCmd.PersistentFlags().StringSliceVar(&inputFiles, "file", []string{}, "Hosts file URL") + if err := bindFlag("file"); err != nil { + return nil, err + } + + parseHostCmd.PersistentFlags().StringVar(&outputFile, "output", "data/hosts", "Output file") + if err := bindFlag("output"); err != nil { + return nil, err + } + + parseHostCmd.PersistentFlags().StringVar(&publicIPv4, "public-ip", "0.0.0.0", "Public IP") + if err := bindFlag("public-ip"); err != nil { + return nil, err + } + + parseHostCmd.PersistentFlags().StringVar(&publicIPv6, "public-ip6", "::", "Public IPv6") + if err := bindFlag("public-ip6"); err != nil { + return nil, err + } + + return parseHostCmd, nil +} + +func start(_ *cobra.Command, _ []string) error { + hosts := []string{} + + for _, file := range inputFiles { + bs, err := getHostFile(file) + if err != nil { + return err + } + + hosts = append(hosts, Hosts(bs, publicIPv4, publicIPv6)...) + } + + // Remove duplicates + slices.Sort(hosts) + hosts = slices.Compact(hosts) + + // Write to file + if err := WriteHosts(hosts, outputFile); err != nil { + return err + } + + return nil +} diff --git a/cmd/parse/utils.go b/cmd/parse/utils.go new file mode 100644 index 0000000..8d21ba1 --- /dev/null +++ b/cmd/parse/utils.go @@ -0,0 +1,112 @@ +package parse + +import ( + "context" + "io" + "net/http" + "os" + "slices" + "strings" +) + +func getHostFile(file string) ([]byte, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, file, http.NoBody) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + bs, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + return bs, nil +} + +func WriteHosts(hosts []string, file string) error { + f, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + + for _, host := range hosts { + if _, err := f.WriteString(host + "\n"); err != nil { + return err + } + } + + return nil +} + +func filterHosts(field string) bool { + // filter unwanted hosts + return slices.Contains([]string{ + "broadcasthost", + "ip6-allhosts", + "ip6-allnodes", + "ip6-allrouters", + "ip6-localhost", + "ip6-localnet", + "ip6-loopback", + "ip6-mcastprefix", + "local", + "localhost", + "localhost.localdomain", + "localhost4", + "localhost4.localdomain4", + "localhost6", + "localhost6.localdomain6", + "localdomain", + "localdomain.local", + "localdomain4", + "0.0.0.0", + }, field) +} + +func Hosts(bs []byte, ipv4, ipv6 string) []string { + hosts := []string{} + + lines := strings.Split(string(bs), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "#") { + continue + } + + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + + if len(fields) > 2 { + continue + } + + if len(fields) == 1 { + if filterHosts(fields[0]) { + continue + } + + hosts = append(hosts, ipv4+" "+fields[0], ipv6+" "+fields[0]) + continue + } + + if len(fields) == 2 { + if filterHosts(fields[1]) { + continue + } + + hosts = append(hosts, ipv4+" "+fields[1], ipv6+" "+fields[1]) + continue + } + } + + return hosts +} diff --git a/cmd/root.go b/cmd/root.go index 3c01fb3..fea54db 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + "go.pixelfactory.io/needle/cmd/parse" "go.pixelfactory.io/pkg/version" ) @@ -20,6 +21,12 @@ func Execute() error { return err } + parseHostCmd, err := parse.NewCmd() + if err != nil { + return err + } + + needleCmd.AddCommand(parseHostCmd) cobra.OnInitialize(initConfig) return needleCmd.Execute() } diff --git a/go.mod b/go.mod index f47a56e..2fc5a87 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 go.pixelfactory.io/pkg/observability/log v1.3.0 - go.pixelfactory.io/pkg/server v0.2.0 + go.pixelfactory.io/pkg/server v0.3.0 go.pixelfactory.io/pkg/version v0.1.0 ) diff --git a/go.sum b/go.sum index ce05e73..7cca5f2 100644 --- a/go.sum +++ b/go.sum @@ -151,8 +151,8 @@ go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.pixelfactory.io/pkg/observability/log v1.3.0 h1:rkRA2+bHwxuy7i3WGsEMOFMGV4psY6McePQNcmSK5O0= go.pixelfactory.io/pkg/observability/log v1.3.0/go.mod h1:GDl9NboAsdd9zF+qOi4xVBr9Tbrj68qNxGPQ398x/8U= -go.pixelfactory.io/pkg/server v0.2.0 h1:bPsVu4xGQug/Dm0ctjtWe0HLX/nHofLe9zfbmtwdOlA= -go.pixelfactory.io/pkg/server v0.2.0/go.mod h1:0+xqwZ0RM2htIO2Fy1oJMjJC7uaU4aayOSVBFpk7ROU= +go.pixelfactory.io/pkg/server v0.3.0 h1:5TTSJkmVwPPCDjrdHop104fK85+J5aSzO5oeIpZPeSs= +go.pixelfactory.io/pkg/server v0.3.0/go.mod h1:0+xqwZ0RM2htIO2Fy1oJMjJC7uaU4aayOSVBFpk7ROU= go.pixelfactory.io/pkg/version v0.1.0 h1:HK+uvMADE1bZ/Rdwet4o1YF7F/c5ZdEwfBnZWyrpa9U= go.pixelfactory.io/pkg/version v0.1.0/go.mod h1:wc4uuUNsbqgcfL0amMAZfxsRZHMqTQGm7f4wkYo1Fs4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= From c34b33407bae40e284f51f1cb87c6bd777a3e839 Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Wed, 11 Sep 2024 18:28:33 +0200 Subject: [PATCH 2/7] update goreleaser action --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 665f0fd..dafbdec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,10 +32,10 @@ jobs: password: ${{ secrets.DOCKER_TOKEN }} - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v4 + uses: goreleaser/goreleaser-action@v6 if: success() && startsWith(github.ref, 'refs/tags/') with: - version: latest - args: release --rm-dist + version: "~> v2" + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 208a12283521db792e5523bd0a6f3134434c9576 Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Wed, 11 Sep 2024 18:32:42 +0200 Subject: [PATCH 3/7] update goreleaser action --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dafbdec..931bd17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,6 @@ jobs: uses: goreleaser/goreleaser-action@v6 if: success() && startsWith(github.ref, 'refs/tags/') with: - version: "~> v2" args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b4ab3797915d128db3811536ac676ca0b7272877 Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Wed, 11 Sep 2024 18:33:37 +0200 Subject: [PATCH 4/7] update go actions --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 931bd17..4c034a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: 1.21.x + go-version: 1.23.x - uses: docker/setup-qemu-action@v2 - uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f1b7762..ec84b3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: 1.21.x + go-version: 1.23.x - name: Install step-cli run: | From 7acc9edf95652913d7bc44600e42fd3c883214af Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Mon, 16 Sep 2024 11:09:42 +0200 Subject: [PATCH 5/7] update goreleaser configuration --- .goreleaser.yml | 100 +++++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index c2edc5b..2c0f207 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,42 +1,47 @@ +version: 2 + env: - GO111MODULE=on before: hooks: - go mod tidy + builds: -- env: - - CGO_ENABLED=0 - goos: - - linux - - darwin - - windows - goarch: - - 386 - - amd64 - - arm - - arm64 - mod_timestamp: '{{ .CommitTimestamp }}' - flags: - - -trimpath - ldflags: - - -s -w - - -X go.pixelfactory.io/pkg/version.REVISION={{ .ShortCommit }} - - -X go.pixelfactory.io/pkg/version.BUILDDATE={{ .CommitDate }} + - env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - "386" + - amd64 + - arm + - arm64 + mod_timestamp: "{{ .CommitTimestamp }}" + flags: + - -trimpath + ldflags: + - -s -w + - -X go.pixelfactory.io/pkg/version.REVISION={{ .ShortCommit }} + - -X go.pixelfactory.io/pkg/version.BUILDDATE={{ .CommitDate }} + checksum: - name_template: '{{ .ProjectName }}_checksums.txt' + name_template: "{{ .ProjectName }}_checksums.txt" + changelog: sort: asc filters: exclude: - - '^docs:' - - '^test:' - - '^ci:' - - Merge pull request - - Merge branch - - go mod tidy + - "^docs:" + - "^test:" + - "^ci:" + - Merge pull request + - Merge branch + - go mod tidy nfpms: - - file_name_template: '{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' + - file_name_template: "{{ .ProjectName }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" homepage: https://github.com/pixelfactoryio/needle description: |- Small HTTP/1.1, HTTP/2, server with TLS support, that block ads and trackers by reponsding to all requests with a transparent 1x1 gif pixel. @@ -44,13 +49,13 @@ nfpms: license: MIT vendor: pixelfactory.io formats: - - apk - - deb - - rpm + - apk + - deb + - rpm dockers: - image_templates: - - 'pixelfactory/needle:{{ .Tag }}-amd64' + - "pixelfactory/needle:{{ .Tag }}-amd64" dockerfile: Dockerfile use: buildx build_flag_templates: @@ -62,7 +67,7 @@ dockers: - "--label=org.opencontainers.image.source={{.GitURL}}" - "--platform=linux/amd64" - image_templates: - - 'pixelfactory/needle:{{ .Tag }}-arm64' + - "pixelfactory/needle:{{ .Tag }}-arm64" dockerfile: Dockerfile use: buildx build_flag_templates: @@ -76,32 +81,33 @@ dockers: goarch: arm64 docker_manifests: - - name_template: 'pixelfactory/needle:{{ .Tag }}' + - name_template: "pixelfactory/needle:{{ .Tag }}" image_templates: - - 'pixelfactory/needle:{{ .Tag }}-amd64' - - 'pixelfactory/needle:{{ .Tag }}-arm64' - - name_template: 'pixelfactory/needle:latest' + - "pixelfactory/needle:{{ .Tag }}-amd64" + - "pixelfactory/needle:{{ .Tag }}-arm64" + - name_template: "pixelfactory/needle:latest" image_templates: - - 'pixelfactory/needle:{{ .Tag }}-amd64' - - 'pixelfactory/needle:{{ .Tag }}-arm64' + - "pixelfactory/needle:{{ .Tag }}-amd64" + - "pixelfactory/needle:{{ .Tag }}-arm64" archives: - - name_template: '{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' - replacements: - darwin: Darwin - linux: Linux - windows: Windows - 386: i386 - amd64: x86_64 + - format: tar.gz + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} format_overrides: - - goos: windows - format: zip + - goos: windows + format: zip files: - LICENSE - README.md brews: - - tap: + - repository: owner: pixelfactoryio name: homebrew-tools commit_author: From 7bc5a0aeb857ef2dd10f08cdd87fb27ab217e818 Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Mon, 16 Sep 2024 14:38:01 +0200 Subject: [PATCH 6/7] fix(Dockerfile): Update needle path --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dc5feac..f89acb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,4 +5,4 @@ COPY needle_*.apk /tmp/ RUN apk add --allow-untrusted /tmp/needle_*.apk \ && rm -fr /tmp/needle_*.apk -ENTRYPOINT ["/usr/local/bin/needle"] +ENTRYPOINT ["/usr/bin/needle"] From 6ee587999cf8acb933accfce8e6d18a1be356e5b Mon Sep 17 00:00:00 2001 From: Amine Benseddik Date: Wed, 11 Dec 2024 19:42:34 +0100 Subject: [PATCH 7/7] feat(docker): Add docker-compose configuration Signed-off-by: Amine Benseddik --- cmd/needle.go | 2 +- docker-compose.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml diff --git a/cmd/needle.go b/cmd/needle.go index a2bf440..a837018 100644 --- a/cmd/needle.go +++ b/cmd/needle.go @@ -141,7 +141,7 @@ func start(_ *cobra.Command, _ []string) error { coredns.WithPort(corednsPort), coredns.WithHostsFile(corednsHostsFile), coredns.WithUpstreams(corednsUpstreams), - coredns.WithCoreFile(corednsCoreFile), + coredns.WithCoreFile(corednsCoreFile), // TODO: include only if flag is set ) // Start CoreDNS Server diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..44af498 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + needle: + image: pixelfactory/needle:v0.3.5-beta + ports: + - "80:80" + - "443:443" + - "53:53" + - "53:53/udp" + volumes: + - ./data:/data + command: + - --ca=/data/certs/root-ca.crt + - --ca-key=data/certs/root-ca.key + - --coredns + - --coredns-hosts-file=/data/hosts + + parse: + image: pixelfactory/needle:v0.3.5-beta + volumes: + - ./data:/data + command: + - parse + - --file=http://sbc.io/hosts/hosts + - --public-ip=198.19.249.228 + - --public-ip6=fd07:b51a:cc66:0:10a4:84ff:fea7:2b18 + - --output=/data/hosts