read.go

package main

// Finding the posts inside whatever somebody points this at.
//
// Nobody should have to unpack anything. Reddit, X and Mastodon all hand
// over an archive, and the archive is what this reads, opened in place: the
// files inside are read into memory and never written out.

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

type Input struct {
	Names []string
	Posts []Post
	// Gaps are what could not be read. They are printed before anything
	// else, because a count built on half an export is half a count.
	Gaps []string
	// Kinds counts what was read, by what it was: "Reddit comments", "X
	// posts". The report says so, so nobody mistakes a count of their
	// comments for a count of everything they have published.
	Kinds map[string]int
	// Skipped is what was left alone on purpose, with the reason.
	Skipped map[string]string
}

// The largest single file this will read. An X archive for somebody who
// has posted every day for ten years is a few hundred megabytes of
// JavaScript, and that is the biggest thing anybody has a reason to hand it.
const tooBig = 1 << 30

func readInputs(paths []string) *Input {
	in := &Input{Kinds: map[string]int{}, Skipped: map[string]string{}}
	for _, p := range paths {
		in.Names = append(in.Names, filepath.Base(strings.TrimRight(p, `/\`)))
		in.readPath(p)
	}
	sort.SliceStable(in.Posts, func(i, j int) bool { return in.Posts[i].When.Before(in.Posts[j].When) })
	return in
}

func (in *Input) readPath(p string) {
	info, err := os.Stat(p)
	if err != nil {
		in.gap("%s: %v", p, err)
		return
	}
	if !info.IsDir() {
		body, err := readAll(p, info.Size())
		if err != nil {
			in.gap("%s: %v", p, err)
			return
		}
		if !in.readFile(filepath.Base(p), body, true) {
			in.gap("%s is not something this reads. It reads the exports Reddit, X and "+
				"Mastodon hand over, Hacker News comments saved from its API, and "+
				"plain text or Markdown.", filepath.Base(p))
		}
		return
	}

	before := len(in.Posts)
	err = filepath.WalkDir(p, func(where string, d fs.DirEntry, err error) error {
		if err != nil {
			in.gap("%s: %v", where, err)
			return nil
		}
		if d.IsDir() {
			if where != p && strings.HasPrefix(d.Name(), ".") {
				return fs.SkipDir
			}
			return nil
		}
		info, err := d.Info()
		if err != nil {
			in.gap("%s: %v", where, err)
			return nil
		}
		if !wanted(d.Name()) {
			return nil
		}
		body, err := readAll(where, info.Size())
		if err != nil {
			in.gap("%s: %v", where, err)
			return nil
		}
		rel, _ := filepath.Rel(p, where)
		in.readFile(filepath.ToSlash(rel), body, false)
		return nil
	})
	if err != nil {
		in.gap("%s: %v", p, err)
	}
	if len(in.Posts) == before {
		in.gap("Nothing in %s was a post this could read.", filepath.Base(p))
	}
}

func readAll(p string, size int64) ([]byte, error) {
	if size > tooBig {
		return nil, fmt.Errorf("%d MB, which is more than this will read into memory", size>>20)
	}
	return os.ReadFile(p)
}

// wanted keeps a folder walk from reading every photograph in an X archive
// only to decide it is not text.
func wanted(name string) bool {
	switch strings.ToLower(path.Ext(name)) {
	case ".csv", ".js", ".json", ".txt", ".md", ".markdown", ".zip", ".gz", ".tgz":
		return true
	}
	return false
}

// readFile reads one file, which may itself be an archive. It says whether
// the file was something it recognised, which is only reported as a gap
// when somebody named that file directly: a folder is allowed to contain
// things that are not posts.
func (in *Input) readFile(name string, body []byte, named bool) bool {
	lower := strings.ToLower(name)
	base := path.Base(lower)
	switch {
	case strings.HasSuffix(lower, ".zip"):
		return in.readZip(name, body)
	case strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz"):
		return in.readTar(name, body)
	}

	if why, private := privateFiles[base]; private {
		in.Skipped[name] = why
		return true
	}
	switch {
	case base == "comments.csv" || base == "posts.csv":
		return in.readReddit(name, body)
	case isTweets(base) || base == "profile.js" || base == "note-tweet.js":
		return in.readX(name, base, body)
	case base == "outbox.json":
		return in.readOutbox(name, body)
	case base == "actor.json":
		return in.readActor(name, body)
	case strings.HasSuffix(base, ".json"):
		return in.readHN(name, body)
	case strings.HasSuffix(base, ".csv"):
		// A CSV with a body and a date column is a Reddit export whatever
		// it has been renamed to.
		if named {
			return in.readReddit(name, body)
		}
		return false
	case strings.HasSuffix(base, ".txt") || strings.HasSuffix(base, ".md") ||
		strings.HasSuffix(base, ".markdown"):
		in.addText(name, string(body))
		return true
	}
	return false
}

// privateFiles are the parts of an export that are not public. The question
// this answers is what a stranger can find out, and a stranger cannot read
// your private messages. Reading them would put things in the count that
// no attacker has.
var privateFiles = map[string]string{
	"messages.csv":             "private messages",
	"messages_archive.csv":     "private messages",
	"message_headers.csv":      "private messages",
	"chat_history.csv":         "private chats",
	"direct-messages.js":       "private messages",
	"direct-messages-group.js": "private messages",
	"ip_logs.csv":              "the addresses Reddit saw you connect from",
	"ip-audit.js":              "the addresses X saw you connect from",
	"account.js":               "the email and phone on the account",
	"drafts.csv":               "unposted drafts",
	"saved_posts.csv":          "other people's posts you saved",
	"saved_comments.csv":       "other people's comments you saved",
	"likes.json":               "other people's posts you liked",
	"like.js":                  "other people's posts you liked",
}

func isTweets(base string) bool {
	return base == "tweet.js" || base == "tweets.js" ||
		(strings.HasPrefix(base, "tweets-part") && strings.HasSuffix(base, ".js")) ||
		(strings.HasPrefix(base, "tweet-part") && strings.HasSuffix(base, ".js"))
}

func (in *Input) readZip(name string, body []byte) bool {
	z, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
	if err != nil {
		in.gap("%s: %v", name, err)
		return true
	}
	before := len(in.Posts) + len(in.Skipped)
	for _, f := range z.File {
		if f.FileInfo().IsDir() || !wanted(f.Name) || f.UncompressedSize64 > tooBig {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			in.gap("%s/%s: %v", name, f.Name, err)
			continue
		}
		inner, err := io.ReadAll(io.LimitReader(rc, tooBig))
		rc.Close()
		if err != nil {
			in.gap("%s/%s: %v", name, f.Name, err)
			continue
		}
		in.readFile(f.Name, inner, false)
	}
	return len(in.Posts)+len(in.Skipped) > before
}

func (in *Input) readTar(name string, body []byte) bool {
	gz, err := gzip.NewReader(bytes.NewReader(body))
	if err != nil {
		in.gap("%s: %v", name, err)
		return true
	}
	before := len(in.Posts) + len(in.Skipped)
	t := tar.NewReader(gz)
	for {
		h, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			in.gap("%s: %v", name, err)
			break
		}
		if h.Typeflag != tar.TypeReg || !wanted(h.Name) || h.Size > tooBig {
			continue
		}
		inner, err := io.ReadAll(io.LimitReader(t, tooBig))
		if err != nil {
			in.gap("%s/%s: %v", name, h.Name, err)
			continue
		}
		in.readFile(h.Name, inner, false)
	}
	return len(in.Posts)+len(in.Skipped) > before
}

func (in *Input) add(p Post, kind string) {
	p.Text = own(p.Text)
	if strings.TrimSpace(p.Text) == "" {
		return
	}
	in.Posts = append(in.Posts, p)
	in.Kinds[kind]++
}

func (in *Input) addText(name, body string) {
	in.add(Post{Text: body, Where: name, Kind: TextFile}, "text files")
}

func (in *Input) gap(format string, args ...any) {
	text := fmt.Sprintf(format, args...)
	for _, g := range in.Gaps {
		if g == text {
			return
		}
	}
	in.Gaps = append(in.Gaps, text)
}