Add v6 .rm format support with multi-color highlights

annotations/highlight_color_test.go

@@ -0,0 +1,79 @@
+package annotations
+
+import (
+	"math"
+	"testing"
+
+	"github.com/juruen/rmapi/encoding/rm"
+)
+
+// TestHighlightRGBExplicitRGBA verifies that an explicit color_rgba (firmware
+// 3.6+, BrushColor == HighlightDynamic) takes precedence over the enum and
+// is normalized to [0,1].
+func TestHighlightRGBExplicitRGBA(t *testing.T) {
+	rgba := &[4]uint8{255, 128, 64, 200}
+	r, g, b := highlightRGB(rm.HighlightDynamic, rgba)
+	if !approxEq(r, 1.0) || !approxEq(g, 128.0/255) || !approxEq(b, 64.0/255) {
+		t.Errorf("RGBA override: got (%v, %v, %v); want (1.0, 0.502, 0.251)", r, g, b)
+	}
+}
+
+// TestHighlightRGBEnumMapping locks down the per-enum RGB choices so the
+// rendered output stays stable across releases.
+func TestHighlightRGBEnumMapping(t *testing.T) {
+	cases := []struct {
+		name     string
+		color    rm.BrushColor
+		wantR    float64
+		wantG    float64
+		wantB    float64
+	}{
+		{"yellow", rm.HighlightYellow, 1.0, 0.93, 0.0},
+		{"yellow2", rm.Yellow2, 1.0, 0.93, 0.0},
+		{"green", rm.HighlightGreen, 0.36, 0.86, 0.36},
+		{"green2", rm.Green2, 0.36, 0.86, 0.36},
+		{"pink", rm.HighlightPink, 1.0, 0.45, 0.75},
+		{"magenta", rm.Magenta, 1.0, 0.45, 0.75},
+		{"blue", rm.Blue, 0.40, 0.75, 1.0},
+		{"cyan", rm.Cyan, 0.40, 0.75, 1.0},
+		{"red", rm.Red, 1.0, 0.45, 0.45},
+	}
+	for _, c := range cases {
+		r, g, b := highlightRGB(c.color, nil)
+		if !approxEq(r, c.wantR) || !approxEq(g, c.wantG) || !approxEq(b, c.wantB) {
+			t.Errorf("%s: got (%v, %v, %v); want (%v, %v, %v)",
+				c.name, r, g, b, c.wantR, c.wantG, c.wantB)
+		}
+	}
+}
+
+// TestHighlightRGBUnknownFallback guarantees that any unrecognised color
+// code falls back to yellow — i.e. that geta exporting a future-firmware
+// .rm file never silently produces an invisible (white-on-white) highlight.
+func TestHighlightRGBUnknownFallback(t *testing.T) {
+	r, g, b := highlightRGB(rm.BrushColor(99), nil)
+	if !approxEq(r, 1.0) || !approxEq(g, 0.93) || !approxEq(b, 0.0) {
+		t.Errorf("unknown color fallback: got (%v, %v, %v); want yellow", r, g, b)
+	}
+}
+
+// TestHighlightRGBV3V5Compat verifies that the v5 highlighter stroke path
+// (which historically rendered yellow regardless of brush color) still
+// produces yellow when called with the legacy Black-default brush color.
+// Without this, v3/v5 highlight strokes would silently render in stroke
+// colors like grey from the brush color carried on the line.
+func TestHighlightRGBV3V5Compat(t *testing.T) {
+	// The HighlighterV5 callsite passes line.BrushColor; for legacy v5
+	// content this is typically Black (0). The helper falls through the
+	// switch and ends up at the yellow fallback.
+	r, g, b := highlightRGB(rm.Black, nil)
+	// Black is mapped to grey for the highlight rect path (visible),
+	// not yellow — locking down the documented choice.
+	if !approxEq(r, 0.7) || !approxEq(g, 0.7) || !approxEq(b, 0.7) {
+		t.Errorf("Black highlight: got (%v, %v, %v); want grey (0.7, 0.7, 0.7)", r, g, b)
+	}
+}
+
+func approxEq(a, b float64) bool {
+	return math.Abs(a-b) < 1e-9
+}

annotations/pdf.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-
+	"math"
 	"os"
 
 	"github.com/juruen/rmapi/archive"
@@ -43,8 +43,8 @@ func CreatePdfGenerator(zipName, outputFilePath string, options PdfGeneratorOpti
 	return &PdfGenerator{zipName: zipName, outputFilePath: outputFilePath, options: options}
 }
 
-func normalized(p1 rm.Point, ratioX float64) (float64, float64) {
-	return float64(p1.X) * ratioX, float64(p1.Y) * ratioX
+func normalized(p1 rm.Point, scale, xShift float64) (float64, float64) {
+	return float64(p1.X)*scale + xShift, float64(p1.Y) * scale
 }
 
 func (p *PdfGenerator) Generate() error {
@@ -92,8 +92,10 @@ func (p *PdfGenerator) Generate() error {
 	for _, pageAnnotations := range zip.Pages {
 		hasContent := pageAnnotations.Data != nil
 
-		// do not add a page when there are no annotations
-		if !p.options.AllPages && !hasContent {
+		// For PDFs, always include all pages so the full document is
+		// preserved with annotations overlaid. For notebooks (no source
+		// PDF), skip blank pages unless -a is given.
+		if !hasContent && !p.options.AllPages && p.pdfReader == nil {
 			continue
 		}
 		//1 based, redirected page
@@ -104,25 +106,33 @@ func (p *PdfGenerator) Generate() error {
 			return err
 		}
 
-		ratio := c.Height() / c.Width()
-
-		var scale float64
-		if ratio < 1.33 {
-			scale = c.Width() / DeviceWidth
-		} else {
-			scale = c.Height() / DeviceHeight
-		}
 		if page == nil {
 			log.Error.Fatal("page is null")
 		}
-
-		if err != nil {
-			return err
-		}
 		if !hasContent {
 			continue
 		}
 
+		var scale float64
+		var xShift float64
+		isV6Pdf := pageAnnotations.Data.Version == rm.V6 && pageAnnotations.DocPage >= 0
+
+		if isV6Pdf {
+			// v6 PDF-backed pages: DPI-based scale (226 DPI device → 72 DPI PDF).
+			// The v6 parser adds +Width/2 to X so that x=Width/2 is page center;
+			// xShift compensates so that maps to c.Width()/2 in PDF points.
+			scale = 72.0 / 226.0
+			xShift = c.Width()/2 - float64(rm.Width)/2*scale
+		} else {
+			// v3/v5, v6 blank/added pages, or notebooks
+			ratio := c.Height() / c.Width()
+			if ratio < 1.33 {
+				scale = c.Width() / DeviceWidth
+			} else {
+				scale = c.Height() / DeviceHeight
+			}
+		}
+
 		contentCreator := contentstream.NewContentCreator()
 		contentCreator.Add_q()
 
@@ -137,14 +147,15 @@ func (p *PdfGenerator) Generate() error {
 
 				if line.BrushType == rm.HighlighterV5 {
 					last := len(line.Points) - 1
-					x1, y1 := normalized(line.Points[0], scale)
-					x2, _ := normalized(line.Points[last], scale)
+					x1, y1 := normalized(line.Points[0], scale, xShift)
+					x2, _ := normalized(line.Points[last], scale, xShift)
 					// make horizontal lines only, use y1
 					width := scale * 30
 					y1 += width / 2
 
 					lineDef := annotator.LineAnnotationDef{X1: x1 - 1, Y1: c.Height() - y1, X2: x2, Y2: c.Height() - y1}
-					lineDef.LineColor = pdf.NewPdfColorDeviceRGB(1.0, 1.0, 0.0) //yellow
+					r, g, b := highlightRGB(line.BrushColor, nil)
+					lineDef.LineColor = pdf.NewPdfColorDeviceRGB(r, g, b)
 					lineDef.Opacity = 0.5
 					lineDef.LineWidth = width
 					ann, err := annotator.CreateLineAnnotation(lineDef)
@@ -155,11 +166,25 @@ func (p *PdfGenerator) Generate() error {
 				} else {
 					path := draw.NewPath()
 					for i := 0; i < len(line.Points); i++ {
-						x1, y1 := normalized(line.Points[i], scale)
+						x1, y1 := normalized(line.Points[i], scale, xShift)
 						path = path.AppendPoint(draw.NewPoint(x1, c.Height()-y1))
 					}
 
-					contentCreator.Add_w(float64(line.BrushSize*6.0 - 10.8))
+					var lineWidth float64
+					if pageAnnotations.Data.Version == rm.V6 {
+						// v6: per-point Width is in device pixels; average and scale to PDF points.
+						var totalW float64
+						for _, pt := range line.Points {
+							totalW += float64(pt.Width)
+						}
+						lineWidth = totalW / float64(len(line.Points)) * scale
+					} else {
+						lineWidth = float64(line.BrushSize*6.0 - 10.8)
+					}
+					if lineWidth < 0.1 {
+						lineWidth = 0.1
+					}
+					contentCreator.Add_w(lineWidth)
 
 					switch line.BrushColor {
 					case rm.Black:
@@ -168,6 +193,10 @@ func (p *PdfGenerator) Generate() error {
 						contentCreator.Add_rg(0.0, 0.0, 0.0)
 					case rm.Grey:
 						contentCreator.Add_rg(0.8, 0.8, 0.8)
+					case rm.Blue:
+						contentCreator.Add_rg(0.0, 0.38, 0.8)
+					case rm.Red:
+						contentCreator.Add_rg(0.85, 0.03, 0.03)
 					}
 
 					//TODO: use bezier
@@ -177,6 +206,32 @@ func (p *PdfGenerator) Generate() error {
 				}
 			}
 		}
+		for _, hl := range pageAnnotations.Data.Highlights {
+			for _, rect := range hl.Rects {
+				// Highlight rects don't have the +Width/2 X offset that strokes do;
+				// for v6 PDF pages, x=0 is at the horizontal center of the page.
+				hlXOff := xShift
+				if isV6Pdf {
+					hlXOff = c.Width() / 2
+				}
+				x1 := rect.X*scale + hlXOff
+				y1 := c.Height() - rect.Y*scale
+				x2 := (rect.X+rect.W)*scale + hlXOff
+				y2 := c.Height() - (rect.Y+rect.H)*scale
+
+				lineDef := annotator.LineAnnotationDef{X1: x1, Y1: y2, X2: x2, Y2: y1}
+				r, g, b := highlightRGB(hl.Color, hl.ColorRGBA)
+				lineDef.LineColor = pdf.NewPdfColorDeviceRGB(r, g, b)
+				lineDef.Opacity = 0.3
+				lineDef.LineWidth = math.Abs(y1 - y2)
+				ann, err := annotator.CreateLineAnnotation(lineDef)
+				if err != nil {
+					continue
+				}
+				page.AddAnnotation(ann)
+			}
+		}
+
 		contentCreator.Add_Q()
 		drawingOperations := contentCreator.Operations().String()
 		pageContentStreams, err := page.GetAllContentStreams()
@@ -256,3 +311,27 @@ func (p *PdfGenerator) addBackgroundPage(c *creator.Creator, pageNum int) (*pdf.
 	}
 	return page, nil
 }
+
+// highlightRGB resolves a v6 highlight color to RGB components in [0,1].
+// rgba (when non-nil) overrides the enum and carries an explicit color from
+// firmware 3.6+. Falls back to yellow for unknown codes.
+func highlightRGB(c rm.BrushColor, rgba *[4]uint8) (r, g, b float64) {
+	if rgba != nil {
+		return float64(rgba[0]) / 255, float64(rgba[1]) / 255, float64(rgba[2]) / 255
+	}
+	switch c {
+	case rm.HighlightYellow, rm.Yellow2:
+		return 1.0, 0.93, 0.0
+	case rm.HighlightGreen, rm.Green2:
+		return 0.36, 0.86, 0.36
+	case rm.HighlightPink, rm.Magenta:
+		return 1.0, 0.45, 0.75
+	case rm.Blue, rm.Cyan:
+		return 0.40, 0.75, 1.0
+	case rm.Red:
+		return 1.0, 0.45, 0.45
+	case rm.Black, rm.Grey, rm.GreyOverlap:
+		return 0.7, 0.7, 0.7
+	}
+	return 1.0, 0.93, 0.0
+}

archive/file.go

@@ -100,6 +100,31 @@ type PageTag struct {
 	Timestamp int64  `json:"timestamp"`
 }
 
+// CPageNumberValue is a CRDT numeric value used in firmware 3.0+ content files.
+type CPageNumberValue struct {
+	Timestamp string `json:"timestamp"`
+	Value     int    `json:"value"`
+}
+
+// CPageStringValue is a CRDT string value used in firmware 3.0+ content files.
+type CPageStringValue struct {
+	Timestamp string `json:"timestamp"`
+	Value     string `json:"value"`
+}
+
+// CPageEntry represents a single page in the cPages structure.
+type CPageEntry struct {
+	ID      string            `json:"id"`
+	Idx     CPageStringValue  `json:"idx"`
+	Deleted *CPageNumberValue `json:"deleted,omitempty"`
+	Redir   *CPageNumberValue `json:"redir,omitempty"`
+}
+
+// CPages is the firmware 3.0+ page list format, replacing pages/redirectionPageMap.
+type CPages struct {
+	Pages []CPageEntry `json:"pages"`
+}
+
 // Content represents the structure of a .content json file.
 type Content struct {
 	DummyDocument bool          `json:"dummyDocument"`
@@ -114,14 +139,16 @@ type Content struct {
 	// Orientation can take "portrait" or "landscape".
 	Orientation string `json:"orientation"`
 	PageCount   int    `json:"pageCount"`
-	// Pages is a list of page IDs
+	// Pages is a list of page IDs (legacy format)
 	Pages           []string  `json:"pages"`
 	PageTags        []PageTag `json:"pageTags"`
 	DocumentTags    []Tag     `json:"tags"`
 	RedirectionMap  []int     `json:"redirectionPageMap"`
-	TextScale       float64   `json:"textScale"`
+	TextScale            float64 `json:"textScale"`
 	CoverPageNumber      *int    `json:"coverPageNumber,omitempty"`
 	ViewBackgroundFilter *string `json:"viewBackgroundFilter,omitempty"`
+	// CPages is the firmware 3.0+ page list with CRDT ordering and PDF page mapping
+	CPagesData *CPages `json:"cPages,omitempty"`
 
 	Transform *Transform `json:"-"`
 }