Skip to content

refactor: extract shared codegen helpers into internal/genhelper and dedup cpp/csharp/go loaders #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Kybxd merged 4 commits into from
Jun 17, 2026
Merged

refactor: extract shared codegen helpers into internal/genhelper and dedup cpp/csharp/go loaders #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1b5b7e8
refactor: extract shared codegen helpers into internal/genhelper and …
Kybxd Jun 16, 2026
c58e517
ci(cpp): improve vcpkg caching for toolchain updates
Kybxd Jun 16, 2026
b6587cb
ci: remove comments from testing-cpp workflow
Kybxd Jun 16, 2026
c23545a
refactor(genhelper): unify MapKeySlice into a single generic type acr…
Kybxd Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions .github/workflows/testing-cpp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,22 +118,29 @@ jobs:
}
EOF

- name: Compute vcpkg cache tag
id: vcpkg_tag
shell: bash
run: echo "image=${ImageVersion:-noimg}" >> "$GITHUB_OUTPUT"

- name: Cache vcpkg_installed
id: cache-vcpkg-installed
uses: actions/cache@v4
with:
path: ${{ env.VCPKG_INSTALLED_DIR }}
key: vcpkg-installed-${{ runner.os }}-${{ matrix.triplet }}-${{ matrix.config.vcpkg-commit }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }}
key: vcpkg-installed-${{ runner.os }}-${{ steps.vcpkg_tag.outputs.image }}-${{ matrix.triplet }}-${{ matrix.config.vcpkg-commit }}-${{ hashFiles('test/cpp-tableau-loader/vcpkg.json') }}

- name: Export GitHub Actions cache env (for vcpkg x-gha)
if: matrix.config.label == 'modern'
uses: actions/github-script@v7
with:
script: |
core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');

- name: Setup vcpkg & install protobuf
# Disable lukka/run-vcpkg's auto-injected `x-gha` binary cache:
# the `x-gha` provider only exists in vcpkg ≳ 2023, while our
# legacy matrix row pins a 2022-era vcpkg snapshot that errors
# out with "unknown binary provider type". `actions/cache` above
# already caches vcpkg_installed/, so we lose nothing here.
# `lukka/run-vcpkg` honours a pre-set VCPKG_BINARY_SOURCES.
env:
VCPKG_BINARY_SOURCES: clear
VCPKG_BINARY_SOURCES: ${{ matrix.config.label == 'legacy-v3' && 'clear' || 'clear;x-gha,readwrite' }}
uses: lukka/run-vcpkg@v11
with:
vcpkgGitCommitId: ${{ matrix.config.vcpkg-commit }}
Expand Down
125 changes: 14 additions & 111 deletions cmd/protoc-gen-cpp-tableau-loader/helper/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"strings"

"github.com/iancoleman/strcase"
"github.com/tableauio/loader/internal/genhelper"
"github.com/tableauio/tableau/proto/tableaupb"
"google.golang.org/protobuf/compiler/protogen"
"google.golang.org/protobuf/proto"
Expand All @@ -14,33 +15,17 @@ import (

func GenerateFileHeader(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, version string) {
GenerateCommonHeader(gen, g, version)
if file.Proto.GetOptions().GetDeprecated() {
g.P("// ", file.Desc.Path(), " is a deprecated file.")
} else {
g.P("// source: ", file.Desc.Path())
}
genhelper.GenerateSourcePath(file, g)
}

func GenerateCommonHeader(gen *protogen.Plugin, g *protogen.GeneratedFile, version string) {
g.P("// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT.")
g.P("// versions:")
g.P("// - protoc-gen-cpp-tableau-loader v", version)
g.P("// - protoc ", protocVersion(gen))
g.P("// - protoc ", genhelper.ProtocVersion(gen))
g.P("// clang-format off")
}

func protocVersion(gen *protogen.Plugin) string {
v := gen.Request.GetCompilerVersion()
if v == nil {
return "(unknown)"
}
var suffix string
if s := v.GetSuffix(); s != "" {
suffix = "-" + s
}
return fmt.Sprintf("v%d.%d.%d%s", v.GetMajor(), v.GetMinor(), v.GetPatch(), suffix)
}

func ParseCppFieldName(fd protoreflect.FieldDescriptor) string {
return escapeIdentifier(string(fd.Name()))
}
Expand Down Expand Up @@ -174,103 +159,21 @@ func ParseLeveledMapPrefix(md protoreflect.MessageDescriptor, mapFd protoreflect
return mapFd.MapValue().Kind().String()
}

type MapKey struct {
Type string
Name string
FieldName string // multi-column index only (may be deduplicated, e.g., "Id" → "Id3")
OrigFieldName string // original FieldName before deduplication (empty if not renamed)
Fd protoreflect.FieldDescriptor // the map field descriptor this key belongs to
}
// MapKey aliases the cross-language shared key descriptor; see genhelper.MapKey.
type MapKey = genhelper.MapKey

type MapKeySlice []MapKey
// cppParamFormatter formats a key as a C++ parameter declaration, using a const
// reference for std::string ("const std::string& name", "int32_t id").
type cppParamFormatter struct{}

// AddMapKey appends a new map key to the slice, automatically deduplicating
// both Name (used as function parameter names) and FieldName (used as struct
// field names in LevelIndex key structs).
//
// Deduplication is needed because different map levels may share the same key
// name. For example, given the following nested proto maps where country_map
// and item_map both use "ID" as their key name:
//
// message Fruit4Conf {
// map<int32, Fruit> fruit_map = 1; // key field: "FruitType"
// message Fruit {
// map<int32, Country> country_map = 2; // key field: "ID"
// message Country {
// map<int32, Item> item_map = 3; // key field: "ID" ← same name!
// }
// }
// }
//
// Without dedup, the generated LevelIndex key struct would have duplicate
// field names, causing a compile error:
//
// struct LevelIndex_Fruit_Country_ItemKey {
// int32_t id; // key of protoconf.Fruit4Conf.fruit_map
// int32_t id; // key of protoconf.Fruit4Conf.Fruit.country_map
// int32_t id; // key of protoconf.Fruit4Conf.Fruit.Country.item_map — COMPILE ERROR!
// };
//
// With dedup, the conflicting name gets a numeric suffix (the 1-based position
// of the new key in the slice), producing valid C++ code:
//
// struct LevelIndex_Fruit_Country_ItemKey {
// int32_t fruit_type; // key of protoconf.Fruit4Conf.fruit_map
// int32_t id; // key of protoconf.Fruit4Conf.Fruit.country_map
// int32_t id3; // key of protoconf.Fruit4Conf.Fruit.Country.item_map (renamed from id)
// };
func (s MapKeySlice) AddMapKey(newKey MapKey) MapKeySlice {
if newKey.Name == "" {
newKey.Name = fmt.Sprintf("key%d", len(s)+1)
}
// Deduplicate Name (used as function parameter, e.g., "id" → "id3").
for _, key := range s {
if key.Name == newKey.Name {
newKey.Name = fmt.Sprintf("%s%d", newKey.Name, len(s)+1)
break
}
}
// Deduplicate FieldName (used as struct field, e.g., "Id" → "Id3").
// This is only relevant for multi-column indexes that generate LevelIndex
// key structs; single-column indexes leave FieldName empty.
if newKey.FieldName != "" {
for _, key := range s {
if key.FieldName == newKey.FieldName {
newKey.OrigFieldName = newKey.FieldName
newKey.FieldName = fmt.Sprintf("%s%d", newKey.FieldName, len(s)+1)
break
}
}
}
return append(s, newKey)
func (cppParamFormatter) FormatParam(key MapKey) string {
return ToConstRefType(key.Type) + " " + key.Name
}

// GenGetParams generates function parameters, which are the names listed in the function's definition.
func (s MapKeySlice) GenGetParams() string {
var params []string
for _, key := range s {
params = append(params, ToConstRefType(key.Type)+" "+key.Name)
}
return strings.Join(params, ", ")
}

// GenGetArguments generates function arguments, which are the real values passed to the function.
func (s MapKeySlice) GenGetArguments() string {
var params []string
for _, key := range s {
params = append(params, key.Name)
}
return strings.Join(params, ", ")
}

// GenOtherArguments generates function arguments for other value of std::tie.
func (s MapKeySlice) GenOtherArguments(other string) string {
var params []string
for _, key := range s {
params = append(params, other+"."+key.Name)
}
return strings.Join(params, ", ")
}
// MapKeySlice is the shared cross-language key slice (see genhelper.MapKeySlice)
// specialized with C++ parameter formatting. All slice methods (AddMapKey /
// GenGetParams / GenGetArguments / GenOtherArguments / ...) come from genhelper.
type MapKeySlice = genhelper.MapKeySlice[cppParamFormatter]

func Indent(depth int) string {
return strings.Repeat(" ", depth)
Expand Down
12 changes: 3 additions & 9 deletions cmd/protoc-gen-cpp-tableau-loader/messager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,9 @@ import (
"github.com/tableauio/loader/cmd/protoc-gen-cpp-tableau-loader/orderedmap"
"github.com/tableauio/loader/internal/extensions"
"github.com/tableauio/loader/internal/index"
"github.com/tableauio/tableau/proto/tableaupb"
"github.com/tableauio/loader/internal/options"
"google.golang.org/protobuf/compiler/protogen"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/descriptorpb"
)

// generateMessager generates protobuf message wrapped classes
Expand Down Expand Up @@ -56,9 +54,7 @@ func generateHppFileContent(file *protogen.File, g *protogen.GeneratedFile) {
g.P("namespace ", *namespace, " {")
var fileMessagers []string
for _, message := range file.Messages {
opts := message.Desc.Options().(*descriptorpb.MessageOptions)
worksheet := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions)
if worksheet != nil {
if options.IsWorksheet(message.Desc) {
genHppMessage(g, message)
messagerName := string(message.Desc.Name())
fileMessagers = append(fileMessagers, messagerName)
Expand Down Expand Up @@ -142,9 +138,7 @@ func generateCppFileContent(file *protogen.File, g *protogen.GeneratedFile) {

g.P("namespace ", *namespace, " {")
for _, message := range file.Messages {
opts := message.Desc.Options().(*descriptorpb.MessageOptions)
worksheet := proto.GetExtension(opts, tableaupb.E_Worksheet).(*tableaupb.WorksheetOptions)
if worksheet != nil {
if options.IsWorksheet(message.Desc) {
genCppMessage(g, message)
}
}
Expand Down
125 changes: 12 additions & 113 deletions cmd/protoc-gen-csharp-tableau-loader/helper/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"unicode"

"github.com/iancoleman/strcase"
"github.com/tableauio/loader/internal/genhelper"
"github.com/tableauio/tableau/proto/tableaupb"
"google.golang.org/protobuf/compiler/protogen"
"google.golang.org/protobuf/proto"
Expand All @@ -21,32 +22,12 @@ func GenerateFileHeader(gen *protogen.Plugin, file *protogen.File, g *protogen.G
g.P("// Code generated by protoc-gen-csharp-tableau-loader. DO NOT EDIT.")
g.P("// versions:")
g.P("// - protoc-gen-csharp-tableau-loader v", version)
g.P("// - protoc ", protocVersion(gen))
if file != nil {
if file.Proto.GetOptions().GetDeprecated() {
g.P("// ", file.Desc.Path(), " is a deprecated file.")
} else {
g.P("// source: ", file.Desc.Path())
}
}
g.P("// - protoc ", genhelper.ProtocVersion(gen))
genhelper.GenerateSourcePath(file, g)
g.P("// </auto-generated>")
g.P("#nullable enable")
}

// protocVersion returns the protoc compiler version string (e.g. "v3.19.3")
// extracted from the code generator request. Returns "(unknown)" if not available.
func protocVersion(gen *protogen.Plugin) string {
v := gen.Request.GetCompilerVersion()
if v == nil {
return "(unknown)"
}
var suffix string
if s := v.GetSuffix(); s != "" {
suffix = "-" + s
}
return fmt.Sprintf("v%d.%d.%d%s", v.GetMajor(), v.GetMinor(), v.GetPatch(), suffix)
}

// ParseIndexFieldName returns the C# property name for an index field descriptor.
// It delegates to ParseCsharpPropertyName to match protoc's C# naming convention.
func ParseIndexFieldName(fd protoreflect.FieldDescriptor) string {
Expand Down Expand Up @@ -321,100 +302,18 @@ func ParseLeveledMapPrefix(md protoreflect.MessageDescriptor, mapFd protoreflect
return mapFd.MapValue().Kind().String()
}

// MapKey represents a single key component in a map or composite index,
// holding the C# type, parameter name, original field name, and the
// associated protobuf field descriptor.
type MapKey struct {
Type string // C# type string (e.g. "int", "string")
Name string // parameter/variable name in generated code
FieldName string // multi-column index only (may be deduplicated, e.g., "Id" → "Id3")
OrigFieldName string // original FieldName before deduplication (empty if not renamed)
Fd protoreflect.FieldDescriptor // the map field descriptor this key belongs to
}
// MapKey aliases the cross-language shared key descriptor; see genhelper.MapKey.
type MapKey = genhelper.MapKey

@wenchy wenchy Jun 16, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

struct embeding is more appropriate (auto reuse methods).

@Kybxd Kybxd Jun 16, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MapKey has no method.


// MapKeySlice is an ordered collection of MapKey entries, providing methods
// to build function parameters, arguments, and custom formatted strings
// for code generation.
type MapKeySlice []MapKey
// csharpParamFormatter formats a key as a C# parameter declaration ("int id").
type csharpParamFormatter struct{}

// AddMapKey appends a new map key to the slice, automatically deduplicating
// both Name (used as function parameter names) and FieldName (used as struct
// field names in LevelIndex key structs).
//
// Deduplication is needed because different map levels may share the same key
// name. For example, given the following nested proto maps where country_map
// and item_map both use "ID" as their key name:
//
// message Fruit4Conf {
// map<int32, Fruit> fruit_map = 1; // key field: "FruitType"
// message Fruit {
// map<int32, Country> country_map = 2; // key field: "ID"
// message Country {
// map<int32, Item> item_map = 3; // key field: "ID" ← same name!
// }
// }
// }
//
// Without dedup, the generated LevelIndex key struct would have duplicate
// field names, causing a compile error:
//
// public readonly struct LevelIndex_Fruit_Country_ItemKey {
// public int Id { get; } // key of protoconf.Fruit4Conf.Fruit.country_map
// public int Id { get; } // key of protoconf.Fruit4Conf.Fruit.Country.item_map — COMPILE ERROR!
// }
//
// With dedup, the conflicting name gets a numeric suffix (the 1-based position
// of the new key in the slice), producing valid C# code:
//
// public readonly struct LevelIndex_Fruit_Country_ItemKey {
// public int Id { get; } // key of protoconf.Fruit4Conf.Fruit.country_map
// public int Id3 { get; } // key of protoconf.Fruit4Conf.Fruit.Country.item_map (renamed from Id)
// }
func (s MapKeySlice) AddMapKey(newKey MapKey) MapKeySlice {
if newKey.Name == "" {
newKey.Name = fmt.Sprintf("key%d", len(s)+1)
}
// Deduplicate Name (used as function parameter, e.g., "id" → "id3").
for _, key := range s {
if key.Name == newKey.Name {
newKey.Name = fmt.Sprintf("%s%d", newKey.Name, len(s)+1)
break
}
}
// Deduplicate FieldName (used as struct field, e.g., "Id" → "Id3").
// This is only relevant for multi-column indexes that generate LevelIndex
// key structs; single-column indexes leave FieldName empty.
if newKey.FieldName != "" {
for _, key := range s {
if key.FieldName == newKey.FieldName {
newKey.OrigFieldName = newKey.FieldName
newKey.FieldName = fmt.Sprintf("%s%d", newKey.FieldName, len(s)+1)
break
}
}
}
return append(s, newKey)
}
func (csharpParamFormatter) FormatParam(key MapKey) string { return key.Type + " " + key.Name }

// GenGetParams generates function parameters, which are the names listed in the function's definition.
func (s MapKeySlice) GenGetParams() string {
return s.GenCustom(func(key MapKey) string { return key.Type + " " + key.Name }, ", ")
}

// GenGetArguments generates function arguments, which are the real values passed to the function.
func (s MapKeySlice) GenGetArguments() string {
return s.GenCustom(func(key MapKey) string { return key.Name }, ", ")
}

// GenCustom generates a string by applying fn to each MapKey and joining
// the results with the given separator. Returns an empty string for empty slices.
func (s MapKeySlice) GenCustom(fn func(MapKey) string, sep string) string {
var params []string
for _, key := range s {
params = append(params, fn(key))
}
return strings.Join(params, sep)
}
// MapKeySlice is the shared cross-language key slice (see genhelper.MapKeySlice)
// specialized with C# parameter formatting. All slice methods (AddMapKey /
// GenGetParams / GenGetArguments / GenCustom / ...) come from genhelper.
type MapKeySlice = genhelper.MapKeySlice[csharpParamFormatter]

// Indent returns a string of 4*depth spaces, used for indenting generated
// C# code blocks at the specified nesting depth.
Expand Down
Loading
Loading