Skip to content

Fix CID conflicts #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mzueva wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix CID conflicts #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .changeset/fix-library-routing.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
'@platforma-open/milaboratories.mixcr-amplicon-alignment.workflow': patch
---

Fix intermittent MiXCR failures (`Can't find library for library + custom`) and `CIDConflictError` during body re-evaluation.

- Give `mixcr-analyze.tpl.tengo` a unique `hash_override` UUID. It was copy-pasted from `mixcr-clonotyping`, so the two templates were being deduplicated as one on the Platforma backend — silently serving the wrong bytecode.
- Route the reference library as a dedicated `extra.referenceLibrary` field instead of embedding it inside `extra.params`. Matches the mixcr-clonotyping pattern.
- Build params JSON resources with canonical (key-sorted) encoding so CIDs are stable across body re-evaluations. Tengo `json.encode` iterates Go map keys in random order, which made `stopCodonReplacements`, `aminoAcidSeqColumnPairs`, and schema maps produce different bytes each pass.
22 changes: 22 additions & 0 deletions workflow/src/canonical-json.lib.tengo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
// Deterministic JSON resource construction for stable CIDs.
//
// The SDK's smart.createJsonResource uses json.encode, which iterates Tengo
// map keys in Go's random hash order. Under BlockModelV3 body re-evaluation
// this produces different bytes (and therefore different CIDs) for the same
// logical value, triggering CIDConflictError when pframes.processColumn
// tries to reuse a previously-assigned field slot.
//
// The SDK's :canonical module sorts keys recursively — this wrapper builds
// a json/object resource from those canonical bytes directly.

canonical := import("@platforma-sdk/workflow-tengo:canonical")
constants := import("@platforma-sdk/workflow-tengo:constants")
smart := import("@platforma-sdk/workflow-tengo:smart")

jsonResource := func(value) {
return smart.createValueResource(constants.RTYPE_JSON, bytes(canonical.encode(value)))
}

export {
jsonResource: jsonResource
}
6 changes: 4 additions & 2 deletions workflow/src/main.tpl.tengo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ pframes := import("@platforma-sdk/workflow-tengo:pframes")
exec := import("@platforma-sdk/workflow-tengo:exec")
json := import("json")

canonicalJson := import(":canonical-json")

processTpl := assets.importTemplate(":process")
repseqioLibraryTpl := assets.importTemplate(":repseqio-library")

Expand Down Expand Up @@ -110,7 +112,7 @@ wf.body(func(args) {
sequenceFragments: fragments,
checksumVersion: 1
}]
referenceLibrary = string(json.encode(library))
referenceLibrary = smart.createJsonResource(library)

@gemini-code-assist gemini-code-assist Bot Apr 23, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

To ensure stable CIDs for the custom reference library, you should use canonicalJson.jsonResource here as well. The library object (defined on line 108) contains maps (e.g., anchorPoints), and using smart.createJsonResource (which uses standard json.encode) will result in non-deterministic byte sequences due to random map iteration order in Tengo/Go. This is consistent with the fixes applied to other params resources in this PR.

referenceLibrary = canonicalJson.jsonResource(library)

} else if mode == "libraryFile" {
fImport := file.importFile(args.libraryFile)
libraryImportHandle = fImport.handle
Expand All @@ -133,7 +135,7 @@ wf.body(func(args) {
referenceLibrary: referenceLibrary,
limitInput: limitInput,

params: smart.createJsonResource(maps.clone({
params: canonicalJson.jsonResource(maps.clone({
blockId: blockId,
perProcessMemGB: perProcessMemGB,
perProcessCPUs: perProcessCPUs,
Expand Down
9 changes: 3 additions & 6 deletions workflow/src/mixcr-analyze.tpl.tengo
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//tengo:hash_override D70EDB25-6FF6-4615-966D-B79B04B5751C
//tengo:hash_override B6265A3B-36E7-4403-955D-9431A6BB254E

// mixcr analyze

Expand Down Expand Up @@ -26,6 +26,7 @@ self.body(func(inputs) {
aggregationAxesNames := inputs[pConstants.AGGREGATION_AXES_NAMES_FIELD_NAME]

params := inputs.params
referenceLibrary := inputs.referenceLibrary
fileExtension := params.fileExtension
limitInput := inputs.limitInput
perProcessMemGB := inputs.perProcessMemGB
Expand Down Expand Up @@ -91,11 +92,7 @@ self.body(func(inputs) {
arg("--species").arg("custom").
arg("--library").arg(libraryFileName)

if is_string(params.referenceLibrary) {
mixcrCmdBuilder.writeFile(libraryFileName, params.referenceLibrary).saveFile(libraryFileName)
} else {
mixcrCmdBuilder.addFile(libraryFileName, params.referenceLibrary)
}
mixcrCmdBuilder.addFile(libraryFileName, referenceLibrary)

mixcrCmdBuilder.
arg("--rna").
Expand Down
7 changes: 2 additions & 5 deletions workflow/src/mixcr-export.tpl.tengo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ self.body(func(inputs) {
clnsFile := inputs[pConstants.VALUE_FIELD_NAME]

params := inputs.params
referenceLibrary := inputs.referenceLibrary
exportArgs := params.exportArgs

clonotypeKeyColumns := params.clonotypeKeyColumns
Expand Down Expand Up @@ -227,11 +228,7 @@ self.body(func(inputs) {
arg("clones.tsv").
saveFile("clones.tsv")

if is_string(params.referenceLibrary) {
mixcrCmdBuilder.writeFile(libraryFileName, params.referenceLibrary).saveFile(libraryFileName)
} else {
mixcrCmdBuilder.addFile(libraryFileName, params.referenceLibrary)
}
mixcrCmdBuilder.addFile(libraryFileName, referenceLibrary)

return mixcrCmdBuilder.
cacheHours(3).
Expand Down
17 changes: 9 additions & 8 deletions workflow/src/process.tpl.tengo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ mixcrExportTpl := assets.importTemplate(":mixcr-export")
aggregateByClonotypeKeyTpl := assets.importTemplate(":aggregate-by-clonotype-key")
exportReportTpl := assets.importTemplate(":export-report")
calculateExportSpecs := import(":calculate-export-specs")
canonicalJson := import(":canonical-json")

self.awaitState("InputsLocked")
self.awaitState("params", "ResourceReady")
Expand Down Expand Up @@ -212,17 +213,17 @@ self.body(func(inputs) {
traceSteps: [{type: "milaboratories.mixcr-amplicon-alignment", id: blockId, importance: 20, label: "MiXCR generic amplicon"}],

extra: {
params: maps.clone({
params: canonicalJson.jsonResource(maps.clone({
fileExtension: fileExtension,
referenceLibrary: referenceLibrary,
cloneClusteringMode: cloneClusteringMode,
hasUMI: hasUMI,
tagPattern: tagPattern,
assemblingFeature: params.assemblingFeature,
imputeGermline: params.imputeGermline,
badQualityThreshold: params.badQualityThreshold,
isLibraryFileGzipped: params.isLibraryFileGzipped
}, { removeUndefs: true }),
}, { removeUndefs: true })),
referenceLibrary: referenceLibrary,
limitInput: limitInput
},

Expand Down Expand Up @@ -274,10 +275,9 @@ self.body(func(inputs) {
exportOutputs,
{
extra: {
params: maps.clone({
params: canonicalJson.jsonResource(maps.clone({
clonotypeKeyColumns: clonotypeKeyColumns,
exportArgs: exportArgs,
referenceLibrary: referenceLibrary,
mixcrChains: mixcrChains,
mainIsProductiveColumn: mainIsProductiveColumn,
aminoAcidSeqColumns: aminoAcidSeqColumns,
Expand All @@ -286,7 +286,8 @@ self.body(func(inputs) {
stopCodonTypes: params.stopCodonTypes,
stopCodonReplacements: params.stopCodonReplacements,
isLibraryFileGzipped: params.isLibraryFileGzipped
}, { removeUndefs: true })
}, { removeUndefs: true })),
referenceLibrary: referenceLibrary
},
metaExtra: {
perProcessMemGB: perProcessMemGB
Expand Down Expand Up @@ -334,13 +335,13 @@ self.body(func(inputs) {
traceSteps: [{type: "milaboratories.mixcr-amplicon-alignment.aggregate", id: blockId + "." + chains, importance: 150, label: "Aggregate " + chains}],

extra: {
params: {
params: canonicalJson.jsonResource({
mainAbundanceColumnNormalized: mainAbundanceColumnNormalized,
mainAbundanceColumnUnnormalized: mainAbundanceColumnUnnormalized,
schemaPerClonotypeNoAggregates: columnsToSchema(columnsSpecPerClonotypeNoAggregates),
schemaPerClonotypeAggregates: columnsToSchema(columnsSpecPerClonotypeAggregates),
schemaPerSample: columnsToSchema(columnsSpecPerSample)
}
})
},
metaExtra: {
perProcessMemGB: perProcessMemGB
Expand Down
Loading