Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pkgs/sdk/server-ai/src/Config/ConfigFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,8 @@ private Func<LdAiConfig, ILdAiConfigTracker> TrackerFactoryFor(Context context,
context,
cfg.Model?.Name,
cfg.Provider?.Name,
cfg.Model?.ModelKey,
cfg.Model?.ModelVersion ?? 1,
graphKey);
}

Expand All @@ -326,7 +328,10 @@ private static LdAiConfigTypes.ModelConfig ParseModel(LdValue modelValue)
var name = modelValue.Get("name").AsString ?? "";
var parameters = LdValueObjectToDictionary(modelValue.Get("parameters"));
var custom = LdValueObjectToDictionary(modelValue.Get("custom"));
return new LdAiConfigTypes.ModelConfig(name, parameters, custom);
var modelKey = modelValue.Get("modelKey").AsString;
var modelVersionValue = modelValue.Get("modelVersion");
var modelVersion = modelVersionValue.IsNull ? 1 : modelVersionValue.AsInt;
return new LdAiConfigTypes.ModelConfig(name, parameters, custom, modelKey, modelVersion);
}

private static LdAiConfigTypes.ProviderConfig ParseProvider(LdValue providerValue)
Expand Down
20 changes: 19 additions & 1 deletion pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,23 @@ public sealed record ModelConfig
/// </summary>
public readonly IReadOnlyDictionary<string, LdValue> Custom;

internal ModelConfig(string name, IReadOnlyDictionary<string, LdValue> parameters, IReadOnlyDictionary<string, LdValue> custom)
/// <summary>
/// The stable, unique key of the model (used for direct lookup; distinct from
/// <see cref="Name"/>, which is not guaranteed unique).
/// </summary>
public readonly string ModelKey;

/// <summary>
/// The pinned version of the model that this config variation references.
/// </summary>
public readonly int ModelVersion;

internal ModelConfig(
string name,
IReadOnlyDictionary<string, LdValue> parameters,
IReadOnlyDictionary<string, LdValue> custom,
string modelKey = null,
int modelVersion = 1)
{
Name = name;
// Materialize into an ImmutableDictionary so a consumer that downcasts to
Expand All @@ -80,6 +96,8 @@ internal ModelConfig(string name, IReadOnlyDictionary<string, LdValue> parameter
// NotSupportedException at runtime, matching the typed read-only promise.
Parameters = parameters?.ToImmutableDictionary() ?? ImmutableDictionary<string, LdValue>.Empty;
Custom = custom?.ToImmutableDictionary() ?? ImmutableDictionary<string, LdValue>.Empty;
ModelKey = modelKey;
ModelVersion = modelVersion;
}
}

Expand Down
13 changes: 10 additions & 3 deletions pkgs/sdk/server-ai/src/LdAiConfigTracker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public class LdAiConfigTracker : ILdAiConfigTracker
/// </summary>
internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey,
string variationKey, int version, Context context, string modelName, string providerName,
string graphKey = null)
string modelKey = null, int modelVersion = 1, string graphKey = null)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_configKey = configKey ?? throw new ArgumentNullException(nameof(configKey));
Expand All @@ -96,6 +96,7 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf
{ "version", LdValue.Of(_version) },
{ "modelName", LdValue.Of(_modelName) },
{ "providerName", LdValue.Of(_providerName) },
{ "modelVersion", LdValue.Of(modelVersion) },
};
if (!string.IsNullOrEmpty(_graphKey))
{
Expand All @@ -105,6 +106,10 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf
{
trackDataBuilder.Add("variationKey", LdValue.Of(_variationKey));
}
if (!string.IsNullOrEmpty(modelKey))
{
trackDataBuilder.Add("modelKey", LdValue.Of(modelKey));
}
_trackData = LdValue.ObjectFrom(trackDataBuilder);

_resumptionToken = new Lazy<string>(BuildResumptionToken);
Expand All @@ -118,6 +123,7 @@ private string BuildResumptionToken()
// Utf8JsonWriter gives stable key ordering and avoids the runtime cost of
// anonymous-type reflection. The wire format omits empty optional fields so that
// resumption tokens round-trip exactly for configs that never carried them.
// modelName, providerName, modelKey, and modelVersion are not included.
using var stream = new MemoryStream();
using (var writer = new Utf8JsonWriter(stream))
{
Expand Down Expand Up @@ -426,7 +432,8 @@ private LdValue MergeTrackData(string key, LdValue value)
/// process or at a later time.
///
/// The reconstructed tracker will have empty model and provider names, as these are not
/// included in the resumption token.
/// included in the resumption token. modelKey and modelVersion are also not included;
/// modelVersion defaults to 1 on reconstructed trackers.
/// </summary>
/// <param name="token">the resumption token obtained from <see cref="ResumptionToken"/></param>
/// <param name="client">the LaunchDarkly client</param>
Expand Down Expand Up @@ -466,7 +473,7 @@ public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyC
}

return new LdAiConfigTracker(client, payload.RunId, payload.ConfigKey,
payload.VariationKey, payload.Version ?? 1, context, "", "", payload.GraphKey);
payload.VariationKey, payload.Version ?? 1, context, "", "", graphKey: payload.GraphKey);
}

private class ResumptionPayload
Expand Down
114 changes: 114 additions & 0 deletions pkgs/sdk/server-ai/test/ConfigFactoryParserTest.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System.Collections.Generic;
using LaunchDarkly.Sdk.Server.Ai.Config;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;
using Moq;
using Xunit;

namespace LaunchDarkly.Sdk.Server.Ai;
Expand Down Expand Up @@ -204,4 +206,116 @@ public void ParseAllFields_WithNoNewFields_ReturnsNullOrEmptyWithoutError()
Assert.Null(judgeConfig);
Assert.Null(evaluationMetricKey);
}

[Fact]
public void CompletionConfigReadsModelKeyAndVersionFromFlag()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var mockLogger = new Mock<ILogger>();
mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object);
mockClient.Setup(x =>
x.JsonVariation("model-config-with-key-version", It.IsAny<Context>(), It.IsAny<LdValue>()))
.Returns(LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["_ldMeta"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["enabled"] = LdValue.Of(true),
["variationKey"] = LdValue.Of("v1"),
["version"] = LdValue.Of(1)
}),
["model"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("gpt-4"),
["modelKey"] = LdValue.Of("my-model"),
["modelVersion"] = LdValue.Of(2)
}),
["provider"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("openai")
}),
["messages"] = LdValue.ArrayOf()
}));

var client = new LdAiClient(mockClient.Object);
var result = client.CompletionConfig("model-config-with-key-version", Context.New("user-key"));

Assert.Equal("my-model", result.Model.ModelKey);
Assert.Equal(2, result.Model.ModelVersion);
}

[Fact]
public void CompletionConfigDefaultsModelVersionWhenAbsent()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var mockLogger = new Mock<ILogger>();
mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object);
mockClient.Setup(x =>
x.JsonVariation("model-config-no-version", It.IsAny<Context>(), It.IsAny<LdValue>()))
.Returns(LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["_ldMeta"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["enabled"] = LdValue.Of(true),
["variationKey"] = LdValue.Of("v1"),
["version"] = LdValue.Of(1)
}),
["model"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("gpt-4")
}),
["provider"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("openai")
}),
["messages"] = LdValue.ArrayOf()
}));

var client = new LdAiClient(mockClient.Object);
var result = client.CompletionConfig("model-config-no-version", Context.New("user-key"));

Assert.Null(result.Model.ModelKey);
Assert.Equal(1, result.Model.ModelVersion);
}

[Fact]
public void CreateTrackerStampsModelKeyAndVersionOnTrackData()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var mockLogger = new Mock<ILogger>();
mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object);
mockClient.Setup(x =>
x.JsonVariation("my-config-key", It.IsAny<Context>(), It.IsAny<LdValue>()))
.Returns(LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["_ldMeta"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["enabled"] = LdValue.Of(true),
["variationKey"] = LdValue.Of("var-abc"),
["version"] = LdValue.Of(7)
}),
["model"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("gpt-4"),
["modelKey"] = LdValue.Of("my-model"),
["modelVersion"] = LdValue.Of(2)
}),
["provider"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
["name"] = LdValue.Of("openai")
}),
["messages"] = LdValue.ArrayOf()
}));

var client = new LdAiClient(mockClient.Object);
var context = Context.New("user-key");
var config = client.CompletionConfig("my-config-key", context);
var tracker = config.CreateTracker();
tracker.TrackSuccess();

mockClient.Verify(x => x.Track("$ld:ai:generation:success", context,
It.Is<LdValue>(d =>
d.Get("modelKey").AsString == "my-model" &&
d.Get("modelVersion").AsInt == 2),
1.0f), Times.Once);
}
}
2 changes: 1 addition & 1 deletion pkgs/sdk/server-ai/test/LdAiAgentGraphConfigTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ private static LdAiConfigTracker MakeTracker(ILaunchDarklyClient client, Context
string graphKey = null)
{
return new LdAiConfigTracker(client, Guid.NewGuid().ToString(), "config-key",
"v1", 1, context, "model", "provider", graphKey);
"v1", 1, context, "model", "provider", graphKey: graphKey);
}

// Test 1: Tracker created with graphKey → events include graphKey in data
Expand Down
4 changes: 3 additions & 1 deletion pkgs/sdk/server-ai/test/LdAiClientTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,9 @@ public void CreateTrackerFromResumptionTokenSetsEmptyModelAndProvider()
d.Get("variationKey").AsString == "var-1" &&
d.Get("version").AsInt == 2 &&
d.Get("modelName").AsString == "" &&
d.Get("providerName").AsString == ""),
d.Get("providerName").AsString == "" &&
d.Get("modelVersion").AsInt == 1 &&
d.Get("modelKey").IsNull),
1.0f), Times.Once);
}

Expand Down
78 changes: 78 additions & 0 deletions pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,84 @@ public void ResumptionTokenContainsExpectedFields()
// modelName and providerName should NOT be in the token
Assert.False(doc.RootElement.TryGetProperty("modelName", out _));
Assert.False(doc.RootElement.TryGetProperty("providerName", out _));
Assert.False(doc.RootElement.TryGetProperty("modelKey", out _));
Assert.False(doc.RootElement.TryGetProperty("modelVersion", out _));
}

[Fact]
public void TrackDataIncludesModelVersionByDefault()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var context = Context.New("key");

var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id",
"config-key", "variation-key", 3, context, "fakeModel", "fakeProvider");
tracker.TrackSuccess();

mockClient.Verify(x => x.Track("$ld:ai:generation:success", context,
It.Is<LdValue>(d => d.Get("modelVersion").AsInt == 1), 1.0f), Times.Once);
}

[Fact]
public void TrackDataIncludesModelKeyWhenSet()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var context = Context.New("key");

var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id",
"config-key", "variation-key", 3, context, "fakeModel", "fakeProvider",
modelKey: "my-model", modelVersion: 2);
tracker.TrackSuccess();

mockClient.Verify(x => x.Track("$ld:ai:generation:success", context,
It.Is<LdValue>(d =>
d.Get("modelKey").AsString == "my-model" &&
d.Get("modelVersion").AsInt == 2),
1.0f), Times.Once);
}

[Fact]
public void TrackDataOmitsModelKeyWhenEmpty()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var context = Context.New("key");

var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id",
"config-key", "variation-key", 3, context, "fakeModel", "fakeProvider",
modelKey: "", modelVersion: 3);
tracker.TrackSuccess();

mockClient.Verify(x => x.Track("$ld:ai:generation:success", context,
It.Is<LdValue>(d =>
d.Get("modelVersion").AsInt == 3 &&
d.Get("modelKey").IsNull),
1.0f), Times.Once);
}

[Fact]
public void FromResumptionTokenDefaultsModelVersionAndOmitsModelKey()
{
var mockClient = new Mock<ILaunchDarklyClient>();
var context = Context.New("key");

var payload = JsonSerializer.Serialize(new
{
runId = "test-run-id",
configKey = "test-key",
variationKey = "var-1",
version = 2,
});
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload));
var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('=');

var tracker = LdAiConfigTracker.FromResumptionToken(token, mockClient.Object, context);
tracker.TrackSuccess();

mockClient.Verify(x => x.Track("$ld:ai:generation:success", context,
It.Is<LdValue>(d =>
d.Get("modelVersion").AsInt == 1 &&
d.Get("modelKey").IsNull),
1.0f), Times.Once);
}

[Fact]
Expand Down
Loading