Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -602,6 +602,12 @@
"model": "gemini-3.1-flash-lite"
}
},
"gemini-3.5-flash": {

Check warning on line 605 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "chat-base-3",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 608 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"gemma-4-31b-it": {
"extends": "chat-base-3",
"modelConfig": {
Expand All @@ -626,6 +632,12 @@
"model": "gemini-3-flash-preview"
}
},
"gemini-3.5-flash-base": {

Check warning on line 635 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"extends": "base",
"modelConfig": {
"model": "gemini-3.5-flash"

Check warning on line 638 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
},
"classifier": {
"extends": "base",
"modelConfig": {
Expand Down Expand Up @@ -868,6 +880,16 @@
"multimodalToolUse": true
}
},
"gemini-3.5-flash": {

Check warning on line 883 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"tier": "flash",
"family": "gemini-3",
"isPreview": false,
"isVisible": true,
"features": {
"thinking": false,
"multimodalToolUse": true
}
},
"gemini-2.5-pro": {
"tier": "pro",
"family": "gemini-2.5",
Expand Down Expand Up @@ -1020,9 +1042,44 @@
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1047 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false
},
"target": "gemini-2.5-flash"
}
]
},
"gemini-3.5-flash": {

Check warning on line 1057 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"default": "gemini-3.5-flash",

Check warning on line 1058 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
"contexts": [
{
"condition": {
"useGemini3_5Flash": false,
"hasAccessToPreview": false
},
"target": "gemini-2.5-flash"
},
{
"condition": {
"useGemini3_5Flash": false
},
"target": "gemini-3-flash-preview"
}
]
},
"gemini-2.5-flash": {
"default": "gemini-2.5-flash",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1082 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
}
]
},
Expand Down Expand Up @@ -1104,6 +1161,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"

Check warning on line 1168 in docs/reference/configuration.md

View workflow job for this annotation

GitHub Actions / Lint

Found sensitive keyword "gemini-3.5". Please make sure this change is appropriate to submit.
},
{
"condition": {
"hasAccessToPreview": false
Expand Down Expand Up @@ -1157,6 +1220,12 @@
"flash": {
"default": "gemini-3-flash-preview",
"contexts": [
{
"condition": {
"useGemini3_5Flash": true
},
"target": "gemini-3.5-flash"
},
{
"condition": {
"hasAccessToPreview": false
Expand Down
28 changes: 24 additions & 4 deletions evals/llm-judge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,30 @@ export class LLMJudge {

for (const res of rawResults) {
// Remove any punctuation the model might have appended
const cleanRes = res.replace(/[^A-Z]/g, '');
if (cleanRes.startsWith('YES')) yes++;
else if (cleanRes.startsWith('NO')) no++;
else other++;
const cleanRes = res.replace(/[^A-Z ]/g, '');
if (
cleanRes.includes('THE ANSWER IS YES') ||
cleanRes.includes('ANSWER IS YES') ||
Comment thread
DavidAPierce marked this conversation as resolved.
cleanRes.endsWith('YES')
) {
yes++;
} else if (
cleanRes.includes('THE ANSWER IS NO') ||
cleanRes.includes('ANSWER IS NO') ||
cleanRes.endsWith('NO')
) {
no++;
} else if (cleanRes.trim() === 'YES') {
yes++;
} else if (cleanRes.trim() === 'NO') {
no++;
} else {
// Fallback: look for YES or NO as standalone words or at the end
const words = cleanRes.split(/\s+/);
if (words.includes('YES')) yes++;
else if (words.includes('NO')) no++;
else other++;
}
}

// Pass if YES > NO and YES > OTHER (plurality)
Expand Down
4 changes: 4 additions & 0 deletions packages/core/src/availability/policyCatalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export interface ModelPolicyOptions {
useGemini31?: boolean;
useGemini31FlashLite?: boolean;
useCustomToolModel?: boolean;
useGemini3_5Flash?: boolean;
}

const DEFAULT_ACTIONS: ModelPolicyActionMap = {
Expand Down Expand Up @@ -94,6 +95,9 @@ export function getModelPolicyChain(
PREVIEW_GEMINI_MODEL,
options.useGemini31,
options.useCustomToolModel,
true,
undefined,
options.useGemini3_5Flash,
);
return [
definePolicy({
Expand Down
5 changes: 5 additions & 0 deletions packages/core/src/availability/policyHelpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export function resolvePolicyChain(
const useGemini31 = config.getGemini31LaunchedSync?.() ?? false;
const useCustomToolModel = config.getUseCustomToolModelSync?.() ?? false;
const hasAccessToPreview = config.getHasAccessToPreviewModel?.() ?? false;
const useGemini3_5Flash = config.hasGemini35FlashGAAccess?.() ?? false;

// Capture the original family intent before any normalization or early downgrade.
const isOriginallyGemini3 = isGemini3Model(modelFromConfig, config);
Expand All @@ -65,6 +66,7 @@ export function resolvePolicyChain(
useCustomToolModel,
hasAccessToPreview,
config,
useGemini3_5Flash,
),
);
const isAutoPreferred = normalizedPreferredModel
Expand All @@ -82,6 +84,7 @@ export function resolvePolicyChain(
const context = {
useGemini3_1: useGemini31,
useCustomTools: useCustomToolModel,
useGemini3_5Flash,
};

if (resolvedModel === DEFAULT_GEMINI_FLASH_LITE_MODEL) {
Expand Down Expand Up @@ -136,6 +139,7 @@ export function resolvePolicyChain(
userTier: config.getUserTier(),
useGemini31,
useCustomToolModel,
useGemini3_5Flash,
});
} else {
// User requested Gemini 3 but has no access. Proactively downgrade
Expand All @@ -146,6 +150,7 @@ export function resolvePolicyChain(
userTier: config.getUserTier(),
useGemini31,
useCustomToolModel,
useGemini3_5Flash,
});
}
} else {
Expand Down
1 change: 1 addition & 0 deletions packages/core/src/code_assist/experiments/flagNames.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export const ExperimentFlags = {
GEMINI_3_1_PRO_LAUNCHED: 45760185,
PRO_MODEL_NO_ACCESS: 45768879,
DEFAULT_REQUEST_TIMEOUT: 45773134,
GEMINI_3_5_FLASH_GA_LAUNCHED: 45780819,
} as const;

export type ExperimentFlagName =
Expand Down
32 changes: 32 additions & 0 deletions packages/core/src/config/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import {
DEFAULT_GEMINI_MODEL_AUTO,
PREVIEW_GEMINI_MODEL_AUTO,
PREVIEW_GEMINI_FLASH_MODEL,
DEFAULT_GEMINI_FLASH_MODEL,
} from './models.js';
import { Storage } from './storage.js';
import type { AgentLoopContext } from './agent-loop-context.js';
Expand Down Expand Up @@ -4346,3 +4347,34 @@ describe('ADKSettings', () => {
expect(config.getAgentSessionNoninteractiveEnabled()).toBe(true);
});
});

describe('hasGemini35FlashGAAccess model setting', () => {
const baseParams: ConfigParameters = {
sessionId: 'test',
targetDir: '.',
debugMode: false,
model: 'test-model',
cwd: '.',
};

it('should set DEFAULT_GEMINI_FLASH_MODEL and PREVIEW_GEMINI_FLASH_MODEL to gemini-3.5-flash if hasGemini35FlashGAAccess returns true', () => {
const config = new Config(baseParams);

// Set experiment to return true for GEMINI_3_5_FLASH_GA_LAUNCHED
config.setExperiments({
experimentIds: [],
flags: {
[ExperimentFlags.GEMINI_3_5_FLASH_GA_LAUNCHED]: {
boolValue: true,
},
},
});

// Call the method
const result = config.hasGemini35FlashGAAccess();
expect(result).toBe(true);

expect(DEFAULT_GEMINI_FLASH_MODEL).toBe('gemini-3.5-flash');
expect(PREVIEW_GEMINI_FLASH_MODEL).toBe('gemini-3.5-flash');
});
});
31 changes: 31 additions & 0 deletions packages/core/src/config/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ import {
isGemini2Model,
PREVIEW_GEMINI_FLASH_MODEL,
resolveModel,
setFlashModels,
} from './models.js';
import { shouldAttemptBrowserLaunch } from '../utils/browser.js';
import type { MCPOAuthConfig } from '../mcp/oauth-provider.js';
Expand Down Expand Up @@ -2054,6 +2055,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);

const isPreview = isPreviewModel(primaryModel, this);
Expand Down Expand Up @@ -2093,6 +2095,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.remaining;
}
Expand All @@ -2108,6 +2111,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.limit;
}
Expand All @@ -2123,6 +2127,7 @@ export class Config implements McpContext, AgentLoopContext {
this.getUseCustomToolModelSync(),
this.getHasAccessToPreviewModel(),
this,
this.hasGemini35FlashGAAccess(),
);
return this.modelQuotas.get(primaryModel)?.resetTime;
}
Expand Down Expand Up @@ -3537,6 +3542,32 @@ export class Config implements McpContext, AgentLoopContext {
);
}

/**
* Returns whether Gemini 3.5 Flash GA has been launched.
*
* Note: This method should only be called after startup, once experiments have been loaded.
*/
hasGemini35FlashGAAccess(): boolean {
const authType = this.contentGeneratorConfig?.authType;
const hasAccess = (() => {
if (this.isGemini31LaunchedForAuthType(authType)) {
return true;
}
return (
this.experiments?.flags[ExperimentFlags.GEMINI_3_5_FLASH_GA_LAUNCHED]
?.boolValue ?? false
);
})();
// Used to set default flash models based on access
// TODO: Remove once the experiment for 3_5 flash rollut can be cleaned up.
if (hasAccess) {
setFlashModels('gemini-3.5-flash', 'gemini-3.5-flash');
} else {
setFlashModels('gemini-3-flash-preview', 'gemini-2.5-flash');
}
return hasAccess;
}

/**
* Returns whether Gemini 3.1 has been launched.
*
Expand Down
Loading
Loading