fix(evaluator): derived --config, region fallback, base-id regex, llm-context

src/cli/primitives/EvaluatorPrimitive.ts

@@ -1,4 +1,11 @@
-import { ConflictError, ResourceNotFoundError, createConfigIO, findConfigRoot, serializeResult, toError } from '../../lib';
+import {
+  ConflictError,
+  ResourceNotFoundError,
+  createConfigIO,
+  findConfigRoot,
+  serializeResult,
+  toError,
+} from '../../lib';
 import type { Result } from '../../lib/result';
 import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
 import {
@@ -553,6 +560,12 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
               let resolvedLevel: EvaluationLevel | undefined = levelResult?.data;
 
               if (evalType === 'derived') {
+                // --config carries a full evaluator config for other types; a derived
+                // evaluator is built from --base-evaluator-id + --model, so reject
+                // --config here rather than silently ignoring it.
+                if (cliOptions.config) {
+                  fail('--config is not supported with --type derived; use --base-evaluator-id and --model');
+                }
                 if (!cliOptions.baseEvaluatorId) {
                   fail('--base-evaluator-id is required for --type derived');
                 }
@@ -796,16 +809,9 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
    * GetEvaluator instead of asking the customer to know it.
    */
   private async resolveBaseEvaluatorLevel(baseEvaluatorId: string): Promise<EvaluationLevel> {
-    // The deploy target region is preferred, but the file may not exist yet
-    // (evaluators can be added before any deploy), so fall back to the environment.
-    let savedRegion: string | undefined;
-    try {
-      const targets = await createConfigIO().resolveAWSDeploymentTargets();
-      savedRegion = targets[0]?.region;
-    } catch {
-      savedRegion = undefined;
-    }
-    const region = savedRegion ?? process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION;
+    // A fresh project has no saved deploy targets, so resolve the region directly
+    // from the environment/profile fallback (env vars, then the AWS profile's region).
+    const region = await createConfigIO().resolveRegionFallback();
     if (!region) {
       throw new Error(
         `Could not resolve an AWS region to look up "${baseEvaluatorId}". Set AWS_REGION or pass --level explicitly.`

src/lib/schemas/io/config-io.ts

@@ -201,8 +201,10 @@ export class ConfigIO {
 
   /**
    * Resolve a fallback region from environment variables or AWS profile config.
+   * Public so callers that need a region before any deploy target is saved (e.g.
+   * resolving a derived evaluator's level) can reuse the same precedence.
    */
-  private async resolveRegionFallback(): Promise<string | undefined> {
+  async resolveRegionFallback(): Promise<string | undefined> {
     // Check env vars first
     const envRegion = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION;
     if (envRegion && AgentCoreRegionSchema.safeParse(envRegion).success) {

src/schema/llm-compacted/agentcore.ts

@@ -280,11 +280,25 @@ interface Evaluator {
   name: string; // @regex ^[a-zA-Z][a-zA-Z0-9_]{0,47}$ @min 1 @max 48
   level: 'SESSION' | 'TRACE' | 'TOOL_CALL';
   description?: string;
-  config: { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never } | { llmAsAJudge?: never; codeBased: CodeBasedConfig };
+  config: EvaluatorConfig; // exactly one of llmAsAJudge | codeBased | derived
   kmsKeyArn?: string;
   tags?: Tags;
 }
 
+// Exactly one arm present.
+type EvaluatorConfig =
+  | { llmAsAJudge: LlmAsAJudgeConfig; codeBased?: never; derived?: never }
+  | { llmAsAJudge?: never; codeBased: CodeBasedConfig; derived?: never }
+  | { llmAsAJudge?: never; codeBased?: never; derived: DerivedEvaluatorConfig };
+
+// A derived evaluator reuses a managed base evaluator's logic (a Builtin.* or
+// ThirdParty.<Provider>.<Metric> metric) on the customer's own model. The base
+// owns the prompt + scoring; level must match the base's (resolved at add time).
+interface DerivedEvaluatorConfig {
+  baseEvaluatorId: string; // "Builtin.<Metric>" or "ThirdParty.<Provider>.<Metric>"
+  model: string; // Bedrock inference profile ID
+}
+
 interface LlmAsAJudgeConfig {
   modelProvider?: 'Bedrock' | 'OpenResponses'; // Defaults to Bedrock when omitted
   model: string; // Bedrock model ID/ARN or OpenAI model ID

src/schema/schemas/primitives/evaluator.ts

@@ -95,7 +95,10 @@ export type LlmAsAJudgeConfig = z.infer<typeof LlmAsAJudgeConfigSchema>;
 // (Builtin.<Metric>). The base owns the prompt and scoring; the customer supplies
 // only the model. The evaluator's `level` must match the base's level (resolved at
 // add time via GetEvaluator), so it lives on the top-level Evaluator, not here.
-export const BASE_EVALUATOR_ID_PATTERN = /^(ThirdParty|Builtin)\.[A-Za-z0-9._-]+$/;
+// Builtin.<Metric> or ThirdParty.<Provider>.<Metric>. Each segment must be
+// non-empty alphanumeric — rejects malformed ids like "ThirdParty.DeepEval",
+// "ThirdParty..ToolUse", or "ThirdParty.DeepEval.".
+export const BASE_EVALUATOR_ID_PATTERN = /^(Builtin\.[A-Za-z0-9]+|ThirdParty\.[A-Za-z0-9]+\.[A-Za-z0-9]+)$/;
 
 export const DerivedEvaluatorConfigSchema = z.object({
   baseEvaluatorId: z