feat: rank growth events with grounded evidence
This commit is contained in:
@@ -22,12 +22,12 @@ func TestAvalAICountsFailedRetryRequestsAndUsage(t *testing.T) {
|
||||
_, _ = w.Write([]byte(`{"usage":{"input_tokens":7,"output_tokens":3},"estimated_cost":"0.01"}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\"},{\"evidenceId\":\"event_yalda\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\"},{\"evidenceId\":\"event_nowruz\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\"}]}"}]}],"usage":{"input_tokens":11,"output_tokens":5},"estimated_cost":"0.02"}`))
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"priorityPercent\":40,\"stance\":\"PROMOTE\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_yalda\",\"priorityPercent\":35,\"stance\":\"PREPARE\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_nowruz\",\"priorityPercent\":25,\"stance\":\"CAUTION\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\",\"insightBasis\":\"BUSINESS_PROFILE\"}]}"}]}],"usage":{"input_tokens":11,"output_tokens":5},"estimated_cost":"0.02"}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewAvalAIClient(AvalAIConfig{BaseURL: server.URL, APIKey: "test", Model: "test", Timeout: time.Second}, server.Client())
|
||||
_, usage, err := client.Generate(context.Background(), json.RawMessage(`{"events":[]}`), map[string]struct{}{"event_teacher": {}, "event_yalda": {}, "event_nowruz": {}})
|
||||
_, usage, err := client.Generate(context.Background(), json.RawMessage(`{"events":[]}`), profileKnownEvidence())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -41,17 +41,36 @@ func TestValidateAICopyAcceptsExactlyThreeGroundedPersianRecommendations(t *test
|
||||
tests := []struct {
|
||||
name string
|
||||
copy AICopy
|
||||
known KnownEvidence
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "grounded Persian recommendations", copy: AICopy{EventRecommendations: []EventRecommendation{{EvidenceID: "event_teacher", RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است."}, {EvidenceID: "event_yalda", RelevanceReason: "این رویداد با کانال فروش شما مرتبط است."}, {EvidenceID: "event_nowruz", RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است."}}}},
|
||||
{name: "numeric claim", copy: AICopy{EventRecommendations: []EventRecommendation{{EvidenceID: "event_teacher", RelevanceReason: "رشد ۲۰ درصدی پیش‌بینی می‌شود."}, {EvidenceID: "event_yalda", RelevanceReason: "این رویداد با کانال فروش شما مرتبط است."}, {EvidenceID: "event_nowruz", RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است."}}}, wantErr: true},
|
||||
{name: "duplicate evidence", copy: AICopy{EventRecommendations: []EventRecommendation{{EvidenceID: "event_teacher", RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است."}, {EvidenceID: "event_teacher", RelevanceReason: "این رویداد با کانال فروش شما مرتبط است."}, {EvidenceID: "event_nowruz", RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است."}}}, wantErr: true},
|
||||
{name: "unknown evidence", copy: AICopy{EventRecommendations: []EventRecommendation{{EvidenceID: "event_teacher", RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است."}, {EvidenceID: "event_yalda", RelevanceReason: "این رویداد با کانال فروش شما مرتبط است."}, {EvidenceID: "unknown", RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است."}}}, wantErr: true},
|
||||
{name: "not exactly three", copy: AICopy{EventRecommendations: []EventRecommendation{{EvidenceID: "event_teacher", RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است."}}}, wantErr: true},
|
||||
{name: "grounded Persian recommendations", copy: validRankedCopy()},
|
||||
{name: "numeric claim", copy: mutateRecommendation(validRankedCopy(), 0, func(item *EventRecommendation) {
|
||||
item.RelevanceReason = "رشد ۲۰ درصدی پیش‌بینی می‌شود."
|
||||
}), wantErr: true},
|
||||
{name: "duplicate evidence", copy: mutateRecommendation(validRankedCopy(), 1, func(item *EventRecommendation) { item.EvidenceID = "event_teacher" }), wantErr: true},
|
||||
{name: "unknown evidence", copy: mutateRecommendation(validRankedCopy(), 2, func(item *EventRecommendation) { item.EvidenceID = "unknown" }), wantErr: true},
|
||||
{name: "bad sum", copy: mutateRecommendation(validRankedCopy(), 2, func(item *EventRecommendation) { item.PriorityPercent = 24 }), wantErr: true},
|
||||
{name: "not descending", copy: AICopy{EventRecommendations: []EventRecommendation{
|
||||
{EvidenceID: "event_teacher", PriorityPercent: 35, Stance: StancePromote, RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
{EvidenceID: "event_yalda", PriorityPercent: 40, Stance: StancePrepare, RelevanceReason: "این رویداد با کانال فروش شما مرتبط است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
{EvidenceID: "event_nowruz", PriorityPercent: 25, Stance: StanceCaution, RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
}}, wantErr: true},
|
||||
{name: "basis mismatch", copy: mutateRecommendation(validRankedCopy(), 0, func(item *EventRecommendation) { item.InsightBasis = InsightBasisHistorical }), wantErr: true},
|
||||
{name: "historical evidence omitted", copy: validRankedCopy(), wantErr: true, known: KnownEvidence{"event_teacher": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_yalda": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_nowruz": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_history": {InsightBasis: InsightBasisHistorical, Promotable: true}}},
|
||||
{name: "promote material decline", copy: validRankedCopy(), wantErr: true, known: KnownEvidence{"event_teacher": {InsightBasis: InsightBasisBusinessProfile, Promotable: true, MaterialDecline: true}, "event_yalda": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_nowruz": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}}},
|
||||
{name: "promote prohibited posture", copy: validRankedCopy(), wantErr: true, known: KnownEvidence{"event_teacher": {InsightBasis: InsightBasisBusinessProfile, Promotable: false}, "event_yalda": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_nowruz": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}}},
|
||||
{name: "reason rune count not UTF-8 byte count", copy: mutateRecommendation(validRankedCopy(), 0, func(item *EventRecommendation) { item.RelevanceReason = strings.Repeat("ا", 200) })},
|
||||
{name: "reason rune limit exceeded", copy: mutateRecommendation(validRankedCopy(), 0, func(item *EventRecommendation) { item.RelevanceReason = strings.Repeat("ا", 321) }), wantErr: true},
|
||||
{name: "invalid stance", copy: mutateRecommendation(validRankedCopy(), 0, func(item *EventRecommendation) { item.Stance = "UNKNOWN" }), wantErr: true},
|
||||
{name: "not exactly three", copy: AICopy{EventRecommendations: validRankedCopy().EventRecommendations[:1]}, wantErr: true},
|
||||
}
|
||||
known := map[string]struct{}{"event_teacher": {}, "event_yalda": {}, "event_nowruz": {}}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
known := tt.known
|
||||
if known == nil {
|
||||
known = profileKnownEvidence()
|
||||
}
|
||||
err := ValidateAICopy(tt.copy, known)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ValidateAICopy() error = %v, wantErr %v", err, tt.wantErr)
|
||||
@@ -71,12 +90,12 @@ func TestAvalAIClientUsesBoundedStatelessRequest(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\"},{\"evidenceId\":\"event_yalda\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\"},{\"evidenceId\":\"event_nowruz\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\"}]}"}]}],"usage":{"input_tokens":10,"output_tokens":20}}`))
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"priorityPercent\":40,\"stance\":\"PROMOTE\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_yalda\",\"priorityPercent\":35,\"stance\":\"PREPARE\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_nowruz\",\"priorityPercent\":25,\"stance\":\"CAUTION\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\",\"insightBasis\":\"BUSINESS_PROFILE\"}]}"}]}],"usage":{"input_tokens":10,"output_tokens":20}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := NewAvalAIClient(AvalAIConfig{BaseURL: server.URL, APIKey: "test-key", Model: "test-model", Timeout: time.Second}, server.Client())
|
||||
copy, usage, err := client.Generate(context.Background(), json.RawMessage(`{"events":[{"evidenceId":"event_teacher"}]}`), map[string]struct{}{"event_teacher": {}, "event_yalda": {}, "event_nowruz": {}})
|
||||
copy, usage, err := client.Generate(context.Background(), json.RawMessage(`{"events":[{"evidenceId":"event_teacher"}]}`), profileKnownEvidence())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -116,12 +135,12 @@ func TestAvalAIClientExecutesAllowlistedToolThenStopsAfterSecondTurn(t *testing.
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"function_call","call_id":"call_1","name":"get_growth_context","arguments":"{\"evidenceId\":\"event_teacher\"}"}],"usage":{"input_tokens":2,"output_tokens":1}}`))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\"},{\"evidenceId\":\"event_yalda\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\"},{\"evidenceId\":\"event_nowruz\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\"}]}"}]}],"usage":{"input_tokens":3,"output_tokens":4}}`))
|
||||
_, _ = w.Write([]byte(`{"status":"completed","output":[{"type":"message","content":[{"type":"output_text","text":"{\"eventRecommendations\":[{\"evidenceId\":\"event_teacher\",\"priorityPercent\":40,\"stance\":\"PROMOTE\",\"relevanceReason\":\"این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_yalda\",\"priorityPercent\":35,\"stance\":\"PREPARE\",\"relevanceReason\":\"این رویداد با کانال فروش شما مرتبط است.\",\"insightBasis\":\"BUSINESS_PROFILE\"},{\"evidenceId\":\"event_nowruz\",\"priorityPercent\":25,\"stance\":\"CAUTION\",\"relevanceReason\":\"این بازه برای برنامه‌ریزی عرضه مناسب است.\",\"insightBasis\":\"BUSINESS_PROFILE\"}]}"}]}],"usage":{"input_tokens":3,"output_tokens":4}}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
executor := &testTools{}
|
||||
client := NewAvalAIClient(AvalAIConfig{BaseURL: server.URL, APIKey: "key", Model: "model", Timeout: time.Second}, server.Client())
|
||||
_, usage, err := client.GenerateWithTools(context.Background(), json.RawMessage(`{}`), map[string]struct{}{"event_teacher": {}, "event_yalda": {}, "event_nowruz": {}}, executor)
|
||||
_, usage, err := client.GenerateWithTools(context.Background(), json.RawMessage(`{}`), profileKnownEvidence(), executor)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -129,3 +148,21 @@ func TestAvalAIClientExecutesAllowlistedToolThenStopsAfterSecondTurn(t *testing.
|
||||
t.Fatalf("calls=%d turns=%d usage=%+v", executor.calls, turn, usage)
|
||||
}
|
||||
}
|
||||
|
||||
func validRankedCopy() AICopy {
|
||||
return AICopy{EventRecommendations: []EventRecommendation{
|
||||
{EvidenceID: "event_teacher", PriorityPercent: 40, Stance: StancePromote, RelevanceReason: "این مناسبت با الگوی فروش کسب‌وکار هم‌خوان است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
{EvidenceID: "event_yalda", PriorityPercent: 35, Stance: StancePrepare, RelevanceReason: "این رویداد با کانال فروش شما مرتبط است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
{EvidenceID: "event_nowruz", PriorityPercent: 25, Stance: StanceCaution, RelevanceReason: "این بازه برای برنامه‌ریزی عرضه مناسب است.", InsightBasis: InsightBasisBusinessProfile},
|
||||
}}
|
||||
}
|
||||
|
||||
func mutateRecommendation(copy AICopy, index int, mutate func(*EventRecommendation)) AICopy {
|
||||
copy.EventRecommendations = append([]EventRecommendation(nil), copy.EventRecommendations...)
|
||||
mutate(©.EventRecommendations[index])
|
||||
return copy
|
||||
}
|
||||
|
||||
func profileKnownEvidence() KnownEvidence {
|
||||
return KnownEvidence{"event_teacher": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_yalda": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}, "event_nowruz": {InsightBasis: InsightBasisBusinessProfile, Promotable: true}}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user