starcoach-hackathon/zarinpal-analytics-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(growth): harden AI jobs and cost controls

Browse Source
This commit is contained in:
Amirhossein Khalili
2026-08-21 01:48:47 +03:30
parent 86e9d7c5f9
commit a15c04cac1
13 changed files with 366 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/growthreport/orchestrator.go
View File

@@ -15,6 +15,8 @@ import (
var ErrDisabled = errors.New("growth report: disabled")
const MaxProviderRequests int64 = 4
type ServiceConfig struct {
Enabled bool
Model, PromptVersion, SchemaVersion string
@@ -42,13 +44,15 @@ func (s *Service) Create(ctx context.Context, userID, merchantID int64, request
if err != nil {
return Record{}, false, err
}
fingerprint := growth.Fingerprint(growth.FingerprintInput{MerchantID: merchantID, TerminalID: request.TerminalID, DatasetWatermark: watermark, ForecastStart: start, HorizonDays: 90, CatalogVersion: growth.CatalogVersion, AnalyticsVersion: growth.AnalyticsVersion, PromptVersion: s.config.PromptVersion, SchemaVersion: s.config.SchemaVersion, Model: s.config.Model, ModelConfigVersion: growth.DefaultModelConfigVersion})
fingerprint := s.fingerprint(merchantID, request.TerminalID, watermark, start)
var refreshDay *time.Time
if request.Refresh {
day := tehranMidnight(now)
refreshDay = &day
}
record, cached, err := s.repository.CreateOrReuse(ctx, CreateRecord{ID: newReportID(), MerchantID: merchantID, TerminalID: request.TerminalID, Fingerprint: fingerprint, DatasetWatermark: watermark, ForecastStartDate: &start, CatalogVersion: growth.CatalogVersion, AnalyticsVersion: growth.AnalyticsVersion, PromptVersion: s.config.PromptVersion, SchemaVersion: s.config.SchemaVersion, Model: s.config.Model, ManualRefresh: request.Refresh, RefreshDay: refreshDay, IdempotencyKey: idempotencyKey, RequestHash: RequestHash(merchantID, request.TerminalID, request.Refresh)})
record, cached, err := s.repository.CreateOrReuse(ctx, CreateRecord{ID: newReportID(), MerchantID: merchantID, TerminalID: request.TerminalID, Fingerprint: fingerprint, DatasetWatermark: watermark, ForecastStartDate: &start, CatalogVersion: growth.CatalogVersion, AnalyticsVersion: growth.AnalyticsVersion, PromptVersion: s.config.PromptVersion, SchemaVersion: s.config.SchemaVersion, Model: s.config.Model, ManualRefresh: request.Refresh, RefreshDay: refreshDay, IdempotencyKey: idempotencyKey, RequestHash: RequestHash(merchantID, request.TerminalID, request.Refresh)}, func() error {
return s.coordinator.AllowUncached(ctx, merchantID, now)
})
if err != nil {
if errors.Is(err, ErrRefreshLimited) {
next, quotaErr := s.repository.NextRefreshAt(ctx, merchantID, now)
@@ -59,12 +63,6 @@ func (s *Service) Create(ctx context.Context, userID, merchantID int64, request
}
return Record{}, false, err
}
if !cached {
if err := s.coordinator.AllowUncached(ctx, merchantID, now); err != nil {
_ = s.repository.Complete(ctx, record.ReportID, StatusFailed, json.RawMessage(`{}`), nil, Usage{}, "UNCACHED_SCOPE_LIMIT")
return Record{}, false, err
}
}
record.Cached = cached
record.NextRefreshAt, err = s.repository.NextRefreshAt(ctx, merchantID, now)
return record, cached, err
@@ -80,11 +78,21 @@ func (s *Service) Latest(ctx context.Context, merchantID int64, terminalID strin
record, err := s.repository.Latest(ctx, merchantID, terminalID)
if err == nil {
record.Cached = true
watermark, start, metadataErr := s.repository.DatasetMetadata(ctx)
if metadataErr != nil {
record.Stale = true
} else {
record.Stale = record.Fingerprint != s.fingerprint(merchantID, terminalID, watermark, start)
}
record.NextRefreshAt, _ = s.repository.NextRefreshAt(ctx, merchantID, s.now())
}
return record, err
}
func (s *Service) fingerprint(merchantID int64, terminalID, watermark string, start time.Time) string {
return growth.Fingerprint(growth.FingerprintInput{MerchantID: merchantID, TerminalID: terminalID, DatasetWatermark: watermark, ForecastStart: start, HorizonDays: 90, CatalogVersion: growth.CatalogVersion, AnalyticsVersion: growth.AnalyticsVersion, PromptVersion: s.config.PromptVersion, SchemaVersion: s.config.SchemaVersion, Model: s.config.Model, ModelConfigVersion: growth.DefaultModelConfigVersion})
}
type Processor struct {
repository *Repository
builder Builder
@@ -108,10 +116,10 @@ func (p *Processor) ProcessNext(ctx context.Context) error {
}
analysis, err := p.builder.Build(ctx, record.MerchantID, terminal)
if err != nil {
_ = p.repository.Complete(ctx, record.ReportID, StatusFailed, json.RawMessage(`{}`), nil, Usage{}, "ANALYTICS_FAILED")
_ = p.repository.CompleteClaimed(ctx, record.ReportID, record.ClaimToken, StatusFailed, json.RawMessage(`{}`), nil, Usage{}, "ANALYTICS_FAILED")
return err
}
if err := p.repository.StoreEvidence(ctx, record.ReportID, analysis.Encoded); err != nil {
if err := p.repository.StoreEvidence(ctx, record.ReportID, record.ClaimToken, analysis.Encoded); err != nil {
return err
}
copy := analysis.Fallback
@@ -119,30 +127,41 @@ func (p *Processor) ProcessNext(ctx context.Context) error {
failure := "AI_DISABLED_OR_UNAVAILABLE"
if analysis.ShouldCallAI && p.aiEnabled && p.avalai != nil {
reservedTokens := EstimatedTokenReservation(analysis.Encoded)
if err := p.coordinator.ReserveBudget(ctx, reservedTokens, p.now()); err == nil {
release, enterErr := p.coordinator.Enter(ctx)
if enterErr == nil {
release, enterErr := p.coordinator.Enter(ctx)
if enterErr == nil {
defer release()
if err := p.coordinator.ReserveBudget(ctx, MaxProviderRequests, reservedTokens, p.now()); err == nil {
generated, usage, aiErr := p.avalai.GenerateWithTools(ctx, analysis.Encoded, analysis.KnownEvidence, NewEvidenceTools(analysis))
release()
usage.ReservedTokens = reservedTokens
if aiErr == nil {
slog.Info("growth AI generation complete", "report_id", record.ReportID, "input_tokens", usage.InputTokens, "output_tokens", usage.OutputTokens, "latency_ms", usage.LatencyMS)
return p.repository.Complete(ctx, record.ReportID, StatusReady, analysis.Encoded, &generated, usage, "")
slog.Info("growth AI generation complete", "report_id", record.ReportID, "provider_requests", usage.ProviderRequests, "reserved_tokens", usage.ReservedTokens, "input_tokens", usage.InputTokens, "output_tokens", usage.OutputTokens, "latency_ms", usage.LatencyMS)
return p.repository.CompleteClaimed(ctx, record.ReportID, record.ClaimToken, StatusReady, analysis.Encoded, &generated, usage, "")
}
slog.Warn("growth AI output rejected", "report_id", record.ReportID, "reason", "AI_REJECTED")
slog.Warn("growth AI output rejected", "report_id", record.ReportID, "reason", "AI_REJECTED", "provider_requests", usage.ProviderRequests, "reserved_tokens", usage.ReservedTokens, "input_tokens", usage.InputTokens, "output_tokens", usage.OutputTokens, "latency_ms", usage.LatencyMS)
failure = "AI_REJECTED"
return p.repository.CompleteClaimed(ctx, record.ReportID, record.ClaimToken, status, analysis.Encoded, &copy, usage, failure)
} else {
failure = "AI_BUDGET_EXHAUSTED"
slog.Warn("growth AI skipped", "report_id", record.ReportID, "reason", failure)
}
} else {
failure = "AI_BUDGET_EXHAUSTED"
failure = "AI_CONCURRENCY_UNAVAILABLE"
slog.Warn("growth AI skipped", "report_id", record.ReportID, "reason", failure)
}
}
if !analysis.ShouldCallAI || !p.aiEnabled {
slog.Info("growth deterministic fallback used", "report_id", record.ReportID, "reason", failure)
}
return p.repository.Complete(ctx, record.ReportID, status, analysis.Encoded, &copy, Usage{}, failure)
return p.repository.CompleteClaimed(ctx, record.ReportID, record.ClaimToken, status, analysis.Encoded, &copy, Usage{}, failure)
}
func EstimatedTokenReservation(evidence json.RawMessage) int64 { return int64(2*len(evidence) + 3200) }
// EstimatedTokenReservation reserves for two complete attempts, each with two
// provider turns, before any network call. Bytes are deliberately counted as
// tokens and the fixed allowance covers tool schemas, instructions, tool
// outputs, and every turn's 1,600-token output ceiling.
func EstimatedTokenReservation(evidence json.RawMessage) int64 {
return int64(12*len(evidence) + 32000)
}
func newReportID() string {
data := make([]byte, 12)