Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ldai/judge/judge.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type Config interface {

type Tracker interface {
TrackJudgeResponse(response datamodel.JudgeResponse) error
TrackUsage(usage ldai.TokenUsage) error
TrackTokens(usage ldai.TokenUsage) error
}

type StructuredResponse struct {
Expand Down Expand Up @@ -91,7 +91,7 @@ func (j *Judge) Evaluate(input, output string, samplingRate float64) (*datamodel
}

if response.Usage.Total > 0 || response.Usage.Input > 0 || response.Usage.Output > 0 {
_ = j.tracker.TrackUsage(response.Usage)
_ = j.tracker.TrackTokens(response.Usage)
}

result := j.parseResponse(response.Content)
Expand Down
2 changes: 1 addition & 1 deletion ldai/judge/judge_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func (m *mockTracker) TrackJudgeResponse(response datamodel.JudgeResponse) error
return nil
}

func (m *mockTracker) TrackUsage(usage ldai.TokenUsage) error {
func (m *mockTracker) TrackTokens(usage ldai.TokenUsage) error {
m.usages = append(m.usages, usage)
return nil
}
Expand Down
15 changes: 11 additions & 4 deletions ldai/tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,8 @@ func (t *Tracker) TrackTimeToFirstToken(dur time.Duration) error {
return t.events.TrackMetric(timeToFirstToken, t.context, float64(dur.Milliseconds()), t.trackData)
}

// TrackUsage tracks the token usage for a model evaluation.
func (t *Tracker) TrackUsage(usage TokenUsage) error {
// TrackTokens tracks the token usage for a model evaluation.
func (t *Tracker) TrackTokens(usage TokenUsage) error {
if usage.Set() {
t.tokens = ldcommon.Some(usage)
}
Expand Down Expand Up @@ -269,6 +269,13 @@ func (t *Tracker) TrackUsage(usage TokenUsage) error {
return nil
}

// TrackUsage tracks token usage.
//
// Deprecated: Use TrackTokens instead.
func (t *Tracker) TrackUsage(usage TokenUsage) error {
return t.TrackTokens(usage)
}

func measureDurationOfTask[T any, A any](
stopwatch Stopwatch,
arg A,
Expand Down Expand Up @@ -335,8 +342,8 @@ func (t *Tracker) TrackRequest(task func(c *Config) (ProviderResponse, error)) (
}

if usage.Usage.Set() {
// TrackUsage logs errors.
_ = t.TrackUsage(usage.Usage)
// TrackTokens logs errors.
_ = t.TrackTokens(usage.Usage)
}

return usage, nil
Expand Down
8 changes: 4 additions & 4 deletions ldai/tracker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,13 +245,13 @@ func TestTracker_TrackFeedback(t *testing.T) {
assert.ElementsMatch(t, []trackEvent{expectedPositiveEvent, expectedNegativeEvent}, events.events)
}

func TestTracker_TrackUsage(t *testing.T) {
func TestTracker_TrackTokens(t *testing.T) {
t.Run("only one field set, only one event", func(t *testing.T) {
events := newMockEvents()
config := &Config{}
tracker := newTracker("key", "variationKey", 8, events, config, ldcontext.New("key"), nil)

assert.NoError(t, tracker.TrackUsage(TokenUsage{
assert.NoError(t, tracker.TrackTokens(TokenUsage{
Total: 42,
}))

Expand All @@ -270,7 +270,7 @@ func TestTracker_TrackUsage(t *testing.T) {
config := &Config{}
tracker := newTracker("key", "variationKey", 9, events, config, ldcontext.New("key"), nil)

assert.NoError(t, tracker.TrackUsage(TokenUsage{
assert.NoError(t, tracker.TrackTokens(TokenUsage{
Total: 42,
Input: 20,
Output: 22,
Expand Down Expand Up @@ -382,7 +382,7 @@ func TestTracker_GetSummary(t *testing.T) {
Input: 40,
Output: 60,
}
_ = tracker.TrackUsage(usage)
_ = tracker.TrackTokens(usage)

summary := tracker.GetSummary()

Expand Down
Loading