feat!: Add per-execution runId, at-most-once tracking, and cross-process tracker resumption

packages/ai-providers/server-ai-langchain/tests/test_langgraph_callback_handler.py

@@ -35,6 +35,7 @@ def _make_graph(mock_ld_client: MagicMock, node_key: str = 'root-agent', graph_k
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key=graph_key,
     )
     graph_tracker = AIGraphTracker(
@@ -402,6 +403,7 @@ def test_flush_with_no_graph_key_on_node_tracker():
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
     )
     node_config = AIAgentConfig(
         key='root-agent',

packages/ai-providers/server-ai-langchain/tests/test_tracking_langgraph.py

@@ -45,6 +45,7 @@ def _make_graph(
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key=graph_key,
     )
 
@@ -142,6 +143,7 @@ def _make_two_node_graph(mock_ld_client: MagicMock) -> 'AgentGraphDefinition':
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key='two-node-graph',
     )
     child_tracker = LDAIConfigTracker(
@@ -152,6 +154,7 @@ def _make_two_node_graph(mock_ld_client: MagicMock) -> 'AgentGraphDefinition':
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key='two-node-graph',
     )
     graph_tracker = AIGraphTracker(

packages/ai-providers/server-ai-openai/tests/test_tracking_openai_agents.py

@@ -40,6 +40,7 @@ def _make_graph(
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key=graph_key,
     )
 
@@ -179,6 +180,7 @@ def _make_two_node_graph(mock_ld_client: MagicMock) -> AgentGraphDefinition:
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key='two-node-graph',
     )
     child_tracker = LDAIConfigTracker(
@@ -189,6 +191,7 @@ def _make_two_node_graph(mock_ld_client: MagicMock) -> AgentGraphDefinition:
         model_name='gpt-4',
         provider_name='openai',
         context=context,
+        run_id='test-run-id',
         graph_key='two-node-graph',
     )
     graph_tracker = AIGraphTracker(

packages/sdk/server-ai/src/ldai/client.py

@@ -1,4 +1,5 @@
-from typing import Any, Dict, List, Optional, Tuple
+import uuid
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import chevron
 from ldclient import Context
@@ -61,14 +62,30 @@ def __init__(self, client: LDClient):
             1,
         )
 
+    def create_tracker(self, token: str, context: Context) -> LDAIConfigTracker:
+        """
+        Reconstruct a tracker from a resumption token.
+
+        Delegates to :meth:`LDAIConfigTracker.from_resumption_token`.
+
+        :param token: A URL-safe Base64-encoded resumption token obtained from
+            :attr:`LDAIConfigTracker.resumption_token`.
+        :param context: The context to use for track events.
+        :return: A new :class:`LDAIConfigTracker` bound to the original
+            ``runId`` from the token.
+        :raises ValueError: If the token is invalid or missing required fields.
+        """
+        return LDAIConfigTracker.from_resumption_token(token, self._client, context)
+
     def _completion_config(
         self,
         key: str,
         context: Context,
         default: AICompletionConfigDefault,
         variables: Optional[Dict[str, Any]] = None,
     ) -> AICompletionConfig:
-        model, provider, messages, instructions, tracker, enabled, judge_configuration, _ = self.__evaluate(
+        (model, provider, messages, instructions,
+         tracker_factory, enabled, judge_configuration, _) = self.__evaluate(
             key, context, default.to_dict(), variables
         )
 
@@ -78,7 +95,7 @@ def _completion_config(
             model=model,
             messages=messages,
             provider=provider,
-            tracker=tracker,
+            create_tracker=tracker_factory,
             judge_configuration=judge_configuration,
         )
 
@@ -134,7 +151,8 @@ def _judge_config(
         default: AIJudgeConfigDefault,
         variables: Optional[Dict[str, Any]] = None,
     ) -> AIJudgeConfig:
-        model, provider, messages, instructions, tracker, enabled, judge_configuration, variation = self.__evaluate(
+        (model, provider, messages, instructions,
+         tracker_factory, enabled, judge_configuration, variation) = self.__evaluate(
             key, context, default.to_dict(), variables
         )
 
@@ -162,7 +180,7 @@ def _extract_evaluation_metric_key(variation: Dict[str, Any]) -> Optional[str]:
             model=model,
             messages=messages,
             provider=provider,
-            tracker=tracker,
+            create_tracker=tracker_factory,
         )
 
         return config
@@ -248,14 +266,14 @@ async def create_judge(
                 key, context, default or AIJudgeConfigDefault.disabled(), extended_variables
             )
 
-            if not judge_config.enabled or not judge_config.tracker:
+            if not judge_config.enabled:
                 return None
 
             provider = RunnerFactory.create_model(judge_config, default_ai_provider)
             if not provider:
                 return None
 
-            return Judge(judge_config, judge_config.tracker, provider)
+            return Judge(judge_config, provider)
         except Exception as error:
             return None
 
@@ -345,7 +363,7 @@ async def create_model(
         log.debug(f"Creating managed model for key: {key}")
         config = self._completion_config(key, context, default or AICompletionConfigDefault.disabled(), variables)
 
-        if not config.enabled or not config.tracker:
+        if not config.enabled:
             return None
 
         runner = RunnerFactory.create_model(config, default_ai_provider)
@@ -361,7 +379,7 @@ async def create_model(
                 default_ai_provider,
             )
 
-        return ManagedModel(config, config.tracker, runner, judges)
+        return ManagedModel(config, runner, judges)
 
     async def create_chat(
         self,
@@ -428,14 +446,14 @@ async def create_agent(
         log.debug(f"Creating managed agent for key: {key}")
         config = self.__evaluate_agent(key, context, default or AIAgentConfigDefault.disabled(), variables)
 
-        if not config.enabled or not config.tracker:
+        if not config.enabled:
             return None
 
         runner = RunnerFactory.create_agent(config, tools or {}, default_ai_provider)
         if not runner:
             return None
 
-        return ManagedAgent(config, config.tracker, runner)
+        return ManagedAgent(config, runner)
 
     def agent_config(
         self,
@@ -465,7 +483,8 @@ def agent_config(
 
             if agent.enabled:
                 research_result = agent.instructions  # Interpolated instructions
-                agent.tracker.track_success()
+                tracker = agent.create_tracker()
+                tracker.track_success()
 
         :param key: The agent configuration key.
         :param context: The context to evaluate the agent configuration in.
@@ -535,7 +554,8 @@ def agent_configs(
             ], context)
 
             research_result = agents["research_agent"].instructions
-            agents["research_agent"].tracker.track_success()
+            tracker = agents["research_agent"].create_tracker()
+            tracker.track_success()
 
         :param agent_configs: List of agent configurations to retrieve.
         :param context: The context to evaluate the agent configurations in.
@@ -727,7 +747,7 @@ async def create_agent_graph(
         if not runner:
             return None
 
-        return ManagedAgentGraph(runner, graph.get_tracker())
+        return ManagedAgentGraph(runner)
 
     def agents(
         self,
@@ -754,7 +774,7 @@ def __evaluate(
         graph_key: Optional[str] = None,
     ) -> Tuple[
         Optional[ModelConfig], Optional[ProviderConfig], Optional[List[LDMessage]],
-        Optional[str], LDAIConfigTracker, bool, Optional[Any], Dict[str, Any]
+        Optional[str], Callable[[], LDAIConfigTracker], bool, Optional[Any], Dict[str, Any]
     ]:
         """
         Internal method to evaluate a configuration and extract components.
@@ -764,7 +784,8 @@ def __evaluate(
         :param default_dict: Default configuration as dictionary.
         :param variables: Variables for interpolation.
         :param graph_key: When set, passed to the tracker so all events include ``graphKey``.
-        :return: Tuple of (model, provider, messages, instructions, tracker, enabled, judge_configuration, variation).
+        :return: Tuple of (model, provider, messages, instructions,
+            tracker_factory, enabled, judge_configuration, variation).
         """
         variation = self._client.variation(key, context, default_dict)
 
@@ -806,16 +827,23 @@ def __evaluate(
                 custom=custom
             )
 
-        tracker = LDAIConfigTracker(
-            self._client,
-            variation.get('_ldMeta', {}).get('variationKey', ''),
-            key,
-            int(variation.get('_ldMeta', {}).get('version', 1)),
-            model.name if model else '',
-            provider_config.name if provider_config else '',
-            context,
-            graph_key=graph_key,
-        )
+        variation_key = variation.get('_ldMeta', {}).get('variationKey', '')
+        version = int(variation.get('_ldMeta', {}).get('version', 1))
+        model_name = model.name if model else ''
+        provider_name = provider_config.name if provider_config else ''
+
+        def tracker_factory() -> LDAIConfigTracker:
+            return LDAIConfigTracker(
+                ld_client=self._client,
+                run_id=str(uuid.uuid4()),
+                config_key=key,
+                variation_key=variation_key,
+                version=version,
+                context=context,
+                model_name=model_name,
+                provider_name=provider_name,
+                graph_key=graph_key,
+            )
 
         enabled = variation.get('_ldMeta', {}).get('enabled', False)
 
@@ -834,7 +862,10 @@ def __evaluate(
                 if judges:
                     judge_configuration = JudgeConfiguration(judges=judges)
 
-        return model, provider_config, messages, instructions, tracker, enabled, judge_configuration, variation
+        return (
+            model, provider_config, messages, instructions,
+            tracker_factory, enabled, judge_configuration, variation,
+        )
 
     def __evaluate_agent(
         self,
@@ -854,7 +885,8 @@ def __evaluate_agent(
         :param graph_key: When set, passed to the tracker so all events include ``graphKey``.
         :return: Configured AIAgentConfig instance.
         """
-        model, provider, messages, instructions, tracker, enabled, judge_configuration, _ = self.__evaluate(
+        (model, provider, messages, instructions,
+         tracker_factory, enabled, judge_configuration, _) = self.__evaluate(
             key, context, default.to_dict(), variables, graph_key=graph_key
         )
 
@@ -867,7 +899,7 @@ def __evaluate_agent(
             model=model or default.model,
             provider=provider or default.provider,
             instructions=final_instructions,
-            tracker=tracker,
+            create_tracker=tracker_factory,
             judge_configuration=judge_configuration or default.judge_configuration,
         )
 

packages/sdk/server-ai/src/ldai/judge/__init__.py

@@ -10,7 +10,6 @@
 from ldai.models import AIJudgeConfig, LDMessage
 from ldai.providers.model_runner import ModelRunner
 from ldai.providers.types import JudgeResult, ModelResponse
-from ldai.tracker import LDAIConfigTracker
 
 
 class Judge:
@@ -24,18 +23,15 @@ class Judge:
     def __init__(
         self,
         ai_config: AIJudgeConfig,
-        ai_config_tracker: LDAIConfigTracker,
         model_runner: ModelRunner,
     ):
         """
         Initialize the Judge.
 
         :param ai_config: The judge AI configuration
-        :param ai_config_tracker: The tracker for the judge configuration
         :param model_runner: The model runner to use for evaluation
         """
         self._ai_config = ai_config
-        self._ai_config_tracker = ai_config_tracker
         self._model_runner = model_runner
         self._evaluation_response_structure = EvaluationSchemaBuilder.build()
 
@@ -73,10 +69,12 @@ async def evaluate(
                 return judge_result
 
             judge_result.sampled = True
+
+            tracker = self._ai_config.create_tracker()
             messages = self._construct_evaluation_messages(input_text, output_text)
             assert self._evaluation_response_structure is not None
 
-            response = await self._ai_config_tracker.track_metrics_of_async(
+            response = await tracker.track_metrics_of_async(
                 lambda: self._model_runner.invoke_structured_model(messages, self._evaluation_response_structure),
                 lambda result: result.metrics,
             )
@@ -125,14 +123,6 @@ def get_ai_config(self) -> AIJudgeConfig:
         """
         return self._ai_config
 
-    def get_tracker(self) -> LDAIConfigTracker:
-        """
-        Returns the tracker associated with this judge.
-
-        :return: The tracker for the judge configuration
-        """
-        return self._ai_config_tracker
-
     def get_model_runner(self) -> ModelRunner:
         """
         Returns the model runner used by this judge.

packages/sdk/server-ai/src/ldai/managed_agent.py

@@ -2,25 +2,22 @@
 
 from ldai.models import AIAgentConfig
 from ldai.providers import AgentResult, AgentRunner
-from ldai.tracker import LDAIConfigTracker
 
 
 class ManagedAgent:
     """
     LaunchDarkly managed wrapper for AI agent invocations.
 
-    Holds an AgentRunner and an LDAIConfigTracker. Handles tracking automatically.
+    Holds an AgentRunner. Handles tracking automatically via ``create_tracker()``.
     Obtain an instance via ``LDAIClient.create_agent()``.
     """
 
     def __init__(
         self,
         ai_config: AIAgentConfig,
-        tracker: LDAIConfigTracker,
         agent_runner: AgentRunner,
     ):
         self._ai_config = ai_config
-        self._tracker = tracker
         self._agent_runner = agent_runner
 
     async def run(self, input: str) -> AgentResult:
@@ -30,7 +27,8 @@ async def run(self, input: str) -> AgentResult:
         :param input: The user prompt or input to the agent
         :return: AgentResult containing the agent's output and metrics
         """
-        return await self._tracker.track_metrics_of_async(
+        tracker = self._ai_config.create_tracker()
+        return await tracker.track_metrics_of_async(
             lambda: self._agent_runner.run(input),
             lambda result: result.metrics,
         )
@@ -46,7 +44,3 @@ def get_agent_runner(self) -> AgentRunner:
     def get_config(self) -> AIAgentConfig:
         """Return the AI agent config."""
         return self._ai_config
-
-    def get_tracker(self) -> LDAIConfigTracker:
-        """Return the config tracker."""
-        return self._tracker