Getting OTEL client instrumentation code to work with latest OTEL SDK version

PLATER/plater-clustered-docker/main.sh

@@ -17,7 +17,8 @@ function webServer {
         ${WEB_HOST}:${WEB_PORT} \
         --workers ${NUM_WORKERS} \
         --timeout ${WORKER_TIMEOUT} \
-        -k uvicorn.workers.UvicornWorker
+        -k uvicorn.workers.UvicornWorker \
+        --config /home/plater/Plater/gunicorn.conf.py
     else
         uvicorn PLATER.services.server:APP --host ${WEB_HOST} --port ${WEB_PORT} --reload
     fi
@@ -46,4 +47,4 @@ webServerPID=$!
 heartbeat &
 heartbeatPID=$!
 # add wait so killing this script would kill the bg processes.
-wait $webServerPID $heartbeatPID
+wait $webServerPID $heartbeatPID

PLATER/requirements.txt

@@ -1,4 +1,4 @@
-fastapi==0.85.0
+fastapi==0.100.0
 pyaml==20.4.0
 pytest==8.3.3
 pytest-asyncio==0.24.0
@@ -9,8 +9,11 @@ httpx==0.27.2
 pytest-httpx==0.32.0
 pydantic==1.10.2
 orjson==3.10.7
-opentelemetry-sdk==1.27.0
-opentelemetry-exporter-otlp-proto-grpc==1.27.0
-opentelemetry-instrumentation-fastapi==0.48b0
+opentelemetry-sdk==1.41.1
+opentelemetry-exporter-otlp-proto-grpc==1.41.1
+opentelemetry-exporter-otlp-proto-common==1.41.1
+opentelemetry-proto==1.41.1
+opentelemetry-api==1.41.1
+opentelemetry-instrumentation-fastapi==0.62b1
 pyinstrument==4.7.3
 neo4j-rust-ext==5.28.0

PLATER/services/otel.py

@@ -0,0 +1,46 @@
+"""OpenTelemetry tracing setup for the Plater FastAPI app.
+
+Must be called *after* the gunicorn fork (from a post_fork hook) so the
+BatchSpanProcessor background thread and gRPC channel are created in the
+worker process, not the master. See
+https://oneuptime.com/blog/post/2026-02-06-troubleshoot-fastapi-uvicorn-reload/view
+"""
+
+
+def setup_otel(worker_pid=None):
+    from PLATER.services.config import config
+
+    if config.get("OTEL_ENABLED", "False") in ("false", "False"):
+        return
+
+    from opentelemetry import trace
+    from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+
+    PLATER_TITLE = config.get('PLATER_TITLE', 'Plater API')
+    resource_attrs = {SERVICE_NAME: config.get("OTEL_SERVICE_NAME", PLATER_TITLE)}
+    if worker_pid is not None:
+        resource_attrs["worker.pid"] = worker_pid
+    resource = Resource.create(attributes=resource_attrs)
+    provider = TracerProvider(resource=resource)
+
+    OTEL_USE_CONSOLE_EXPORTER = config.get(
+        "OTEL_USE_CONSOLE_EXPORTER", "False"
+    ) not in ("false", "False")
+
+    if OTEL_USE_CONSOLE_EXPORTER:
+        from opentelemetry.sdk.trace.export import ConsoleSpanExporter
+        processor = BatchSpanProcessor(ConsoleSpanExporter())
+    else:
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+        otlp_host = config.get("JAEGER_HOST", "http://localhost").rstrip('/')
+        otlp_port = config.get("JAEGER_PORT", "4317")
+        processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=f'{otlp_host}:{otlp_port}'))
+
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)
+
+    from PLATER.services.app_trapi import APP
+    FastAPIInstrumentor.instrument_app(APP, excluded_urls="docs,openapi.json")

PLATER/services/server.py

@@ -18,37 +18,9 @@
     allow_headers=["*"],
 )
 
-if config.get("OTEL_ENABLED", "False") not in ("false", "False"):
-    from opentelemetry import trace
-    from opentelemetry.sdk.resources import SERVICE_NAME, Resource
-    from opentelemetry.sdk.trace import TracerProvider
-    from opentelemetry.sdk.trace.export import BatchSpanProcessor
-    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-
-    OTEL_USE_CONSOLE_EXPORTER = config.get("OTEL_USE_CONSOLE_EXPORTER", "False") not in ("false", "False")
-    if OTEL_USE_CONSOLE_EXPORTER:
-        from opentelemetry.sdk.trace.export import ConsoleSpanExporter
-    else:
-        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
-
-    plater_service_name = PLATER_TITLE
-    resource = Resource.create(attributes={
-        SERVICE_NAME: config.get("OTEL_SERVICE_NAME", plater_service_name),
-    })
-    provider = TracerProvider(resource=resource)
-    if OTEL_USE_CONSOLE_EXPORTER:
-        processor = BatchSpanProcessor(ConsoleSpanExporter())
-    else:
-        otlp_host = config.get("JAEGER_HOST", "http://localhost/").rstrip('/')
-        otlp_port = config.get("JAEGER_PORT", "4317")
-        otlp_endpoint = f'{otlp_host}:{otlp_port}'
-        otlp_exporter = OTLPSpanExporter(endpoint=f'{otlp_endpoint}')
-        processor = BatchSpanProcessor(otlp_exporter)
-
-    provider.add_span_processor(processor)
-    trace.set_tracer_provider(provider)
-    FastAPIInstrumentor.instrument_app(APP, tracer_provider=provider, excluded_urls="docs,openapi.json")
 
 if __name__ == '__main__':
     import uvicorn
+    from PLATER.services.otel import setup_otel
+    setup_otel()
     uvicorn.run(APP, host='0.0.0.0', port=8080)

gunicorn.conf.py

@@ -0,0 +1,9 @@
+# gunicorn.conf.py
+# Thin shim: real OTEL setup lives in PLATER/services/otel.py.
+# It must run in post_fork so OTEL background threads/channels are created
+# in the worker process, not the master.
+
+
+def post_fork(server, worker):
+    from PLATER.services.otel import setup_otel
+    setup_otel(worker_pid=worker.pid)

main.sh

@@ -6,7 +6,7 @@ source .env
 set +a
 
 if [ "$MODE" == "deploy" ]; then
-    gunicorn PLATER.services.server:APP -b ${WEB_HOST}:${WEB_PORT} -w 4 -k uvicorn.workers.UvicornWorker
+    gunicorn PLATER.services.server:APP -b ${WEB_HOST}:${WEB_PORT} -w 4 -k uvicorn.workers.UvicornWorker --config gunicorn.conf.py
 else
     uvicorn PLATER.services.server:APP --host ${WEB_HOST} --port ${WEB_PORT} --reload
 fi