refactor(api/observability): unify OTEL_ENABLED gate, isolate instrumentors, pass sampler vars through compose

- Extract shared is_otel_enabled() + extend_logging_config() into
  plane.observability.logging so the bootstrap, Celery, and Django LOGGING
  gates all accept the documented tokens (incl. 'on') identically.
- Isolate each instrumentor in _instrument_libraries() so one failure can't
  block startup.
- Pass OTEL_TRACES_SAMPLER / _ARG / OTEL_RESOURCE_ATTRIBUTES through the
  x-otel-env compose anchor (documented in .env.example).
- Reset _TRACER_PROVIDER/_METER_PROVIDER in the test fixture.

apps/api/plane/celery.py

@@ -26,14 +26,14 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production")
 # Bootstrap OpenTelemetry before Celery wires up so CeleryInstrumentor can
 # patch task execution. No-op unless OTEL_ENABLED=1.
 from plane.observability.setup import configure_otel, flush_otel  # noqa: E402
-from plane.observability.logging import TraceContextFilter  # noqa: E402
+from plane.observability.logging import TraceContextFilter, is_otel_enabled  # noqa: E402
 
 configure_otel()
 
-# Whether to trace-correlate worker logs. Matches the Django LOGGING gate in
-# plane/settings/{local,production}.py; the bootstrap in configure_otel() uses
-# its own (superset) token check.
-_OTEL_LOG_ENABLED = os.environ.get("OTEL_ENABLED", "0").strip().lower() in ("1", "true", "yes")
+# Whether to trace-correlate worker logs. Uses the same shared is_otel_enabled()
+# gate as the bootstrap (setup.configure_otel) and the Django LOGGING gate, so
+# every documented OTEL_ENABLED token behaves identically across processes.
+_OTEL_LOG_ENABLED = is_otel_enabled()
 
 # Base JSON log fmt (unchanged off-path); the OTel variant appends the
 # trace-context fields that TraceContextFilter populates.

apps/api/plane/observability/logging.py

@@ -19,6 +19,21 @@ import os
 
 from opentelemetry import trace
 
+# Accepted OTEL_ENABLED tokens — the single source of truth shared by the
+# bootstrap gate (setup.configure_otel), the Celery worker-log gate, and the
+# Django LOGGING gate, so enabling via any documented token (see the README)
+# behaves identically everywhere.
+_TRUTHY_VALUES = ("1", "true", "yes", "on")
+
+# Trace-context fields appended to the JSON log formatter when OTel is enabled;
+# populated by TraceContextFilter.
+_TRACE_LOG_FIELDS = "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s"
+
+
+def is_otel_enabled() -> bool:
+    """Return True when OTEL_ENABLED is set to a recognized truthy token."""
+    return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES
+
 
 class TraceContextFilter(logging.Filter):
     """Inject trace_id, span_id, trace_flags, service_name into LogRecord."""
@@ -35,3 +50,27 @@ class TraceContextFilter(logging.Filter):
             record.trace_flags = 0
         record.service_name = os.environ.get("OTEL_SERVICE_NAME", "plane-api")
         return True
+
+
+def extend_logging_config(logging_config: dict) -> None:
+    """Trace-correlate a Django LOGGING dict in place. No-op unless OTel is enabled.
+
+    When enabled, extends the JSON formatter's fmt with the trace-context fields
+    and attaches TraceContextFilter to every handler. Attaching at the handler
+    level (rather than the root logger) is required because most plane.* loggers
+    set propagate=False; runtime mutation also wouldn't survive Django's
+    dictConfig. The off path leaves the log schema byte-for-byte unchanged.
+
+    Shared by settings/local.py and settings/production.py so the two stay in
+    lockstep.
+    """
+    if not is_otel_enabled():
+        return
+    logging_config["formatters"]["json"]["fmt"] = (
+        "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + _TRACE_LOG_FIELDS
+    )
+    logging_config.setdefault("filters", {})["trace_context"] = {
+        "()": "plane.observability.logging.TraceContextFilter",
+    }
+    for handler in logging_config["handlers"].values():
+        handler["filters"] = ["trace_context"]

apps/api/plane/observability/setup.py

@@ -24,6 +24,8 @@ from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
+from plane.observability.logging import is_otel_enabled
+
 logger = logging.getLogger(__name__)
 
 _CONFIGURED = False
@@ -33,10 +35,6 @@ _CONFIGURED = False
 _TRACER_PROVIDER: "TracerProvider | None" = None
 _METER_PROVIDER: "MeterProvider | None" = None
 
-# Accepted "on" tokens — kept in sync with the pi/node services so OTEL_ENABLED
-# behaves identically across every runtime.
-_TRUTHY_VALUES = ("1", "true", "yes", "on")
-
 _NOISY_OTEL_LOGGERS = (
     "opentelemetry",
     "opentelemetry.exporter.otlp",
@@ -49,10 +47,6 @@ _NOISY_OTEL_LOGGERS = (
 _HTTP_PROTOCOLS = ("http/protobuf", "http")
 
 
-def _is_enabled() -> bool:
-    return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES
-
-
 def _has_endpoint() -> bool:
     return bool(os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip())
 
@@ -140,13 +134,24 @@ def _instrument_libraries() -> None:
     request that enqueues a task is linked to that task's execution span.
     The others add child spans so latency can be decomposed (SQL query,
     Redis op, outbound HTTP).
+
+    Each instrumentor is isolated: a single failure (version mismatch, missing
+    optional dependency) is logged and skipped so it neither blocks the other
+    instrumentors nor takes down application startup.
     """
-    DjangoInstrumentor().instrument()
-    CeleryInstrumentor().instrument()
-    PsycopgInstrumentor().instrument(enable_commenter=False)
-    RedisInstrumentor().instrument()
-    RequestsInstrumentor().instrument()
-    HTTPXClientInstrumentor().instrument()
+    instrumentors = (
+        (DjangoInstrumentor, {}),
+        (CeleryInstrumentor, {}),
+        (PsycopgInstrumentor, {"enable_commenter": False}),
+        (RedisInstrumentor, {}),
+        (RequestsInstrumentor, {}),
+        (HTTPXClientInstrumentor, {}),
+    )
+    for instrumentor_cls, kwargs in instrumentors:
+        try:
+            instrumentor_cls().instrument(**kwargs)
+        except Exception as exc:
+            logger.warning("Failed to instrument %s: %s", instrumentor_cls.__name__, exc)
 
 
 def _quiet_otel_loggers() -> None:
@@ -172,7 +177,7 @@ def configure_otel() -> None:
     global _CONFIGURED
     if _CONFIGURED:
         return
-    if not _is_enabled():
+    if not is_otel_enabled():
         return
     if not _has_endpoint():
         logger.warning("OTEL_ENABLED=1 but OTEL_EXPORTER_OTLP_ENDPOINT is not set; OpenTelemetry bootstrap skipped")

apps/api/plane/settings/local.py

@@ -88,18 +88,8 @@ LOGGING = {
     },
 }
 
-# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON
-# formatter and attach TraceContextFilter to every handler. Attaching at
-# the handler level (rather than the root logger) is required because
-# most plane.* loggers have propagate=False; runtime mutation also wouldn't
-# survive Django's dictConfig. Off path leaves the log schema unchanged.
-if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"):
-    LOGGING["formatters"]["json"]["fmt"] = (
-        "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s "
-        "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s"
-    )
-    LOGGING["filters"] = {
-        "trace_context": {"()": "plane.observability.logging.TraceContextFilter"},
-    }
-    for _handler in LOGGING["handlers"].values():
-        _handler["filters"] = ["trace_context"]
+# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled.
+# No-op (log schema unchanged) otherwise. Shared with settings/production.py.
+from plane.observability.logging import extend_logging_config  # noqa: E402
+
+extend_logging_config(LOGGING)

apps/api/plane/settings/production.py

@@ -98,18 +98,8 @@ LOGGING = {
     },
 }
 
-# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON
-# formatter and attach TraceContextFilter to every handler. Attaching at
-# the handler level (rather than the root logger) is required because
-# most plane.* loggers have propagate=False; runtime mutation also wouldn't
-# survive Django's dictConfig. Off path leaves the log schema unchanged.
-if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"):
-    LOGGING["formatters"]["json"]["fmt"] = (
-        "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s "
-        "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s"
-    )
-    LOGGING["filters"] = {
-        "trace_context": {"()": "plane.observability.logging.TraceContextFilter"},
-    }
-    for _handler in LOGGING["handlers"].values():
-        _handler["filters"] = ["trace_context"]
+# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled.
+# No-op (log schema unchanged) otherwise. Shared with settings/local.py.
+from plane.observability.logging import extend_logging_config  # noqa: E402
+
+extend_logging_config(LOGGING)

apps/api/plane/tests/unit/observability/conftest.py

@@ -25,6 +25,8 @@ def isolate_otel_state(monkeypatch):
     from plane.observability import setup as otel_setup
 
     monkeypatch.setattr(otel_setup, "_CONFIGURED", False, raising=False)
+    monkeypatch.setattr(otel_setup, "_TRACER_PROVIDER", None, raising=False)
+    monkeypatch.setattr(otel_setup, "_METER_PROVIDER", None, raising=False)
 
     saved = {k: v for k, v in os.environ.items() if k.startswith("OTEL_")}
     for key in list(saved.keys()):

deployments/cli/community/docker-compose.yml

@@ -70,6 +70,9 @@ x-otel-env: &otel-env
   OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-}
   OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-plane-api}
   OTEL_ENVIRONMENT: ${OTEL_ENVIRONMENT:-}
+  OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-}
+  OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-}
+  OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-}
 
 services:
   web: