diff --git a/apps/api/plane/celery.py b/apps/api/plane/celery.py index 3a1b861490..92215c7c80 100644 --- a/apps/api/plane/celery.py +++ b/apps/api/plane/celery.py @@ -26,14 +26,14 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "plane.settings.production") # Bootstrap OpenTelemetry before Celery wires up so CeleryInstrumentor can # patch task execution. No-op unless OTEL_ENABLED=1. from plane.observability.setup import configure_otel, flush_otel # noqa: E402 -from plane.observability.logging import TraceContextFilter # noqa: E402 +from plane.observability.logging import TraceContextFilter, is_otel_enabled # noqa: E402 configure_otel() -# Whether to trace-correlate worker logs. Matches the Django LOGGING gate in -# plane/settings/{local,production}.py; the bootstrap in configure_otel() uses -# its own (superset) token check. -_OTEL_LOG_ENABLED = os.environ.get("OTEL_ENABLED", "0").strip().lower() in ("1", "true", "yes") +# Whether to trace-correlate worker logs. Uses the same shared is_otel_enabled() +# gate as the bootstrap (setup.configure_otel) and the Django LOGGING gate, so +# every documented OTEL_ENABLED token behaves identically across processes. +_OTEL_LOG_ENABLED = is_otel_enabled() # Base JSON log fmt (unchanged off-path); the OTel variant appends the # trace-context fields that TraceContextFilter populates. diff --git a/apps/api/plane/observability/logging.py b/apps/api/plane/observability/logging.py index c57c015679..9e260c9d0b 100644 --- a/apps/api/plane/observability/logging.py +++ b/apps/api/plane/observability/logging.py @@ -19,6 +19,21 @@ import os from opentelemetry import trace +# Accepted OTEL_ENABLED tokens — the single source of truth shared by the +# bootstrap gate (setup.configure_otel), the Celery worker-log gate, and the +# Django LOGGING gate, so enabling via any documented token (see the README) +# behaves identically everywhere. +_TRUTHY_VALUES = ("1", "true", "yes", "on") + +# Trace-context fields appended to the JSON log formatter when OTel is enabled; +# populated by TraceContextFilter. +_TRACE_LOG_FIELDS = "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" + + +def is_otel_enabled() -> bool: + """Return True when OTEL_ENABLED is set to a recognized truthy token.""" + return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES + class TraceContextFilter(logging.Filter): """Inject trace_id, span_id, trace_flags, service_name into LogRecord.""" @@ -35,3 +50,27 @@ class TraceContextFilter(logging.Filter): record.trace_flags = 0 record.service_name = os.environ.get("OTEL_SERVICE_NAME", "plane-api") return True + + +def extend_logging_config(logging_config: dict) -> None: + """Trace-correlate a Django LOGGING dict in place. No-op unless OTel is enabled. + + When enabled, extends the JSON formatter's fmt with the trace-context fields + and attaches TraceContextFilter to every handler. Attaching at the handler + level (rather than the root logger) is required because most plane.* loggers + set propagate=False; runtime mutation also wouldn't survive Django's + dictConfig. The off path leaves the log schema byte-for-byte unchanged. + + Shared by settings/local.py and settings/production.py so the two stay in + lockstep. + """ + if not is_otel_enabled(): + return + logging_config["formatters"]["json"]["fmt"] = ( + "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " + _TRACE_LOG_FIELDS + ) + logging_config.setdefault("filters", {})["trace_context"] = { + "()": "plane.observability.logging.TraceContextFilter", + } + for handler in logging_config["handlers"].values(): + handler["filters"] = ["trace_context"] diff --git a/apps/api/plane/observability/setup.py b/apps/api/plane/observability/setup.py index 55e6559acc..09790c2996 100644 --- a/apps/api/plane/observability/setup.py +++ b/apps/api/plane/observability/setup.py @@ -24,6 +24,8 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from plane.observability.logging import is_otel_enabled + logger = logging.getLogger(__name__) _CONFIGURED = False @@ -33,10 +35,6 @@ _CONFIGURED = False _TRACER_PROVIDER: "TracerProvider | None" = None _METER_PROVIDER: "MeterProvider | None" = None -# Accepted "on" tokens — kept in sync with the pi/node services so OTEL_ENABLED -# behaves identically across every runtime. -_TRUTHY_VALUES = ("1", "true", "yes", "on") - _NOISY_OTEL_LOGGERS = ( "opentelemetry", "opentelemetry.exporter.otlp", @@ -49,10 +47,6 @@ _NOISY_OTEL_LOGGERS = ( _HTTP_PROTOCOLS = ("http/protobuf", "http") -def _is_enabled() -> bool: - return os.environ.get("OTEL_ENABLED", "0").strip().lower() in _TRUTHY_VALUES - - def _has_endpoint() -> bool: return bool(os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()) @@ -140,13 +134,24 @@ def _instrument_libraries() -> None: request that enqueues a task is linked to that task's execution span. The others add child spans so latency can be decomposed (SQL query, Redis op, outbound HTTP). + + Each instrumentor is isolated: a single failure (version mismatch, missing + optional dependency) is logged and skipped so it neither blocks the other + instrumentors nor takes down application startup. """ - DjangoInstrumentor().instrument() - CeleryInstrumentor().instrument() - PsycopgInstrumentor().instrument(enable_commenter=False) - RedisInstrumentor().instrument() - RequestsInstrumentor().instrument() - HTTPXClientInstrumentor().instrument() + instrumentors = ( + (DjangoInstrumentor, {}), + (CeleryInstrumentor, {}), + (PsycopgInstrumentor, {"enable_commenter": False}), + (RedisInstrumentor, {}), + (RequestsInstrumentor, {}), + (HTTPXClientInstrumentor, {}), + ) + for instrumentor_cls, kwargs in instrumentors: + try: + instrumentor_cls().instrument(**kwargs) + except Exception as exc: + logger.warning("Failed to instrument %s: %s", instrumentor_cls.__name__, exc) def _quiet_otel_loggers() -> None: @@ -172,7 +177,7 @@ def configure_otel() -> None: global _CONFIGURED if _CONFIGURED: return - if not _is_enabled(): + if not is_otel_enabled(): return if not _has_endpoint(): logger.warning("OTEL_ENABLED=1 but OTEL_EXPORTER_OTLP_ENDPOINT is not set; OpenTelemetry bootstrap skipped") diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.py index cd726d8e33..7148011a1c 100644 --- a/apps/api/plane/settings/local.py +++ b/apps/api/plane/settings/local.py @@ -88,18 +88,8 @@ LOGGING = { }, } -# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON -# formatter and attach TraceContextFilter to every handler. Attaching at -# the handler level (rather than the root logger) is required because -# most plane.* loggers have propagate=False; runtime mutation also wouldn't -# survive Django's dictConfig. Off path leaves the log schema unchanged. -if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): - LOGGING["formatters"]["json"]["fmt"] = ( - "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " - "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" - ) - LOGGING["filters"] = { - "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, - } - for _handler in LOGGING["handlers"].values(): - _handler["filters"] = ["trace_context"] +# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled. +# No-op (log schema unchanged) otherwise. Shared with settings/production.py. +from plane.observability.logging import extend_logging_config # noqa: E402 + +extend_logging_config(LOGGING) diff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.py index d9c5ed781b..0702d84191 100644 --- a/apps/api/plane/settings/production.py +++ b/apps/api/plane/settings/production.py @@ -98,18 +98,8 @@ LOGGING = { }, } -# OpenTelemetry APM: only when OTEL_ENABLED=1 do we extend the JSON -# formatter and attach TraceContextFilter to every handler. Attaching at -# the handler level (rather than the root logger) is required because -# most plane.* loggers have propagate=False; runtime mutation also wouldn't -# survive Django's dictConfig. Off path leaves the log schema unchanged. -if os.environ.get("OTEL_ENABLED", "0").lower() in ("1", "true", "yes"): - LOGGING["formatters"]["json"]["fmt"] = ( - "%(levelname)s %(asctime)s %(module)s %(name)s %(message)s " - "%(service_name)s %(trace_id)s %(span_id)s %(trace_flags)s" - ) - LOGGING["filters"] = { - "trace_context": {"()": "plane.observability.logging.TraceContextFilter"}, - } - for _handler in LOGGING["handlers"].values(): - _handler["filters"] = ["trace_context"] +# OpenTelemetry APM: trace-correlate the JSON logs when OTEL is enabled. +# No-op (log schema unchanged) otherwise. Shared with settings/local.py. +from plane.observability.logging import extend_logging_config # noqa: E402 + +extend_logging_config(LOGGING) diff --git a/apps/api/plane/tests/unit/observability/conftest.py b/apps/api/plane/tests/unit/observability/conftest.py index f140e7a9d2..023f1abc4b 100644 --- a/apps/api/plane/tests/unit/observability/conftest.py +++ b/apps/api/plane/tests/unit/observability/conftest.py @@ -25,6 +25,8 @@ def isolate_otel_state(monkeypatch): from plane.observability import setup as otel_setup monkeypatch.setattr(otel_setup, "_CONFIGURED", False, raising=False) + monkeypatch.setattr(otel_setup, "_TRACER_PROVIDER", None, raising=False) + monkeypatch.setattr(otel_setup, "_METER_PROVIDER", None, raising=False) saved = {k: v for k, v in os.environ.items() if k.startswith("OTEL_")} for key in list(saved.keys()): diff --git a/deployments/cli/community/docker-compose.yml b/deployments/cli/community/docker-compose.yml index d82902374c..7c613cc3bd 100644 --- a/deployments/cli/community/docker-compose.yml +++ b/deployments/cli/community/docker-compose.yml @@ -70,6 +70,9 @@ x-otel-env: &otel-env OTEL_EXPORTER_OTLP_HEADERS: ${OTEL_EXPORTER_OTLP_HEADERS:-} OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-plane-api} OTEL_ENVIRONMENT: ${OTEL_ENVIRONMENT:-} + OTEL_TRACES_SAMPLER: ${OTEL_TRACES_SAMPLER:-} + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-} + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-} services: web: