spec file now creates the /var/lib/microshift dir for the file_storag…

…e extension systemd unit sets microshift.service for RequiredBy=, meaning that if observability fails to start, microshift should be killed. otel config now includes microshift-etcd, microshift-tuned services in log collection. Removed debugging exporter. Added filelog pipeline Signed-off-by: Jon Cope <[email protected]>
openshift · Jan 22, 2025 · ccfea22 · ccfea22
1 parent 150549f
commit ccfea22
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 27 deletions.
diff --git a/packaging/observability/microshift-observability.service b/packaging/observability/microshift-observability.service
@@ -13,4 +13,8 @@ TimeoutStartSec=60
 RestartSec=200ms
 
 [Install]
-RequiredBy=multi-user.target
+# Observability starts after microshift because of a dependency on microshift-generated
+# certs. If observability fails to start however, we don't want microshift to continue running
+# and sending metrics and logs into the void. Setting the RequiredBy=microshift.service
+# will cause microshift to stop if observability fails to start.
+RequiredBy=multi-user.target microshift.service
diff --git a/packaging/observability/opentelemetry-collector.yaml b/packaging/observability/opentelemetry-collector.yaml
@@ -1,11 +1,15 @@
+# Basic Opentelemetry-collector configuration file for MicroShift
+# This file provides a starting config for users to customize to their needs. Exporters are disabled (nop) by default.
+# This is because the exporter endpoints are not knowable at installation. Users are therefore encouraged
+# to specify their own copy of this file, customized to their needs, including the otlp exporter endpoint.
 receivers:
-#  otlp:
-#    protocol:
-#      grpc: $MICROSHIFT_IP:6666
-#  filelog/workload:
-#    Path: /….
-#  filelog/kube-system:
-#    Path: /….
+  filelog/workload:
+    include:
+      - /var/log/pods/
+    exclude:
+      - /var/log/pods/openshift-*
+  filelog/kube-system:
+    include: /var/log/pods/openshift-*
   kubeletstats:
     auth_type: tls
     ca_file: /etc/pki/microshift-opentelemetry-collector-client/client-ca.crt
@@ -35,47 +39,46 @@ receivers:
   journald:
     units:
     - microshift
+    - microshift-etcd
+    - microshift-ovs-init
+    - microshift-tuned
     - crio
 
 processors:
   resourcedetection/system:
-    detectors: [system]
+    detectors: ["system", "env"]
     system:
+      hostname_sources: ["os"]
 
 exporters:
+# EXAMPLE OTLP (Prometheus) ENDPOINT CONFIG
+# The otlp exporter requires an endpoint listening for OTLP connections. To prevent spamming the log with Go
+# stack traces, the exporter is disabled. The endpoint is not known at installation, thus a tire-kicking of the
+# microshift-observability package would result in stack traces spam in logs.
   otlp:
     sending_queue:
       storage: file_storage
-    endpoint: # IP|HOSTNAME REQUIRED
+    endpoint: localhost:12345 # Valid OTLP endpoint required
     tls:
       insecure: true
-  debug:
 
 extensions:
   file_storage:
-    directory: /var/lib/otelcol
-    create_directory: true
-    fsync: true
-    compaction:
-      directory: /var/lib/otelcol
-      on_rebound: true
-      # when allocated data exceeds this amount, the "compaction needed" flag will be enabled
-      # rebound_needed_threshold_mib: 100 # Default
-      # if the "compaction needed" flag is set and allocated data drops below this amount, compaction will begin and the "compaction needed" flag will be cleared
-      # rebound_trigger_threshold_mib: 10 # Default
-      # specifies how frequently the conditions for compaction are being checked
-      check_interval: 5s # Default
+    directory: /var/lib/microshift-observability
 
 service:
   extensions: [file_storage]
   pipelines:
     metrics/kubeletstats:
       receivers: [kubeletstats]
-      exporters: [otlp]
+      exporters:
+      - otlp
     metrics:
       receivers: [hostmetrics, hostmetrics/disk, hostmetrics/network]
       processors: [resourcedetection/system]
-      exporters: [otlp, debug]
+      exporters:
+      - otlp
     logs:
-      receivers: [journald]
-      exporters: [otlp, debug]
+      receivers: [journald, filelog/kube-system, filelog/workload]
+      exporters:
+      - otlp
diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec
@@ -499,6 +499,7 @@ install -p -m644 assets/optional/gateway-api/release-gateway-api-{x86_64,aarch64
 #observability
 install -p -m644 packaging/observability/opentelemetry-collector.yaml -D %{buildroot}%{_sysconfdir}/microshift/opentelemetry-collector.yaml
 install -p -m644 packaging/observability/microshift-observability.service %{buildroot}%{_unitdir}/
+install -d -m755 %{buildroot}%{_sharedstatedir}/microshift-observability
 
 %pre networking
 
@@ -669,6 +670,7 @@ fi
 %files observability
 %config %{_sysconfdir}/microshift/opentelemetry-collector.yaml
 %config %{_unitdir}/microshift-observability.service
+%dir %{_sharedstatedir}/microshift-observability
 
 # Use Git command to generate the log and replace the VERSION string
 # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec