Skip to content

feat: collect pod logs #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vmihailenco wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: collect pod logs #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions charts/uptrace/templates/otelcol-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,17 @@ spec:
volumeMounts:
- name: config
mountPath: /etc/config
- name: varlogpods
mountPath: /var/log/pods
readOnly: true
imagePullPolicy: IfNotPresent
env:
{{- toYaml .Values.uptrace.env | nindent 12 }}
volumes:
- name: config
configMap:
name: otelcol
- name: varlogpods
hostPath:
path: /var/log/pods
{{ end }}
217 changes: 137 additions & 80 deletions charts/uptrace/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,11 @@ fullnameOverride: ''
podAnnotations: {}

# Security context for pod templates.
podSecurityContext:
{}
podSecurityContext: {}
# fsGroup: 2000

# Security context for containers.
containerSecurityContext:
{}
containerSecurityContext: {}
# capabilities:
# drop:
# - ALL
Expand Down Expand Up @@ -52,8 +50,7 @@ clickhouse:
http_port: 8123
tcp_port: 9000

resources:
{}
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
Expand Down Expand Up @@ -81,8 +78,7 @@ postgresql:
size: 4Gi
storageClass: ''

resources:
{}
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
Expand Down Expand Up @@ -111,6 +107,72 @@ otelcol:
k8s_cluster:
auth_type: serviceAccount
collection_interval: 15s
kubeletstats:
collection_interval: 15s
auth_type: 'serviceAccount'
endpoint: 'https://${env:K8S_NODE_NAME}:10250'
insecure_skip_verify: true
filelog:
include: [/var/log/pods/*/*/*.log]
include_file_name: false
include_file_path: true
start_at: end # Recommended: use "end" for production
operators:
- type: router
id: get-format
routes:
- expr: body matches "^\\{"
output: parser-docker
- expr: body matches "^[^ Z]+ "
output: parser-crio
- expr: body matches "^[^ Z]+Z"
output: parser-containerd
- type: regex_parser
id: parser-crio
output: extract_metadata_from_filepath
regex: ^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
timestamp:
layout: 2006-01-02T15:04:05.999999999Z07:00
layout_type: gotime
parse_from: attributes.time
- type: regex_parser
id: parser-containerd
output: extract_metadata_from_filepath
regex: ^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$
timestamp:
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
parse_from: attributes.time
- type: json_parser
id: parser-docker
output: extract_metadata_from_filepath
timestamp:
layout: '%Y-%m-%dT%H:%M:%S.%LZ'
parse_from: attributes.time
- type: regex_parser
id: extract_metadata_from_filepath
parse_from: attributes["log.file.path"]
regex: ^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<restart_count>\d+)\.log$
- type: move
from: attributes.stream
to: attributes["log.iostream"]
- type: move
from: attributes.container_name
to: resource["k8s.container.name"]
- type: move
from: attributes.namespace
to: resource["k8s.namespace.name"]
- type: move
from: attributes.pod_name
to: resource["k8s.pod.name"]
- type: move
from: attributes.restart_count
to: resource["k8s.container.restart_count"]
- type: move
from: attributes.uid
to: resource["k8s.pod.uid"]
- type: move
from: attributes.log
to: body
processors:
resourcedetection:
detectors: ['system']
Expand All @@ -131,16 +193,15 @@ otelcol:
processors: [batch]
exporters: [otlp/local]
metrics:
receivers: [otlp, k8s_cluster]
receivers: [otlp, k8s_cluster, kubeletstats]
processors: [cumulativetodelta, batch]
exporters: [debug, otlp/local]
logs:
receivers: [otlp, k8s_cluster]
processors: [batch]
receivers: [otlp, k8s_cluster, filelog]
processors: [batch, resourcedetection]
exporters: [debug, otlp/local]

resources:
{}
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
Expand Down Expand Up @@ -201,8 +262,7 @@ otelcolDaemonset:
ingress:
enabled: true
className: ''
annotations:
{}
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
Expand Down Expand Up @@ -234,12 +294,10 @@ uptrace:
type: ClusterIP # or LoadBalancer
http_port: 80
grpc_port: 4317
annotations:
{}
annotations: {}
# service.beta.kubernetes.io/aws-load-balancer-type: "external"
# service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
loadBalancerSourceRanges:
[]
loadBalancerSourceRanges: []
# - "10.0.0.0/8"
# - "20.0.0.0/8"

Expand All @@ -257,8 +315,7 @@ uptrace:
## https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container
envFrom: []

resources:
{}
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
Expand Down Expand Up @@ -292,8 +349,8 @@ uptrace:
# -----------------------------------------------------------------------------
# Core service settings that define the runtime environment and security
service:
env: hosted # Environment: hosted, development, production
secret: FIXME # Secret key for cryptographic operations (CHANGE THIS!)
env: hosted # Environment: hosted, development, production
secret: FIXME # Secret key for cryptographic operations (CHANGE THIS!)

# -----------------------------------------------------------------------------
# Site Configuration
Expand Down Expand Up @@ -374,54 +431,54 @@ uptrace:

# Default users created on first startup
users:
- key: user1 # Internal reference key (used in relationships below)
name: Admin # Display name in UI
- key: user1 # Internal reference key (used in relationships below)
name: Admin # Display name in UI
email: admin@uptrace.local # Login email (must be unique)
password: admin # Plain text password (CHANGE THIS IMMEDIATELY!)
password: admin # Plain text password (CHANGE THIS IMMEDIATELY!)
email_confirmed: true

# API tokens for user authentication
# These tokens can be used for API access and programmatic operations
user_tokens:
- key: user_token1 # Internal reference key
user_key: user1 # References user.key above
token: user1_secret # API token value (CHANGE THIS!)
- key: user_token1 # Internal reference key
user_key: user1 # References user.key above
token: user1_secret # API token value (CHANGE THIS!)

# Organizations for multi-tenant deployments
# Organizations group users and projects together
orgs:
- key: org1 # Internal reference key
name: Org1 # Organization display name
- key: org1 # Internal reference key
name: Org1 # Organization display name

# Organization membership and roles
# Defines which users belong to which organizations
org_users:
- key: org_user1 # Internal reference key
org_key: org1 # References org.key above
user_key: user1 # References user.key above
role: owner # Role: owner, admin, or member
- key: org_user1 # Internal reference key
org_key: org1 # References org.key above
user_key: user1 # References user.key above
role: owner # Role: owner, admin, or member

# Projects contain telemetry data and are isolated from each other
# Each project has its own spans, logs, metrics, and dashboards
projects:
- key: project1 # Internal reference key
name: Project1 # Project display name
org_key: org1 # References org.key above
- key: project1 # Internal reference key
name: Project1 # Project display name
org_key: org1 # References org.key above

# Project-specific tokens for telemetry data ingestion
# These tokens are used in OTLP DSN strings for sending data
project_tokens:
- key: project_token1 # Internal reference key
project_key: project1 # References project.key above
token: project1_secret # Token value for DSN (CHANGE THIS!)
- key: project_token1 # Internal reference key
project_key: project1 # References project.key above
token: project1_secret # Token value for DSN (CHANGE THIS!)

# Project user permissions
# Controls who can access and modify project data
project_users:
- key: project_user1 # Internal reference key
project_key: project1 # References project.key above
org_user_key: org_user1 # References org_user.key above
perm_level: admin # Permission level: admin, editor, or viewer
- key: project_user1 # Internal reference key
project_key: project1 # References project.key above
org_user_key: org_user1 # References org_user.key above
perm_level: admin # Permission level: admin, editor, or viewer

# -----------------------------------------------------------------------------
# ClickHouse Database Configuration
Expand All @@ -447,15 +504,15 @@ uptrace:
# Each shard can have multiple replicas for redundancy
shards:
- replicas:
- addr: ${CH_ADDR:clickhouse-uptrace1:9000} # ClickHouse server address
database: ${CH_DATABASE:uptrace} # Database name (must exist)
user: ${CH_USER:uptrace} # Database user with write permissions
password: ${CH_PASSWORD:uptrace} # Database password
- addr: ${CH_ADDR:clickhouse-uptrace1:9000} # ClickHouse server address
database: ${CH_DATABASE:uptrace} # Database name (must exist)
user: ${CH_USER:uptrace} # Database user with write permissions
password: ${CH_PASSWORD:uptrace} # Database password

# Connection timeout settings
dial_timeout: 3s # Time to wait for connection establishment
write_timeout: 5s # Time to wait for write operations
max_retries: 3 # Number of retry attempts for failed operations
dial_timeout: 3s # Time to wait for connection establishment
write_timeout: 5s # Time to wait for write operations
max_retries: 3 # Number of retry attempts for failed operations

# Query execution timeout
# Prevents long-running queries from consuming resources
Expand All @@ -464,21 +521,21 @@ uptrace:
# TLS configuration for secure database connections
# Uncomment for production deployments with SSL/TLS
#tls:
#cert_file: /etc/clickhouse-server/ssl/client.crt
#key_file: /etc/clickhouse-server/ssl/client.key
#ca_file: /etc/clickhouse-server/ssl/ca.crt
#insecure_skip_verify: true # WARNING: Only use for self-signed certificates
#cert_file: /etc/clickhouse-server/ssl/client.crt
#key_file: /etc/clickhouse-server/ssl/client.key
#ca_file: /etc/clickhouse-server/ssl/ca.crt
#insecure_skip_verify: true # WARNING: Only use for self-signed certificates

# -----------------------------------------------------------------------------
# PostgreSQL Database Configuration
# -----------------------------------------------------------------------------
# Metadata storage for application data (users, projects, dashboards, alerts, etc.)
# PostgreSQL provides ACID compliance for critical application state
pg:
addr: ${PG_ADDR:uptrace-postgresql-rw:5432} # PostgreSQL server address
user: ${PG_USER:uptrace} # Database user with full permissions
password: ${PG_PASSWORD:uptrace} # Database password
database: ${PG_DATABASE:uptrace} # Database name (must exist)
addr: ${PG_ADDR:uptrace-postgresql-rw:5432} # PostgreSQL server address
user: ${PG_USER:uptrace} # Database user with full permissions
password: ${PG_PASSWORD:uptrace} # Database password
database: ${PG_DATABASE:uptrace} # Database name (must exist)

# TLS configuration for secure database connections
# Recommended for production deployments
Expand All @@ -500,14 +557,14 @@ uptrace:

# Storage policies for different data types
# Allows using different storage tiers (SSD, HDD, S3) for different data
spans_index: { storage_policy: default } # Span search indexes
spans_data: { storage_policy: default } # Raw span data
span_links: { storage_policy: default } # Span relationship data
logs_index: { storage_policy: default } # Log search indexes
logs_data: { storage_policy: default } # Raw log data
events_index: { storage_policy: default } # Event search indexes
events_data: { storage_policy: default } # Raw event data
metrics: { storage_policy: default } # Metrics time-series data
spans_index: { storage_policy: default } # Span search indexes
spans_data: { storage_policy: default } # Raw span data
span_links: { storage_policy: default } # Span relationship data
logs_index: { storage_policy: default } # Log search indexes
logs_data: { storage_policy: default } # Raw log data
events_index: { storage_policy: default } # Event search indexes
events_data: { storage_policy: default } # Raw event data
metrics: { storage_policy: default } # Metrics time-series data

# -----------------------------------------------------------------------------
# Redis Cache Configuration
Expand All @@ -521,9 +578,9 @@ uptrace:
1: ${REDIS_ADDR:redis-master:6379}

# Redis authentication credentials
username: '' # Redis username (Redis 6.0+)
password: '' # Redis password
db: 0 # Redis database number (0-15)
username: '' # Redis username (Redis 6.0+)
password: '' # Redis password
db: 0 # Redis database number (0-15)

# TLS configuration for secure Redis connections
# Recommended for production deployments
Expand All @@ -535,9 +592,9 @@ uptrace:
# -----------------------------------------------------------------------------
# Automatic certificate issuance and renewal via Let's Encrypt ACME protocol
certmagic:
enabled: false # Enable automatic certificate management
staging_ca: false # Use Let's Encrypt staging environment for testing
http_challenge_addr: :80 # Address for HTTP-01 challenge validation
enabled: false # Enable automatic certificate management
staging_ca: false # Use Let's Encrypt staging environment for testing
http_challenge_addr: :80 # Address for HTTP-01 challenge validation

# -----------------------------------------------------------------------------
# Email Configuration
Expand All @@ -547,12 +604,12 @@ uptrace:
# Documentation: https://uptrace.dev/features/alerting
mailer:
smtp:
enabled: false # Enable email notifications
host: localhost # SMTP server hostname
port: 1025 # SMTP server port (25, 465, 587, 1025)
username: mailhog # SMTP authentication username
password: mailhog # SMTP authentication password
from: no-reply@uptrace.local # Sender email address (must be authorized)
enabled: false # Enable email notifications
host: localhost # SMTP server hostname
port: 1025 # SMTP server port (25, 465, 587, 1025)
username: mailhog # SMTP authentication username
password: mailhog # SMTP authentication password
from: no-reply@uptrace.local # Sender email address (must be authorized)

# TLS configuration
# Most production SMTP servers require TLS
Expand Down