rbg/examples/inference/agg-leader-worker.yaml at main · sgl-project/rbg · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# Example: Aggregated with LeaderWorker pattern (v1alpha2)
# Uses leaderWorkerPattern for tensor-parallel inference where each instance
# requires multiple GPUs (1 leader + N workers per instance).
#
# Architecture:
#   - backend: 2 instances, each with 1 leader + 3 workers (8x GPU total)
#
# Model: Qwen/Qwen3-0.6B (lightweight model suitable for testing)
#
# Features demonstrated:
#   - LeaderWorkerPattern for distributed inference deployment
#   - scalingAdapter for HPA integration
#   - rolloutStrategy with InPlaceIfPossible
#
# Note: When scalingAdapter is enabled in a role, the corresponding
# RoleBasedGroupScalingAdapter CR will be automatically created by the
# controller. Its lifecycle is bound to the RoleBasedGroup, so you can
# directly use it for HPA without manually creating it.
#
# Prerequisites:
#   - SGLang docker image: lmsysorg/sglang:v0.5.9-cu124 (or later)
#   - GPU nodes with CUDA support
#   - Model weights mounted at /models or use HuggingFace download

---
apiVersion: workloads.x-k8s.io/v1alpha2
kind: RoleBasedGroup
metadata:
  name: agg-lwd
  labels:
    app: llm-inference-tp
    deployment-type: aggregated-tensor-parallel
    model: qwen-0.6b
spec:
  roles:
    # Backend: 2 instances, tensor parallel size = 2 (1 leader + 2 workers)
    - name: backend
      replicas: 1
      rolloutStrategy:
        type: RollingUpdate
        rollingUpdate:
          type: InPlaceIfPossible
          maxUnavailable: 1
      leaderWorkerPattern:
        size: 2   # 1 leader + 1 workers per instance
        restartPolicyConfig:
          type: None
        template:
          metadata:
            labels:
              app: llm-inference-tp
          spec:
            containers:
              - name: sglang
                image: lmsysorg/sglang
                command:
                  - python3
                  - -m
                  - sglang.launch_server
                  - --model-path
                  - "Qwen/Qwen3-0.6B"
                  - --host
                  - "0.0.0.0"
                  - --port
                  - "8001"
                  - --tp-size
                  - "2"
                  - --dist-init-addr
                  - $(RBG_LWP_LEADER_ADDRESS):6379
                  - --nnodes
                  - $(RBG_LWP_GROUP_SIZE)
                  - --node-rank
                  - $(RBG_LWP_WORKER_INDEX)
                ports:
                  - name: grpc
                    containerPort: 8001
                resources:
                  requests:
                    nvidia.com/gpu: "1"
                  limits:
                    nvidia.com/gpu: "1"
                volumeMounts:
                  - name: dshm
                    mountPath: /dev/shm
            volumes:
              - name: dshm
                emptyDir:
                  medium: Memory
                  sizeLimit: 30Gi
        leaderTemplatePatch:
          metadata:
            labels:
              role: leader
        workerTemplatePatch:
          metadata:
            labels:
              role: worker