Skip to content

fix(sglang): never skip kill_process_tree when router removal fails in shutdown #32

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JunzheJoe wants to merge 1 commit into
base: zhenyu/m11-mvp-test
Choose a base branch
from
Open

fix(sglang): never skip kill_process_tree when router removal fails in shutdown #32

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 40 additions & 25 deletions miles/backends/sglang_utils/sglang_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -569,33 +569,48 @@ def shutdown(self):
return

logger.info(f"Shutdown engine {self.server_host}:{self.server_port}...")
# Router removal is best-effort: the router may already be dead in
# failure-cleanup scenarios (which is exactly when
# RolloutManager.shutdown_hard drives this method). A raise here must
# not skip kill_process_tree below — the CUDA context lives in the
# SGLang server child processes, and ray.kill on the actor does NOT
# terminate them, so skipping the kill orphans their VRAM until a
# manual pkill.
if self.node_rank == 0:
worker_url = f"http://{self.server_host}:{self.server_port}"
response = None
if parse(sglang_router.__version__) <= parse("0.2.1") or self.args.use_miles_router:
response = requests.post(
f"http://{self.router_ip}:{self.router_port}/remove_worker?url=http://{self.server_host}:{self.server_port}"
try:
worker_url = f"http://{self.server_host}:{self.server_port}"
response = None
if parse(sglang_router.__version__) <= parse("0.2.1") or self.args.use_miles_router:
response = requests.post(
f"http://{self.router_ip}:{self.router_port}/remove_worker?url=http://{self.server_host}:{self.server_port}"
)
elif parse(sglang_router.__version__) < parse("0.3.0"):
worker_url = quote(worker_url, safe="")
response = requests.delete(f"http://{self.router_ip}:{self.router_port}/workers/{worker_url}")
else:
try:
all_workers = requests.get(f"http://{self.router_ip}:{self.router_port}/workers").json()[
"workers"
]
for worker in all_workers:
if worker["url"] == worker_url:
worker_id = worker["id"]
response = requests.delete(
f"http://{self.router_ip}:{self.router_port}/workers/{worker_id}"
)
break
else:
logger.warning(f"Worker {worker_url} not found in router during shutdown.")
except Exception as e:
logger.warning(f"Failed to fetch workers list or remove worker: {e}")

if response is not None:
response.raise_for_status()
except Exception as e:
logger.warning(
f"shutdown: router worker removal failed (router may already be down); "
f"proceeding to kill the server process tree: {e}"
)
elif parse(sglang_router.__version__) < parse("0.3.0"):
worker_url = quote(worker_url, safe="")
response = requests.delete(f"http://{self.router_ip}:{self.router_port}/workers/{worker_url}")
else:
try:
all_workers = requests.get(f"http://{self.router_ip}:{self.router_port}/workers").json()["workers"]
for worker in all_workers:
if worker["url"] == worker_url:
worker_id = worker["id"]
response = requests.delete(
f"http://{self.router_ip}:{self.router_port}/workers/{worker_id}"
)
break
else:
logger.warning(f"Worker {worker_url} not found in router during shutdown.")
except Exception as e:
logger.warning(f"Failed to fetch workers list or remove worker: {e}")

if response is not None:
response.raise_for_status()
kill_process_tree(self.process.pid)

def get_weight_version(self):
Expand Down