From c93831996c99cb683a27cc4b619869173b603530 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 17:00:50 +0000 Subject: [PATCH 1/6] chore(container): update SGLANG version and simplify Dynamo clone in Dockerfiles --- container/Dockerfile.sglang | 2 +- container/Dockerfile.sglang-wideep | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/container/Dockerfile.sglang b/container/Dockerfile.sglang index eca8b77ee9..c3190e7c27 100644 --- a/container/Dockerfile.sglang +++ b/container/Dockerfile.sglang @@ -27,7 +27,7 @@ ARG ARCH=amd64 ARG ARCH_ALT=x86_64 # Make sure to update the dependency version in pyproject.toml when updating this -ARG SGLANG_VERSION="0.4.9.post1" +ARG SGLANG_VERSION="0.4.9.post6" ################################## ########## Base Image ############ diff --git a/container/Dockerfile.sglang-wideep b/container/Dockerfile.sglang-wideep index 1cd7b684f2..b21d56e115 100644 --- a/container/Dockerfile.sglang-wideep +++ b/container/Dockerfile.sglang-wideep @@ -80,10 +80,7 @@ WORKDIR /sgl-workspace ENV SGL_FORCE_SHUTDOWN=1 WORKDIR /sgl-workspace -# include flush cache endpoint and server support -# https://github.com/ai-dynamo/dynamo/pull/1769 -ARG DYNAMO_COMMIT="bd91dca6141e05bcfbe9bd4dea54cc58b9e37d75" -RUN git clone https://github.com/ai-dynamo/dynamo.git && cd dynamo && git checkout ${DYNAMO_COMMIT} +RUN git clone https://github.com/ai-dynamo/dynamo.git # install dynamo in editable mode WORKDIR /sgl-workspace/dynamo From f7095843c33a5c7e0860f3bcb3bb080bb5e2374a Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 18:09:59 +0000 Subject: [PATCH 2/6] chore(docs): update README and SGLang container instructions --- README.md | 4 ++-- components/backends/sglang/docs/dsr1-wideep-h100.md | 8 +++----- container/Dockerfile.sglang-wideep | 11 ++--------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 389221f3ae..b7f823da36 100644 --- a/README.md +++ b/README.md @@ -172,8 +172,8 @@ uv pip install ai-dynamo[sglang] ``` Run the backend/worker like this: -``` -python -m dynamo.sglang.worker --help #Note the '.worker' in the module path for SGLang +```bash +python -m dynamo.sglang.worker --help ``` You can pass any sglang flags directly to this worker, see https://docs.sglang.ai/backend/server_arguments.html . See there to use multiple GPUs. diff --git a/components/backends/sglang/docs/dsr1-wideep-h100.md b/components/backends/sglang/docs/dsr1-wideep-h100.md index 5a6a21799f..d766bc3edf 100644 --- a/components/backends/sglang/docs/dsr1-wideep-h100.md +++ b/components/backends/sglang/docs/dsr1-wideep-h100.md @@ -9,15 +9,13 @@ Dynamo supports SGLang's implementation of wide expert parallelism and large sca ## Instructions -1. Build the SGLang DeepEP container. +1. Pull the SGLang container. ```bash -git clone -b v0.4.9.post2 https://github.com/sgl-project/sglang.git -cd sglang/docker -docker build -f Dockerfile -t sgl-widepep . +docker pull lmsysorg/sglang:latest ``` -You will now have a `sgl-widepep:latest` image +You can also pull a specific tag from the [lmsys dockerhub](https://hub.docker.com/r/lmsysorg/sglang/tags) 2. Build the Dynamo container diff --git a/container/Dockerfile.sglang-wideep b/container/Dockerfile.sglang-wideep index b21d56e115..b6c49113a9 100644 --- a/container/Dockerfile.sglang-wideep +++ b/container/Dockerfile.sglang-wideep @@ -13,9 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Note this container is built from a local dockerfile -# Please see instructions in examples/sglang/README.md -FROM sgl-wideep:latest +FROM lmsysorg/sglang:latest # Add NIXL build dependencies RUN apt-get update -y && \ @@ -123,8 +121,6 @@ RUN cargo build --release RUN cd lib/bindings/python && pip install --break-system-packages -e . && cd ../../.. RUN pip install --break-system-packages -e . -ENV PYTHONPATH=/sgl-workspace/dynamo/components/planner/src:/sgl-workspace/dynamo/examples/sglang:$PYTHONPATH - RUN wget --tries=3 --waitretry=5 https://github.com/nats-io/nats-server/releases/download/v2.10.28/nats-server-v2.10.28-${ARCH}.deb && \ dpkg -i nats-server-v2.10.28-${ARCH}.deb && rm nats-server-v2.10.28-${ARCH}.deb @@ -165,7 +161,4 @@ ENV PATH=/sgl-workspace/perf_analyzer/build/perf_analyzer/src/perf-analyzer-buil RUN pip install --break-system-packages genai-perf -COPY examples/sglang/configs/deepseek_r1/wideep/* /sgl-workspace/dynamo/examples/sglang/configs/ -COPY examples/sglang/utils/benchmarking/* /sgl-workspace/dynamo/examples/sglang/utils/ - -WORKDIR /sgl-workspace/dynamo/examples/sglang +WORKDIR /sgl-workspace/dynamo/components/backends/sglang From 218700a2f9271eaa2123bf5694c8b5d5341db6f3 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 18:14:38 +0000 Subject: [PATCH 3/6] chore(pyproject): update sglang dependency to version 0.4.9.post6 --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 245711df19..0508b584fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,9 +73,8 @@ vllm = [ sglang = [ "uvloop", "nixl", - "sglang[runtime_common]==0.4.9.post1", + "sglang[all]==0.4.9.post6", "einops", - "sgl-kernel==0.2.4", "sentencepiece", ] From 06ce5bd098ed2c37b1365a21a5f3c2145174bb5d Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 18:18:55 +0000 Subject: [PATCH 4/6] chore(pyproject): remove unused dependencies einops and sentencepiece --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0508b584fe..b15ec28cf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,8 +74,6 @@ sglang = [ "uvloop", "nixl", "sglang[all]==0.4.9.post6", - "einops", - "sentencepiece", ] llama_cpp = [ From f5a1fc1162bd50ee013ca99b8400eb81160428fb Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 19:14:08 +0000 Subject: [PATCH 5/6] feat(container): add sglang-router dependency to Dockerfile --- container/Dockerfile.sglang-wideep | 3 +++ 1 file changed, 3 insertions(+) diff --git a/container/Dockerfile.sglang-wideep b/container/Dockerfile.sglang-wideep index b6c49113a9..08a3ffbe38 100644 --- a/container/Dockerfile.sglang-wideep +++ b/container/Dockerfile.sglang-wideep @@ -161,4 +161,7 @@ ENV PATH=/sgl-workspace/perf_analyzer/build/perf_analyzer/src/perf-analyzer-buil RUN pip install --break-system-packages genai-perf +# https://pypi.org/project/sglang-router/0.1.5 is latest +RUN pip install sglang-router==0.1.5 + WORKDIR /sgl-workspace/dynamo/components/backends/sglang From dd1382fa22f18792fb1518bd99bab290dd5887a9 Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Tue, 29 Jul 2025 20:05:16 +0000 Subject: [PATCH 6/6] chore(container): pin sglang version in Dockerfile to v0.4.9.post6-cu126 --- container/Dockerfile.sglang-wideep | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/container/Dockerfile.sglang-wideep b/container/Dockerfile.sglang-wideep index 08a3ffbe38..0bbcb3af23 100644 --- a/container/Dockerfile.sglang-wideep +++ b/container/Dockerfile.sglang-wideep @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM lmsysorg/sglang:latest +# This should be pinned to the sglang version that is installed with Dynamo +# in the pyproject.toml +FROM lmsysorg/sglang:v0.4.9.post6-cu126 # Add NIXL build dependencies RUN apt-get update -y && \