optimize streaming when tp

sgl-project · Sep 3, 2024 · 66b98d5 · 66b98d5
1 parent 8a94676
commit 66b98d5
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py
@@ -375,6 +375,14 @@ def compute_step(self):
                         self.running_batch = None
                         self.phase_indicator = Phase.PREPARE_PREFILL
                         break
+
+                    if (
+                        self.serialized_memory_access
+                        and self.out_pyobjs_queue.qsize()
+                        and self.running_batch.has_stream()
+                    ):
+                        self.phase_indicator = Phase.PREPARE_PREFILL
+                        break
             else:
                 self.new_token_ratio = global_config.init_new_token_ratio
                 self.phase_indicator = Phase.PREPARE_PREFILL