From 9778d8660791644c1c105b7eda41ebc7ed7f7b1a Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Thu, 30 May 2024 16:06:27 +0000 Subject: [PATCH 1/3] everything but audioframe and endpipeframe --- examples/foundational/08-bots-arguing.py | 4 ++-- examples/foundational/websocket-server/sample.py | 4 ++-- src/pipecat/pipeline/merge_pipeline.py | 2 +- src/pipecat/processors/aggregators/gated.py | 2 +- .../processors/aggregators/vision_image_frame.py | 2 +- .../serializers/abstract_frame_serializer.py | 2 +- src/pipecat/serializers/protobuf_serializer.py | 4 ++-- tests/integration/integration_azure_llm.py | 6 +++--- tests/integration/integration_ollama_llm.py | 5 ++--- tests/integration/integration_openai_llm.py | 1 + tests/test_aggregators.py | 13 ++++++------- tests/test_ai_services.py | 2 +- tests/test_pipeline.py | 7 ++++--- tests/test_protobuf_serializer.py | 2 +- tests/test_websocket_transport.py | 2 +- 15 files changed, 29 insertions(+), 29 deletions(-) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index 3be829ba..6e53234a 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -3,14 +3,14 @@ import asyncio import logging import os -from pipecat.pipeline.aggregators import SentenceAggregator +from pipeline.processors.aggregators import SentenceAggregator from pipecat.pipeline.pipeline import Pipeline from pipecat.transports.daily_transport import DailyTransport from pipecat.services.azure_ai_services import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService from pipecat.services.fal_ai_services import FalImageGenService -from pipecat.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame +from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame from runner import configure diff --git a/examples/foundational/websocket-server/sample.py b/examples/foundational/websocket-server/sample.py index b3a4a731..104452ba 100644 --- a/examples/foundational/websocket-server/sample.py +++ b/examples/foundational/websocket-server/sample.py @@ -2,8 +2,8 @@ import aiohttp import logging import os -from pipecat.pipeline.frame_processor import FrameProcessor -from pipecat.pipeline.frames import TextFrame, TranscriptionFrame +from pipeline.processors.frame_processor import FrameProcessor +from pipecat.frames.frames import TextFrame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.services.elevenlabs_ai_services import ElevenLabsTTSService from pipecat.transports.websocket_transport import WebsocketTransport diff --git a/src/pipecat/pipeline/merge_pipeline.py b/src/pipecat/pipeline/merge_pipeline.py index 019db55e..f6f9a5eb 100644 --- a/src/pipecat/pipeline/merge_pipeline.py +++ b/src/pipecat/pipeline/merge_pipeline.py @@ -1,5 +1,5 @@ from typing import List -from pipecat.pipeline.frames import EndFrame, EndPipeFrame +from pipecat.frames.frames import EndFrame, EndPipeFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 3c80e464..a3c8d346 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -17,7 +17,7 @@ class GatedAggregator(FrameProcessor): Yields gate-opening frame before any accumulated frames, then ensuing frames until and not including the gate-closed frame. - >>> from pipecat.pipeline.frames import ImageFrame + >>> from pipecat.frames.frames import ImageFrame >>> async def print_frames(aggregator, frame): ... async for frame in aggregator.process_frame(frame): diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index 45fd6756..ed79f6e3 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -12,7 +12,7 @@ class VisionImageFrameAggregator(FrameProcessor): """This aggregator waits for a consecutive TextFrame and an ImageFrame. After the ImageFrame arrives it will output a VisionImageFrame. - >>> from pipecat.pipeline.frames import ImageFrame + >>> from pipecat.frames.frames import ImageFrame >>> async def print_frames(aggregator, frame): ... async for frame in aggregator.process_frame(frame): diff --git a/src/pipecat/serializers/abstract_frame_serializer.py b/src/pipecat/serializers/abstract_frame_serializer.py index 8da0bd11..d50c7181 100644 --- a/src/pipecat/serializers/abstract_frame_serializer.py +++ b/src/pipecat/serializers/abstract_frame_serializer.py @@ -1,6 +1,6 @@ from abc import abstractmethod -from pipecat.pipeline.frames import Frame +from pipecat.frames.frames import Frame class FrameSerializer: diff --git a/src/pipecat/serializers/protobuf_serializer.py b/src/pipecat/serializers/protobuf_serializer.py index 04b348b8..e4108060 100644 --- a/src/pipecat/serializers/protobuf_serializer.py +++ b/src/pipecat/serializers/protobuf_serializer.py @@ -1,7 +1,7 @@ import dataclasses from typing import Text -from pipecat.pipeline.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame -import pipecat.pipeline.protobufs.frames_pb2 as frame_protos +from pipecat.frames.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame +import pipecat.frames.protobufs.frames_pb2 as frame_protos from pipecat.serializers.abstract_frame_serializer import FrameSerializer diff --git a/tests/integration/integration_azure_llm.py b/tests/integration/integration_azure_llm.py index 62527baa..70109cb2 100644 --- a/tests/integration/integration_azure_llm.py +++ b/tests/integration/integration_azure_llm.py @@ -1,8 +1,8 @@ import asyncio import os -from pipecat.pipeline.openai_frames import OpenAILLMContextFrame -from pipecat.services.azure_ai_services import AzureLLMService -from pipecat.services.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.services.azure import AzureLLMService +from pipecat.services.openai import OpenAILLMContext from openai.types.chat import ( ChatCompletionSystemMessageParam, diff --git a/tests/integration/integration_ollama_llm.py b/tests/integration/integration_ollama_llm.py index e85425f8..1c5692ce 100644 --- a/tests/integration/integration_ollama_llm.py +++ b/tests/integration/integration_ollama_llm.py @@ -1,11 +1,10 @@ import asyncio -from pipecat.pipeline.openai_frames import OpenAILLMContextFrame -from pipecat.services.openai_llm_context import OpenAILLMContext +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame, OpenAILLMContext from openai.types.chat import ( ChatCompletionSystemMessageParam, ) -from pipecat.services.ollama_ai_services import OLLamaLLMService +from pipecat.services.ollama import OLLamaLLMService if __name__ == "__main__": async def test_chat(): diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py index c993a9ad..76163a1b 100644 --- a/tests/integration/integration_openai_llm.py +++ b/tests/integration/integration_openai_llm.py @@ -3,6 +3,7 @@ import os from typing import List + from pipecat.services.openai import OpenAILLMContextFrame, OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.frames.frames import ( diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index 47f65c90..5675547e 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -3,13 +3,12 @@ import functools import unittest -from pipecat.pipeline.aggregators import ( - GatedAggregator, - ParallelPipeline, - SentenceAggregator, - StatelessTextTransformer, -) -from pipecat.pipeline.frames import ( +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.text_transformer import StatelessTextTransformer +from pipecat.processors.aggregators.gated import GatedAggregator +from pipecat.pipeline.parallel_pipeline import ParallelPipeline + +from pipecat.frames.frames import ( AudioFrame, EndFrame, ImageFrame, diff --git a/tests/test_ai_services.py b/tests/test_ai_services.py index ec44d562..2365f0b8 100644 --- a/tests/test_ai_services.py +++ b/tests/test_ai_services.py @@ -3,7 +3,7 @@ from typing import AsyncGenerator from pipecat.services.ai_services import AIService -from pipecat.pipeline.frames import EndFrame, Frame, TextFrame +from pipecat.frames.frames import EndFrame, Frame, TextFrame class SimpleAIService(AIService): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c116b2c8..0b1f277a 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,9 +2,10 @@ import unittest from unittest.mock import Mock -from pipecat.pipeline.aggregators import SentenceAggregator, StatelessTextTransformer -from pipecat.pipeline.frame_processor import FrameProcessor -from pipecat.pipeline.frames import EndFrame, TextFrame +from pipecat.processors.text_transformer import StatelessTextTransformer +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.frames.frames import EndFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/tests/test_protobuf_serializer.py b/tests/test_protobuf_serializer.py index 7109d728..5a6a95ce 100644 --- a/tests/test_protobuf_serializer.py +++ b/tests/test_protobuf_serializer.py @@ -1,6 +1,6 @@ import unittest -from pipecat.pipeline.frames import AudioFrame, TextFrame, TranscriptionFrame +from pipecat.frames.frames import AudioFrame, TextFrame, TranscriptionFrame from pipecat.serializers.protobuf_serializer import ProtobufFrameSerializer diff --git a/tests/test_websocket_transport.py b/tests/test_websocket_transport.py index 601ba21a..56c5df16 100644 --- a/tests/test_websocket_transport.py +++ b/tests/test_websocket_transport.py @@ -2,7 +2,7 @@ import unittest from unittest.mock import AsyncMock, patch, Mock -from pipecat.pipeline.frames import AudioFrame, EndFrame, TextFrame, TTSEndFrame, TTSStartFrame +from pipecat.frames.frames import AudioFrame, EndFrame, TextFrame, TTSEndFrame, TTSStartFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.transports.websocket_transport import WebSocketFrameProcessor, WebsocketTransport From ba42cffcc28b670049f08d4f68758e4fa811b730 Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Fri, 31 May 2024 14:16:53 +0000 Subject: [PATCH 2/3] test cleanup --- src/pipecat/frames/frames.py | 2 +- src/pipecat/pipeline/merge_pipeline.py | 5 ++--- src/pipecat/serializers/protobuf_serializer.py | 4 ++-- src/pipecat/transports/services/daily.py | 2 +- tests/test_aggregators.py | 18 +++++++++--------- .../test_protobuf_serializer.py | 0 .../test_websocket_transport.py | 14 +++++++------- 7 files changed, 22 insertions(+), 23 deletions(-) rename tests/{ => to_be_updated}/test_protobuf_serializer.py (100%) rename tests/{ => to_be_updated}/test_websocket_transport.py (91%) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 8eb32664..5c2c3054 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -307,7 +307,7 @@ class UserStoppedSpeakingFrame(ControlFrame): @dataclass class TTSStartedFrame(ControlFrame): """Used to indicate the beginning of a TTS response. Following - AudioRawFrames are part of the TTS response until an TTSEndFrame. These + AudioRawFrames are part of the TTS response until an TTSStoppedFrame. These frames can be used for aggregating audio frames in a transport to optimize the size of frames sent to the session, without needing to control this in the TTS service. diff --git a/src/pipecat/pipeline/merge_pipeline.py b/src/pipecat/pipeline/merge_pipeline.py index f6f9a5eb..0043d9e4 100644 --- a/src/pipecat/pipeline/merge_pipeline.py +++ b/src/pipecat/pipeline/merge_pipeline.py @@ -1,5 +1,5 @@ from typing import List -from pipecat.frames.frames import EndFrame, EndPipeFrame +from pipecat.frames.frames import EndFrame from pipecat.pipeline.pipeline import Pipeline @@ -16,8 +16,7 @@ async def run_pipeline(self): while True: frame = await pipeline.sink.get() if isinstance( - frame, EndFrame) or isinstance( - frame, EndPipeFrame): + frame, EndFrame): break await self.sink.put(frame) diff --git a/src/pipecat/serializers/protobuf_serializer.py b/src/pipecat/serializers/protobuf_serializer.py index e4108060..2bf9899a 100644 --- a/src/pipecat/serializers/protobuf_serializer.py +++ b/src/pipecat/serializers/protobuf_serializer.py @@ -1,6 +1,6 @@ import dataclasses from typing import Text -from pipecat.frames.frames import AudioFrame, Frame, TextFrame, TranscriptionFrame +from pipecat.frames.frames import AudioRawFrame, Frame, TextFrame, TranscriptionFrame import pipecat.frames.protobufs.frames_pb2 as frame_protos from pipecat.serializers.abstract_frame_serializer import FrameSerializer @@ -8,7 +8,7 @@ class ProtobufFrameSerializer(FrameSerializer): SERIALIZABLE_TYPES = { TextFrame: "text", - AudioFrame: "audio", + AudioRawFrame: "audio", TranscriptionFrame: "transcription" } diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index e4c0a976..636edf5b 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -17,8 +17,8 @@ from typing import Any, Callable, Mapping from daily import ( - CallClient, Daily, + CallClient, EventHandler, VirtualCameraDevice, VirtualMicrophoneDevice, diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index 5675547e..9b48e4be 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -9,9 +9,9 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.frames.frames import ( - AudioFrame, + AudioRawFrame, EndFrame, - ImageFrame, + ImageRawFrame, LLMResponseEndFrame, LLMResponseStartFrame, Frame, @@ -45,26 +45,26 @@ async def test_sentence_aggregator(self): async def test_gated_accumulator(self): gated_aggregator = GatedAggregator( gate_open_fn=lambda frame: isinstance( - frame, ImageFrame), gate_close_fn=lambda frame: isinstance( + frame, ImageRawFrame), gate_close_fn=lambda frame: isinstance( frame, LLMResponseStartFrame), start_open=False, ) frames = [ LLMResponseStartFrame(), TextFrame("Hello, "), TextFrame("world."), - AudioFrame(b"hello"), - ImageFrame(b"image", (0, 0)), - AudioFrame(b"world"), + AudioRawFrame(b"hello", 1, 1), + ImageRawFrame(b"image", (0, 0)), + AudioRawFrame(b"world", 1, 1), LLMResponseEndFrame(), ] expected_output_frames = [ - ImageFrame(b"image", (0, 0)), + ImageRawFrame(b"image", (0, 0)), LLMResponseStartFrame(), TextFrame("Hello, "), TextFrame("world."), - AudioFrame(b"hello"), - AudioFrame(b"world"), + AudioRawFrame(b"hello", 1, 1), + AudioRawFrame(b"world", 1, 1), LLMResponseEndFrame(), ] for frame in frames: diff --git a/tests/test_protobuf_serializer.py b/tests/to_be_updated/test_protobuf_serializer.py similarity index 100% rename from tests/test_protobuf_serializer.py rename to tests/to_be_updated/test_protobuf_serializer.py diff --git a/tests/test_websocket_transport.py b/tests/to_be_updated/test_websocket_transport.py similarity index 91% rename from tests/test_websocket_transport.py rename to tests/to_be_updated/test_websocket_transport.py index 56c5df16..59b42432 100644 --- a/tests/test_websocket_transport.py +++ b/tests/to_be_updated/test_websocket_transport.py @@ -2,7 +2,7 @@ import unittest from unittest.mock import AsyncMock, patch, Mock -from pipecat.frames.frames import AudioFrame, EndFrame, TextFrame, TTSEndFrame, TTSStartFrame +from pipecat.frames.frames import AudioRawFrame, EndFrame, TextFrame, TTSStoppedFrame, TTSStartedFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.transports.websocket_transport import WebSocketFrameProcessor, WebsocketTransport @@ -52,10 +52,10 @@ async def test_frame_processor(self): processor = WebSocketFrameProcessor(audio_frame_size=4) source_frames = [ - TTSStartFrame(), - AudioFrame(b"1234"), - AudioFrame(b"5678"), - TTSEndFrame(), + TTSStartedFrame(), + AudioRawFrame(b"1234", 1, 1), + AudioRawFrame(b"5678", 1, 1), + TTSStoppedFrame(), TextFrame("hello world") ] @@ -65,9 +65,9 @@ async def test_frame_processor(self): frames.append(output_frame) self.assertEqual(len(frames), 3) - self.assertIsInstance(frames[0], AudioFrame) + self.assertIsInstance(frames[0], AudioRawFrame) self.assertEqual(frames[0].data, b"1234") - self.assertIsInstance(frames[1], AudioFrame) + self.assertIsInstance(frames[1], AudioRawFrame) self.assertEqual(frames[1].data, b"5678") self.assertIsInstance(frames[2], TextFrame) self.assertEqual(frames[2].text, "hello world") From ff1b2961d8b6f92a8f46edf9ed43fd5419d3d43a Mon Sep 17 00:00:00 2001 From: Chad Bailey Date: Fri, 31 May 2024 14:20:46 +0000 Subject: [PATCH 3/3] fixup --- examples/foundational/08-bots-arguing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index 6e53234a..9d8d0d69 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -3,7 +3,7 @@ import asyncio import logging import os -from pipeline.processors.aggregators import SentenceAggregator +from pipecat.processors.aggregators import SentenceAggregator from pipecat.pipeline.pipeline import Pipeline from pipecat.transports.daily_transport import DailyTransport