From ba172e0f1e0d8e80edfbe843b8456f7e1874ea76 Mon Sep 17 00:00:00 2001 From: Sagar Agarwal Date: Wed, 5 Jun 2024 19:12:47 +0530 Subject: [PATCH 1/3] Adding support for video recording on daily side --- api.ts | 19 +++++++++++++++++++ vapi.ts | 27 +++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/api.ts b/api.ts index 1fd4b82..06e433f 100644 --- a/api.ts +++ b/api.ts @@ -1811,6 +1811,14 @@ export interface AnalysisPlan { structuredDataSchema?: JsonSchema; } +export interface ArtifactPlan { + /** + * This sets whether the video is recorded with Daily. Defaults to false. + * @example true + */ + videoRecordingEnabled?: boolean; +} + export interface CreateAssistantDTO { /** These are the options for the assistant's transcriber. */ transcriber?: DeepgramTranscriber | TalkscriberTranscriber; @@ -1981,6 +1989,7 @@ export interface CreateAssistantDTO { */ serverUrlSecret?: string; analysisPlan?: AnalysisPlan; + artifactPlan?: ArtifactPlan; } export interface Assistant { @@ -2153,6 +2162,7 @@ export interface Assistant { */ serverUrlSecret?: string; analysisPlan?: AnalysisPlan; + artifactPlan?: ArtifactPlan; /** This is the unique identifier for the assistant. */ id: string; /** This is the unique identifier for the org that this assistant belongs to. */ @@ -2339,6 +2349,7 @@ export interface UpdateAssistantDTO { */ serverUrlSecret?: string; analysisPlan?: AnalysisPlan; + artifactPlan?: ArtifactPlan; } export interface AnalysisCostBreakdown { @@ -2394,6 +2405,11 @@ export interface Analysis { successEvaluation?: string; } +export interface Artifact { + /** Video recording url if assistant.artifactPlan.videoRecordingEnabled was set to true */ + videoRecording?: string; +} + export interface PhoneCallTwilioDetails { statusCallbackEvent?: 'initiated' | 'ringing' | 'answered' | 'completed'; machineDetection?: 'Enable' | 'DetectMessageEnd'; @@ -2598,6 +2614,7 @@ export interface OverrideAssistantDTO { */ serverUrlSecret?: string; analysisPlan?: AnalysisPlan; + artifactPlan?: ArtifactPlan; } export interface SquadMemberDTO { @@ -2803,6 +2820,8 @@ export interface Call { stereoRecordingUrl?: string; /** This is the analysis of the call. Customize the analysis by setting `assistant.analysisPlan`. */ analysis?: Analysis; + /** This is the artifacts related to the call. Customize the analysis by setting `assistant.artifactPlan`. */ + artifact?: Artifact; /** These are the messages that were spoken during the call. */ messages?: object[]; /** diff --git a/vapi.ts b/vapi.ts index 21b14b8..3f3a6a0 100644 --- a/vapi.ts +++ b/vapi.ts @@ -29,13 +29,13 @@ async function buildAudioPlayer(track: any, participantId: string) { await startPlayer(player, track); return player; } -function subscribeToTracks(e: DailyEventObjectParticipant, call: DailyCall) { +function subscribeToTracks(e: DailyEventObjectParticipant, call: DailyCall, isVideoRecordingEnabled?: boolean) { if (e.participant.local) return; call.updateParticipant(e.participant.session_id, { setSubscribedTracks: { audio: true, - video: false, + video: isVideoRecordingEnabled, }, }); } @@ -145,14 +145,22 @@ export default class Vapi extends VapiEventEmitter { if (this.call) { this.cleanup(); } + const isVideoRecordingEnabled = + webCall.assistantOverrides?.artifactPlan?.videoRecordingEnabled ?? + webCall.assistant?.artifactPlan?.videoRecordingEnabled ?? + false; + this.call = DailyIframe.createCallObject({ audioSource: true, - videoSource: false, + videoSource: isVideoRecordingEnabled, }); this.call.iframe()?.style.setProperty('display', 'none'); this.call.on('left-meeting', () => { this.emit('call-end'); + if (isVideoRecordingEnabled) { + this.call?.stopRecording(); + } this.cleanup(); }); @@ -182,7 +190,7 @@ export default class Vapi extends VapiEventEmitter { this.call.on('participant-joined', (e) => { if (!e || !this.call) return; - subscribeToTracks(e, this.call); + subscribeToTracks(e, this.call, isVideoRecordingEnabled); }); await this.call.join({ @@ -190,6 +198,17 @@ export default class Vapi extends VapiEventEmitter { subscribeToTracksAutomatically: false, }); + if (isVideoRecordingEnabled) { + await this.call.startRecording({ + width: 1280, + height: 720, + backgroundColor: '#FF1F2D3D', + layout: { + preset: 'default', + }, + }); + } + this.call.startRemoteParticipantsAudioLevelObserver(100); this.call.on('remote-participants-audio-level', (e) => { From 5d316b5beeebd6b14727545822a15d434344dbfb Mon Sep 17 00:00:00 2001 From: Sagar Agarwal Date: Wed, 5 Jun 2024 22:17:26 +0530 Subject: [PATCH 2/3] fixing variable name and adding stop recording in case of error --- api.ts | 2 +- vapi.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api.ts b/api.ts index 06e433f..dc34005 100644 --- a/api.ts +++ b/api.ts @@ -2407,7 +2407,7 @@ export interface Analysis { export interface Artifact { /** Video recording url if assistant.artifactPlan.videoRecordingEnabled was set to true */ - videoRecording?: string; + videoRecordingUrl?: string; } export interface PhoneCallTwilioDetails { diff --git a/vapi.ts b/vapi.ts index 3f3a6a0..14458fc 100644 --- a/vapi.ts +++ b/vapi.ts @@ -171,6 +171,9 @@ export default class Vapi extends VapiEventEmitter { this.call.on('error', (error: any) => { this.emit('error', error); + if (isVideoRecordingEnabled) { + this.call?.stopRecording(); + } }); this.call.on('camera-error', (error: any) => { From f23dfe31370371cd8a2f6f93ce77b8d99d089e63 Mon Sep 17 00:00:00 2001 From: Sagar Agarwal Date: Thu, 6 Jun 2024 15:16:09 +0530 Subject: [PATCH 3/3] reading artifactPlan from call object instead of assistant to handle assistantId case --- api.ts | 2 ++ vapi.ts | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api.ts b/api.ts index dc34005..31089ba 100644 --- a/api.ts +++ b/api.ts @@ -2822,6 +2822,8 @@ export interface Call { analysis?: Analysis; /** This is the artifacts related to the call. Customize the analysis by setting `assistant.artifactPlan`. */ artifact?: Artifact; + /** This stores a copy of assistant.artifactPlan. */ + artifactPlan?: ArtifactPlan; /** These are the messages that were spoken during the call. */ messages?: object[]; /** diff --git a/vapi.ts b/vapi.ts index 14458fc..91d1222 100644 --- a/vapi.ts +++ b/vapi.ts @@ -146,8 +146,7 @@ export default class Vapi extends VapiEventEmitter { this.cleanup(); } const isVideoRecordingEnabled = - webCall.assistantOverrides?.artifactPlan?.videoRecordingEnabled ?? - webCall.assistant?.artifactPlan?.videoRecordingEnabled ?? + webCall?.artifactPlan?.videoRecordingEnabled ?? false; this.call = DailyIframe.createCallObject({