From f57bb65297d0bf45f06c55969847a14312cccaa2 Mon Sep 17 00:00:00 2001 From: "j.foucher" Date: Tue, 24 Feb 2026 18:08:23 +0100 Subject: [PATCH] WIP: Emotion facial expressions + client tool support - ElevenLabs client tool call/result WebSocket support (set_emotion) - EElevenLabsEmotion + EElevenLabsEmotionIntensity enums - Emotion poses in PoseMap data asset (Normal/Medium/Extreme per emotion) - Standalone ElevenLabsFacialExpressionComponent (separated from LipSync) - Two-layer architecture: emotion base (eyes/brows/cheeks) + lip sync on top (mouth) - Smooth emotion blending (~500ms configurable transitions) - LipSync reads emotion curves from FacialExpressionComponent via GetCurrentEmotionCurves() Co-Authored-By: Claude Opus 4.6 --- .../PS_AI_Agent/Content/Demo_VoiceOnly.umap | Bin 59461 -> 59573 bytes .../Content/MetaHumans/Taro/BP_Taro.uasset | Bin 591107 -> 591109 bytes ...ElevenLabsConversationalAgentComponent.cpp | 75 ++++++ .../ElevenLabsFacialExpressionComponent.cpp | 239 ++++++++++++++++++ .../Private/ElevenLabsLipSyncComponent.cpp | 41 ++- .../Private/ElevenLabsWebSocketProxy.cpp | 62 +++++ .../ElevenLabsConversationalAgentComponent.h | 38 +++ .../Public/ElevenLabsDefinitions.h | 47 ++++ .../ElevenLabsFacialExpressionComponent.h | 116 +++++++++ .../Public/ElevenLabsLipSyncPoseMap.h | 38 +++ .../Public/ElevenLabsWebSocketProxy.h | 20 ++ 11 files changed, 675 insertions(+), 1 deletion(-) create mode 100644 Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsFacialExpressionComponent.cpp create mode 100644 Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsFacialExpressionComponent.h diff --git a/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap b/Unreal/PS_AI_Agent/Content/Demo_VoiceOnly.umap index b00a0021bbf7a8852df9d1b0174c47b25c722b69..a7094bb9cf3c924e176e9c093c53b3528bf3442c 100644 GIT binary patch delta 832 zcmX?lfqCmi<_+5z1(cr|nzG&R@!L>h;4Qf0y5!_zj0LidFBnk3e;^R3VqnN;WMF9B z9L%KU%Q$gzoqs;pM4$i&Ea;!S&{bk`e4x|hzyJ{@!--&SzOUBgl>s_Tw-8*(Koz*A z=1EXZesCd?DPW=VfniKbr-GUGL19d4GvLPE4ANmTo(1I^O%4oJVLA-q&JQ!1d@xvr z^~x-udnPv?7oWVwm1i<X*8KJ zQibW_N-)HT`Rdd?UfrtKTRT>qFbrkfkViVntvFllWCGYw;b%w1pw^JBx96nBF~1>-`Ps&|7$ z>*IjFI|OFF2Qo`ufS8-(;}e;+{mT8zj64H9&3r5}O+xYtGhEVw0!(rY!`+?BBZ^G) zgTow?ebTZDO|mQ|8z#$c=15i$V>i_^G}ALO*sNbEFDP-L-#z8-0ow!DY$8|e-e7n@ zj{B=dv`DnWWQ8r}o9A{tX5j=Sr@O$!mI`)(K%d#>-ThXK7{aoX4@_LS`M|_Km{Mk& zzfVTjvDs|uPZojf?V_8eY<6>AEBv%+Kg-Y6o9$=mFbi<&Y`c|P9@Xntm_ExVy0?CF k_1xPE;yPBAmSqkM3<9O;A4-8V1QbqYJSDlgZQp)P0QloI`2YX_ delta 717 zcmY+-Ur5tY6u|M{?`~4m2EQR|F&NE|f-Q72*QA#mlnpj@e{Pv;X}?^81vyJ83z9)k zQo4DQa5%%XQ8At4+MumqTbT8ftCt`M!~`KR_7F4(>h3Iz&eP|9&w=|qFIifVEF^is z;Pnlqf6QCb%2{n5Zw1I3-mg(?aR1G3Ho{Gg>*YC4N!>`-%ny*;t-ZSjSSZb%QDQl- zCJF~e#UL=i_$UM{IPBYpwlPSe^IyskNO>nlrg5Ofg;DYWFv0kG0$An7$Q=fT+_=IQ zX8)4HvWmn`$cP+UQMA15!x$MS;ck`jQ<=_F7|leWW*OJZG@iz&Fa>p*5j_Md+Aumi z0}L=GXMu%MjRM6p7|EXi^NiwCprsw756^%J#se{6l?S7SIdZH;O+t)ob5QxS7=4_F z+Q&E)hq}smaslY{V$_p>gDhie5okQe_yP{5851d}iVlo+r=k8_mXUiMR)li!EU=Aq zqeLopc1rYDT-Wh6dw;jdtiRaQ=4rQFvSWwd)9v>U+FZWo+B(tYJ8r*xp&mC7W0!{R z?9%SpVW=_G*VLJ5?chGe-eOK3Gvut9iM!RC+*EZYqmwi$Ro+GtTr|_j@S2naZ&eUs zeKQ2#S^P51ZrJoCPDKIePfd(S=hd%oAxB|o~aA3Z~qL@L1MlCDru zDH8H5yPp;C>L0y(=hE|zbsc((C6Ugk+=d;s{K5$F=NJ4bNajaKdO1}dTkbwT8GC+~4hlZvDrSd* zOPzRQX*?ZEu%mVMiKjz@1 z07ORqD%R%20D-d*>%1O2;Yx}dWI0O%Z&;W7y^G55cfo$L+F|Xk~<2JwS?XT zcPfY=g1842tY1fn@P$xKgfR%uHsBF!E}-C-79qUx0u^X${{?p1VtCy~o}$2$s0ZeA zX9Ma&9bT*gCgCLqIJ02sg@V4>2KPz?SX7XyM+moK$${`e5 zBOHRgQSi?6Z!i1x6(N3XsgD8(Wd45*hPu@Z*Uhh_v7bK5pmq1(FwFKM1K91yq2wY8 zI7jft@MU$(>JkMUJwbqG2P;P{leRj9lkMpr9j&&|7kq1{y*Mu;v;K+`3JGSO~lC5J2oY z4J?kqfq5MTs}p&>>e%FU3Q*_Zz&#;LKN|R)ii6J$b*$Ns0)}%4LkwY;Gce8-(#@9-shg<2BT=eK+Cw4>&B1SH((h(!hzX zS%9a!CXfc&`f%W$GAED%21an`2t+~5I3e68hz6EV5nvcX5`MgF{o zIwl-R0V_iY;07UtjYIGqb*vx4)EykS4+irNT$s_k2ozknhc~#(c1;uwj3(nyau=?F zC<^#~pAaE-mjgef=x6JP+%YbcRbt67#5kg^65Y;)Qe&REDEv zAqpB^@EWQZsfY$tUlPCpf>RX%Iw0iL5P(?>(@!l9!No9RH{gwu61YJ;p@2J01mHmU z@Rk4)B^03fjsVvoc(?HY1<&5|8gPRsr2z6r9EP95LIJ|@E&>=lrGVrf0^~#Z+{XjB z>kaT4FhM?}0H;9$2tTKQydeU(L6{!KVQCrMBp~RH@&E;w$9WC-P?S-?i|+*3R89e+ zQv~2ZI6Z?yM>z^oe-V1Gzo3AxB=VnJ$UWR7g!TLcu&SVdAVC5&L-nAgC+i;7|?MK{W-aF?fIi?bUcA_BDJSYG~lc8p3$n8V2~WjsR^C%r+9h=rseB zXz+dq3Ou#&2B}dUGpMBicU@jDoC65PTM3|8M*$++2oMgT48@_g4h0vt6MEh1DL~s8 z2ksI2A?Py+5&Ie_0PG}0R6r2j!vhouTksmNP-vup1qTV^C7LLp`Y-{mL-0FFfJq43 zPT;WS4GQ>96MCE9P(YD20Sw;4Ij|)_K7MjUy7f^=uR;ZO~8K?@CJ zyYhO~F}D^9aPS~RbifE#FC2p3!J_#cT&X@hK*85bgzz=3G%$Dt2ktrwXk~!K0fY## zHX5J?5x@t67=}Yh8w#p!;SKKLtlB9c_BJ8Bww(r2A_=hhJq;w?B@D@bPXWhbaTtD& zf*ben#?p^47kq#>HHiSi9q^{6;J}>}hYlJLNh3f91hEX>5O{|F@ft7}e58Qu*@W;l zoiq@VM~L8b!i8Bt0EsTRFpF?tcENYWCwL<^3vQ5Ibbvm^fjfsr-3$;?L4ZOCp*1*g z|FojkLkFxz9DI6E@b5c9_}pFw82>=#{@q9(i*=@BzP)r%^qCMb2qVsYA;7^t28iN8 z9}0|y@LukCy?#2dn1zl{a1Q$6i`Q4ecbMu=#JP&|%<*dTm8o*?w*4^lwDcO1BLIQI(!q)igQ6GGz;!Vuvh z3V1w)!{#9r#7`4?`(dx&3<36ig-^;)0#rcQ`-=y#Xdb2m9Ypv~l6M$J!D15L*qlMf z1V-qfVHUV&Xgk6H5qyM*ZWyr#5Mb9gI-v6Zfrqi*lEq0wxrV}|3w$iYfS2>Tx2m1_goMF3!QwVGal>DTQ^QV#8>l8OGd>pY;r^OG zd*Mvyw(+r~g$kl;Vbf`6!Q#4r8~wV%vTf7s1EM5p76k0*4^yykNF*deml@m?URZdV z&q7J1w<%z7Kv!7LAj6%R;cm+BtmIh>zndE<*=v~cXYJho?svePmFvL8o zAbQ4uLy}(4NAFo?op6m2s%6^QS!~QrL$rSOCv!e+my!t`ghv-+nteWOxPcqb(xaM^ z%#vEg#SNqFO+hl82tV!>My+h0R?I@ovhlF!!B_)pRnv!df`=l$pz`J*K+N*Fu%HH!L6Her@uX8 zu3k2^=|XNz$(-u7|48{C{qHSH;3(;Z^gz9)xt&bH(cHd!Zm#dIly(1@aj1uXh%0O? zXlbpWUgDSPd@ZB5(D$mufh+NLkF5;@iiT4w7x)=kAR#)Bys~~AXSqb{3aI@Y>NHL) zZks;Mr=bw%Efwv$Gkbw&huLcqyh@;%Iiw$^Wu)zgoC?$GpChg=EnvW8>9ns$iY&+D zUuic~Dv&#FQ`kB=O9mAS>YEf3kVSv;XESd;m#@_}+E zJzZD3ye{Wn%gJ*&<#rA(FIhI-F(3Su6w7QXvzIg#7KJ(bMFxs!AnLM}g*s?m+x#f! zH4%;)fb*3v3m>v)3SR=I{*W73qJ5@{h(>R?=T=gK!j>c$ljdT6oJ zo~5r>lniYv88VLZmNc#wSb5tsnN?^MBQ$xnB~*G^@@BLwyzC){N9_rCh0au@E_R!~ zwcTw`&d`LBp63YVI-`XMK%LyReAceFL) z#Bzz+A#_SGP}So#=nt%e|F2tE?4v7sFX^*(!x`-cf)&eLh_A3j6;Z zVHascb2xGr%T!9$r>i$lR=dn7`DT~jy%QZ8Sp8sIsr(b6*1MyzE2S^mo3$28)gJfV zJ*WA+gIQMBT@#t8l<*TL1q9W*JnFWuXsFs`Y*w4O<<7AbJr(nkM_v!8Al`d> z^ITPrP3UK&vrm_nOC3luzs-7In<%)nINL@uHeu3z;ejs^8|a4rC{)?5G8UHFqvXDK z(1vplDUtgsuqWR|(N|*q^~#fpM@4mCw~h0sp37N0cq(1fan0Mk^E@QK9QR#qE1wh8 zvneRH{D9b@{SOSf2iBg^6qWMSb827mW|a)uaY3VV=+vD(MiK`d9|npUCB0&Yi@_Xx zu1BF)=GnPJC-qLKk35VEo$PcNo@sGb(=flkj%oE^!m8oTR2EXHsCb%GG2y%2G^t%& z4WTmi5WUPog}3duRH&pyWur-a)FWd_J54465Q9QbtBnuR^!1QUS2}~MR=`eu_*;HQ zDYQ6Rjh!qlGcx=!XYZv&x1LP0wxQBB-A2mS8os{2y7JAbRa@onoSF#n`7xi>eJCjN zspZ+6*K#*$W#anQRRwvOUmN{R1r--`h@2Aj{8?D#s8Mr6wDM?1gkzFi)@@m~`=>w^ zwe()m`~j^EY8Q+?k}Y#>R5=dIdnGi>hYl>U3)_*)FtIrMu_lZjZEpT9ec&=nc5sho zPgiw{Q-gSLZGYV&jNd-q@16d5YLqsge|efsxrFlG_@kfssmuJkk8gh4bB%h*Z1R-X zmo`P8-lAnoojGM5t4F9aLZ%+uuPW%u+b2yuX!MCvv#q>m zsee0gLaA7^M=ul(YQjf?CJUHl~vb~Z)Aoo0_>>vlNN!mX;C)TFg_3Uvj`M&pmU?+nc% zNnT7^nV{qTqfVzyO&Wnwr`wCN#i=AMDpOZr%P+*WA$!S8r|#T)331-CaQm}rU^Pr- zeL|RrB0CB^R?l0a>~io&)X0~kt!ECL0{vC^1?7Qe-_u=N?*B{k{PO){q^P4(#=^>b?vj_+ zW%v!5mi$7t`m^3Ke~P=weBi9ThTW`+?o$rT4l{F^tjo~|>B;J3EU)gYxxY~F**PzI zctX=jfr-ndobfUiC$(!TE}(U%V$NzG4Mg(k<+ue}c`nd*v)e)&lPAQLK}s{2f2)JV=kf3f}PVx*0Q?ik(v{ z2n{yaROV^psxVT%Qan86Sf%;R05k81CqunjHjd)24khNNsSK~sn(H)-Zn0M+RSwL_ z-Fe$nCn&XPsw_)w_j}Hc%{A&X%34LkX_R@_9^AgTWy@KKa?AFD;g16z9{aC{9Je~& z%A}qzV>QkQ8o70=960%L`yRS^kV&BmOb4GtL~KUkVtF%F9DwL^_i=8G7H9HU$pW!} ziB&Ex;cZdqnZt(2?+b?COAYVL3nj^mFZxr1@l{N4ZY>LVs#3M;`DhH8Gdwn}_-=7s z@Ri!&=ZJ=1poe=cthh)u6D%W@2a6S>Esn_FoNpMWCUf+|xnN!C^>kW>&Bop8IUiVy zzc*b(OjW-q>+J5Km3*=gG@qf(Ov~Bv zu>|wH58re}6F9I$`+d^&@RB2RP0<0P_Vz52%Iu}_XTgLh4)JD>!L82isZy zG-%{c%5~mlB^ZJgiw9Y0TQ z4f?xM)O`X|DNj(3m9(H#1nX@4)K` z!3sE8wweLcflb&;SVzG++~aFJ{1$Vj>ow?0bi35fExP#NBueYJbof|UXx|!t>1*|Z z!Z+u5r+tl_BvNRpemdx^)I7J;v_mH%z9~71Z+Ww$ba(NhN9Kl;>H)~WclX^w*VgxE z%^dacV0p)#XgMsji4LNhLUd0x{S4cou+6Pn`EI+`fQM1etq%*=S9^0#kVw^9RVxH+ z?J?K2_ub2*|sE8k`eZ$p7Wn)3vx3gEQDM2?=d;0=AY9~D?lZ-&5vNOmn3as z$sIO7cwzD7y<``1gTjGT_71P#th9Ajx#!z)gyt{h?V5JOglcj+T`%EYNbYo2h}JU? zZ(|)p-Hp$NC`A{^aU;`_PV;}(i{(X6dyJ-Do{Eb)@2ocXioJN=@vyC7$6D)Q#yz5< zLN(>L<+qT!VOT#ppIX$lw^V8Y2>Ks(}`_}R{o`4Lf6J2+fK4WD-61TYbUKNAoECji#Tzb7Dq`;HyY;=a#u-_Gmm;E}jX zg`2?Ylaa)V36`X@+RwrN&HILk;^{bV9k#j@$wELA{)-{(DqD&KKYZ{5a@kV+2OIfX z1=yJx{0rH_BmC+>ZgUy?iIIRLAG+o0kr96Q5r7}T+#@4GzH8)NWs;DCm0D=Fh`rOU z(&W;Q@LwPOe|y2Uv=NYC=P8g`xp8#zTOdC&dG?`8gURN*kCI!epY}cMgroky{TY=D zM}_3E3kS&6xtu;S9k3^Q$vJG>Ua}Wkua~UAmhUC6XAk#~)!8LIWGQx351G#Pf!}iH zbdz03?2{d28oRy=_PBJB50X}~iw4M9x%wZ;jwJTS4)O^R8F*+C{DdRyj1IC4JF$aI iWrx83h%oS;a)iCJgRHc0HQtuthX47=_4`bAqx~Q8UG`D{ delta 9629 zcmZA52|QHm`vCAWUNVXrHEJrPny#f$RD>eknjHJ?Mk&&QvW6^aXir%WCP^r3QCE=^ zX+t9WUI@t+vL$Q(cidw-|DR9OXP)o-yfbHcm*>o-0Q!Xh`oDrCQVu3Wx=h_CN-Tj`gLh6nRH4v5;m0c3LwP4-|#Odg%=_5AV@W#%^Zv8S^eYy&n8;{ zo7rnU_%>h}_9&1@gu@QJYv*nDItqL}JQ2GIM!i-p<~FdgESCT>4bOzac`?EOrD zKnS+YIJBKYfm92j)xm)QJ`!L)Vc3D%<`5d&xsyGFIS72RZGy3Ok{|zXK0Q zWppgji4O9<{^^4WIWvGuCl2xMV%Rz7jo{ojLPkH#;OoX=w+jkh_7GY>x-h_SF9B3t z89@L0A7G2?!^@QAN(Z<5|0pJQngKcoaBw+|0{Ra^>kMqYHADahHwGXN6QCJF;5@{; zZ^MqcQ^B1-fC9=Gp3Z4ydQgEr5waiz5n;m_D)>l*2nZ*B;-67@1_jg!!Uwv0Qo(&9 zbV67?L&z}oqJn!2zW)Xe=NJkgw9f;{8wIoSgw{lFD!93Y03tqAuw^R&Es zq}#D1faX3f(6L$;9b80ln41}X46sxehfqHhG#U_E1N~{3f4%WBiFqg|T@cS|j8JAJ;;~JsW@CpsIhTzb4RS7G;LIG9b z1W>+80oY9fq(U&c%>@*Q#Nh$w>-?|LfNT;D!q=6siE9*~m`cd74xoYFG#p%lm9WMD z3K)FK?E=s6IiWS@It3KHBY^Y`3aBmPro(FhA)tx?0)Z5;s}6_VfhZVlB((Ox*3f1G zmy|VDl9R&PmA!)4->099)7?&@)WvzcPdZ?))UA`-jlL+9?7IL5TWI z0P9d1Pyh??=@*KE144Mfxn!DQ6hILsKn{cgDh`}WCXG=*pEwSjL%xL3!Mk-hRAMOb z`is!tE1Uv0$q>?O!|C9HJOP*ybZ}6C0G-#USaAdeh^fqDBnssA;sNIvQX?rqO^*O8 z*%Y9&p8(e&I2<9s1O)694)QlqfI4$q<*>?|G+^e9L&jw}Yqw@HQNrvD^$4cqefC z!*LKp1H&meyuJgkK@0`_d_c$$zC!~iG6>)gp*4%!1qE?AgjVZV8kl%VfJz81Z*kx( zQnGOjpz?km;1OoT(ZFO04nuJ$u&BU;xp5Fr2TN)RFb3gWBLR-xr30@P95U{rK(vF< z%DhJh+1&)lf?za&Lt6s81_?B%}KLm+gT%co}X>^ddmjI^ebnr?8hxl|99M-~vxv}t&0dn*R5CcKN z7zfg0xGp@RfOlpDh=Op;0*A^+aK>2@TD|_IfUj2ER(K5{ux$uno&QkWL8pUO32SqQK}J9#}txYeFUs zSojmt6|(5S>;fSpBZ~$qt`Rasp3*_-b#4Y+6N2zy=px)7VQXy2d}|dQJC;ockzw=w z;Z6OR%>XiN99$mIG3GNmI2w&Z=ra`9#c~5UA985G@!p^QY(G6%Qby#!d6*q?? z%I6e7O(VcH2=f1O0R?)Qc)K${!YMAG;P?zTfMei21>FBlfNKy2NaX+4LC#lBK-j`d0J~xe2wH$c zd@%}o$#}rIsk=YGd$@=I84z5B2(Y$<2A(bngY?mc!jbp3q-tfQ-49)4{K; zILcSR>rhSs2HUxSf_xS3XTWh#K>rk-w*q_#A{P0HsGJNI2-vQ?SZ2vT3jrA0eYfk{p26#Oj2#^C| zn==kW4e%krl^ehUx{(6X+;Je4$zh}>8hGMK$na{S04Eg9QW11<0~1Be9T(DoSxOGD;co8a2eOaV(U95~-F)Jy{<5d^Sq zp@G<&gf5LO6p#{)gK#Sf67LYwGg@iDIvxkk8Ls_81J@F8aOr^8;0wIL$pjE+gE#m- z4!he>Ku+fda6G`)CI1rA&D$xUZRi%3;=>G!RnE1r&sp;{m40yKn{fKzi^nl1_KQ6U;+p00TfhE64C_*Dd71O0lXl@PZOXA!qpi9nE!w?Wfq6f zA1E;U&26P)8;0mW6%qJT_y!Kac|gM9_3$=KWS9!7=7DpB4#Ny^lZTsu0#m>Z;15e9 zRIrp62hw9YcGI5vq;bSqgw&s{B|tjM!?L(GI~2E|l1U}eS#C3Sqwi-2JM`mtq9mKA z?M92$20PS+VIz_{eo5OoGi9yrz6^iyH>Ppb5hIv0tD}5 zlEg$d6^!0HGHJl0G0xsEND^a3uc!*V-lHz4pDUwfN7|=has204oPcdgc6R#e=sLfp z0hrZX2VQp>-%8C4U+IJhB{3-;3mMt|zI`y24;ws3ZStOeD~ahfybLFasO-x7W&Cq< z;(DQb8w(J%fa^Zqm3eu#q(OLa`bPV|3`>7nQRTPpZX(P+b(p1v1O!OM@xYUmk%>_2 zj^j6k*NGbI^CPJ`VXF0#qTfO{mS?qH<=8GQBrGg4%x~3i#I44CJv?6np1R1`#F8^B zBP8xv-rHdlti1Efxrkc}JRccVxYHPEJL4X&c@ihD`NK!Y9l>y+c+N1fH!At!gopV?cO z60eP`8-}HBGlV1F+bd2LCZv2CTvZw`m5>-Guzo7JPd`axoVCgqq1I1j_MV(+LW1CV z@pJCy-dInQOaz`6QqJ(H>~SrYS91{i9F9O+I@XHZ11z_6SaVzDA|jNySxVbkV5DhhJA->nE8< zzc7E-owMb#wHD7V=`pPCn5sbr8LYVr;x3EGr5J{4!mDtab7MJaoWOxfQl*y5 z`BZ*e;*I$3{f&55=e{09O|sYLAzK2@MEa%-muDd=yPluR?6umVw9V^6i{Ba@Y=D#h z_n9A3r10>WD@}n{n*4;-#@x<{SJ>1nY1(Wa_E}UW=&|=mgP9P2Ugnds(7i9%E=%BK zkT|e+nEg^EHZ@h^eBqXsgNIH?Oi%YudYk!X72UZV8y;NtsPN5(S8I%XsxIF45O}Ta zI(>Stz{0c@j|bWl?TULndxC0p9zWr^0K}v>VD`r0owBQ`(uFDsMzuRvC-)9%%HB#( zuiLWl#|_^ir?+E|lyfp~6}D@iyjkj6=={Nl^-o$)y?^*aua8#0Mo@Eye@rgz@)BQu zYVm`HS+QMf?5i86*jsQy{FgT+0J^b`WuPZ7VWY3gO<$qCv6Tp zFQ3ez2EIx=8eIAt(JxCJ5+;}W7;DftzVE-yB8y0D#=vUo~533HTEIT+5Jxtfy^?d6AZS1^; z$cH>y#11k3fPVVUaHZ@HxbA6z<5Fr`NdJ34ywl8UNg%x_)SWxdc#%5RQaV5q6Bz7Wpt z+J(Mt#s(xN)qwZF>Z-uN9-~Ti;Zsgpt|9mM)79cxxwbsIeSMmFzV<3oq|)u80lHVJ zV%9Wo>qV|xk0whsxq2qoMlV{au(4bAh@izki-uV@zMR=s^(-p2Eq}58!?MGIkuI?< zTKwId>+bz`krq~*T_4`(ebVk0E3Y=yX!-UXTbsQvOdNT^;cBKzV}i@z9;KT7B9BNegxjg`V2W6CtRgX zHjlFc;l|mr4GD)kG3bB``YD!Mbqa~;?R2^Sd#=){^u{y2?2?@8ES?S}@e9H4X73zb zEM$LDB`wVUO>doL;*q^(dWzFGm8L)1Er?8ud=%G8wST+6-qFz@R??%PXlnh9Syk4m z!ni7v?Jfz*YClrj93IDyrz{&ZDITxyLyxn@@|xe+e30pA$+F7JSCtJ39v?dR8)-DX zE2uVp)ZhzI@Xwj^ok0-ot~=S z>r`yRPaYW&my~QtG6@q@&^Y+5O+T1Z8o05%{pg8smt2tFtRRNAy4ASN?S-8E&?I%7 z$uvrCrXrbh^^L_bHF($Jcy7n&!nLPm8)?G^St?RDm2}-o=nd%G<6Etl$tr3xR?EEe z%1QN^?XxeTHR2Irj>ofI!#3&bOxFD{STB0p?D&IVY5gfvDS@=bv#0B_B@B*?DSWhZ zJ)+}NuS>ls$rB>_^xqfmnpdhP7JhWfJ-Nr!VKV%k=tywY6S$>UBHaNKSLsKVIhXBN zJl48o#eq*22X(Lq+JCiQ>l@x1WV0?I>I`z@G zEsu)ST|ZH{C}b`5Z8NIifg_51IxPXP)SMG~A# z3bLH-y8_t4Vl`|RTtACA<&E*tsr*2dXvyX{a!O(Xr(8;v**5hiflA7^-X7(Y;s3$K zRDqSVB1b{tam>eCriN<$_P>HuvkkZ3w;A3w<6Q!)&B#}>RHJGh$*GzqpRsA_ zaKyM{sxVqFc+h}194_BNtmpPJ5wkrhDahRbDN>8|&wHh6HBC|^7JQq`C%DC@hEox% z3&47AXU3rZUBC5`C%{<3$dW7ur+PI2xpSqC(3Y4r#_EfQi}EBRRdv+@5Nm_npql9N zEM?ATgQP4(J!_AJL;7M>A=}2dlT0H$jngfUvvRLrt@)|nXGoDcB|cWsy*cY?ti`MJ zX4*y`R!O1s`vsn%d$dcVFL3*vjSM{eWEi_rz;ZVy`@$?asn;TAWl2i*0yzsNO7pAFP*R@Op^{PUxD= z)*iR-)WFILo}msrL-DzHoW)Fk9N#sX;0izw9;ogKov9vU3$G%r=3mvEAb5{gi@kLf zNtw0oQM&t&L&8>T8`c}dJLkME()!x}FflVEmjQ$?ge(!suxpR%H+`R(8D{nyi447%Mlho>&UI*pLYFh-ZD8W_Ha7kX1$U-#Xqy(J1lE@k52k}C%9?QNy zU5SkMJvyAq5IJ5rde6q&1a5=Weww4urm->jD3ZU>0nyI$wM*M0xY#bddv*-@Yd($F zN<>*z`cr+~m8qn=aRO(8Ew(>OXEKo;HgGk`lhKmmQDp`8=owM1#*$5(#siUE^IuH* zzFqXuI?To!PQ*-D8NoN<4$6*vYH)d&wk4f;l)k#Y?k>Etbh%f3x(zMibI;1m|MqlQ zxX8i>Up}o^XA`)_HX-gX64o3~h0n`la}SZ6C(3(Mg?{wAdT>6^Rm5;Z5PAKt{A?9bk_8|94+*^z zB}#%X9{2*;QKGz-wLIN`UD?39jJ;@>R|&+a7qWM2@~r}KyJOfH#k{mDKEu55#Rp#t zvVDg6Ps!6?DPRw`xf}ldL@&y}Z1Bg~cKBoC{~9|^iX4@lKqof7w|Wnl*reBC6mfA^8a*<*dM zC8Uphj3l3ZxQpyUB5h*pwvnH)!~4kA?4v>al*k^kAiJTPESi0!i|j#S@9QFqve$Ky z>FfoZchA3(b=Y^lk>xn=sO-^B@;UaFPV$><@piHVU>|HFzhpbt^73YfekZ%p{trcD Bmw*5O diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsConversationalAgentComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsConversationalAgentComponent.cpp index 1ef314a..0adf3cf 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsConversationalAgentComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsConversationalAgentComponent.cpp @@ -179,6 +179,8 @@ void UElevenLabsConversationalAgentComponent::StartConversation() &UElevenLabsConversationalAgentComponent::HandleAgentResponseStarted); WebSocketProxy->OnAgentResponsePart.AddDynamic(this, &UElevenLabsConversationalAgentComponent::HandleAgentResponsePart); + WebSocketProxy->OnClientToolCall.AddDynamic(this, + &UElevenLabsConversationalAgentComponent::HandleClientToolCall); } // Pass configuration to the proxy before connecting. @@ -429,6 +431,8 @@ void UElevenLabsConversationalAgentComponent::HandleDisconnected(int32 StatusCod GeneratingTickCount = 0; TurnIndex = 0; LastClosedTurnIndex = 0; + CurrentEmotion = EElevenLabsEmotion::Neutral; + CurrentEmotionIntensity = EElevenLabsEmotionIntensity::Medium; { FScopeLock Lock(&MicSendLock); MicAccumulationBuffer.Reset(); @@ -540,6 +544,77 @@ void UElevenLabsConversationalAgentComponent::HandleAgentResponsePart(const FStr } } +void UElevenLabsConversationalAgentComponent::HandleClientToolCall(const FElevenLabsClientToolCall& ToolCall) +{ + // Built-in handler for the "set_emotion" tool: parse emotion + intensity, auto-respond, broadcast. + if (ToolCall.ToolName == TEXT("set_emotion")) + { + // Parse emotion + EElevenLabsEmotion NewEmotion = EElevenLabsEmotion::Neutral; + const FString* EmotionStr = ToolCall.Parameters.Find(TEXT("emotion")); + if (EmotionStr) + { + const FString Lower = EmotionStr->ToLower(); + if (Lower == TEXT("joy") || Lower == TEXT("happy") || Lower == TEXT("happiness")) + NewEmotion = EElevenLabsEmotion::Joy; + else if (Lower == TEXT("sadness") || Lower == TEXT("sad")) + NewEmotion = EElevenLabsEmotion::Sadness; + else if (Lower == TEXT("anger") || Lower == TEXT("angry")) + NewEmotion = EElevenLabsEmotion::Anger; + else if (Lower == TEXT("surprise") || Lower == TEXT("surprised")) + NewEmotion = EElevenLabsEmotion::Surprise; + else if (Lower == TEXT("fear") || Lower == TEXT("afraid") || Lower == TEXT("scared")) + NewEmotion = EElevenLabsEmotion::Fear; + else if (Lower == TEXT("disgust") || Lower == TEXT("disgusted")) + NewEmotion = EElevenLabsEmotion::Disgust; + else if (Lower == TEXT("neutral")) + NewEmotion = EElevenLabsEmotion::Neutral; + else + UE_LOG(LogElevenLabsAgent, Warning, TEXT("Unknown emotion '%s', defaulting to Neutral."), **EmotionStr); + } + + // Parse intensity (default: medium) + EElevenLabsEmotionIntensity NewIntensity = EElevenLabsEmotionIntensity::Medium; + const FString* IntensityStr = ToolCall.Parameters.Find(TEXT("intensity")); + if (IntensityStr) + { + const FString Lower = IntensityStr->ToLower(); + if (Lower == TEXT("low") || Lower == TEXT("subtle") || Lower == TEXT("light")) + NewIntensity = EElevenLabsEmotionIntensity::Low; + else if (Lower == TEXT("medium") || Lower == TEXT("moderate") || Lower == TEXT("normal")) + NewIntensity = EElevenLabsEmotionIntensity::Medium; + else if (Lower == TEXT("high") || Lower == TEXT("strong") || Lower == TEXT("extreme") || Lower == TEXT("intense")) + NewIntensity = EElevenLabsEmotionIntensity::High; + else + UE_LOG(LogElevenLabsAgent, Warning, TEXT("Unknown intensity '%s', defaulting to Medium."), **IntensityStr); + } + + CurrentEmotion = NewEmotion; + CurrentEmotionIntensity = NewIntensity; + const double T = FPlatformTime::Seconds() - SessionStartTime; + UE_LOG(LogElevenLabsAgent, Log, TEXT("[T+%.2fs] Agent emotion changed to: %s (%s)"), + T, *UEnum::GetValueAsString(NewEmotion), *UEnum::GetValueAsString(NewIntensity)); + + OnAgentEmotionChanged.Broadcast(NewEmotion, NewIntensity); + + // Auto-respond to the tool call so the agent can continue. + if (WebSocketProxy) + { + WebSocketProxy->SendClientToolResult( + ToolCall.ToolCallId, + FString::Printf(TEXT("emotion set to %s (%s)"), + *UEnum::GetValueAsString(NewEmotion), + *UEnum::GetValueAsString(NewIntensity)), + false); + } + } + else + { + // Unknown tool — forward to Blueprint for custom handling. + OnAgentClientToolCall.Broadcast(ToolCall); + } +} + // ───────────────────────────────────────────────────────────────────────────── // Audio playback // ───────────────────────────────────────────────────────────────────────────── diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsFacialExpressionComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsFacialExpressionComponent.cpp new file mode 100644 index 0000000..1213771 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsFacialExpressionComponent.cpp @@ -0,0 +1,239 @@ +// Copyright ASTERION. All Rights Reserved. + +#include "ElevenLabsFacialExpressionComponent.h" +#include "ElevenLabsConversationalAgentComponent.h" +#include "ElevenLabsLipSyncPoseMap.h" +#include "Animation/AnimSequence.h" +#include "Components/SkeletalMeshComponent.h" + +DEFINE_LOG_CATEGORY_STATIC(LogElevenLabsFacialExpr, Log, All); + +// ───────────────────────────────────────────────────────────────────────────── +// Construction +// ───────────────────────────────────────────────────────────────────────────── + +UElevenLabsFacialExpressionComponent::UElevenLabsFacialExpressionComponent() +{ + PrimaryComponentTick.bCanEverTick = true; + PrimaryComponentTick.TickGroup = TG_PrePhysics; +} + +// ───────────────────────────────────────────────────────────────────────────── +// BeginPlay / EndPlay +// ───────────────────────────────────────────────────────────────────────────── + +void UElevenLabsFacialExpressionComponent::BeginPlay() +{ + Super::BeginPlay(); + + // Find the agent component on the same actor + AActor* Owner = GetOwner(); + if (!Owner) + { + UE_LOG(LogElevenLabsFacialExpr, Warning, TEXT("No owner actor — facial expressions disabled.")); + return; + } + + auto* Agent = Owner->FindComponentByClass(); + if (Agent) + { + AgentComponent = Agent; + Agent->OnAgentEmotionChanged.AddDynamic( + this, &UElevenLabsFacialExpressionComponent::OnEmotionChanged); + + UE_LOG(LogElevenLabsFacialExpr, Log, + TEXT("Facial expression bound to agent component on %s."), *Owner->GetName()); + } + else + { + UE_LOG(LogElevenLabsFacialExpr, Warning, + TEXT("No ElevenLabsConversationalAgentComponent found on %s — " + "facial expression will not respond to emotion changes."), + *Owner->GetName()); + } + + // Extract emotion curves from PoseMap + InitializeEmotionPoses(); +} + +void UElevenLabsFacialExpressionComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) +{ + if (AgentComponent.IsValid()) + { + AgentComponent->OnAgentEmotionChanged.RemoveDynamic( + this, &UElevenLabsFacialExpressionComponent::OnEmotionChanged); + } + + Super::EndPlay(EndPlayReason); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Emotion pose initialization +// ───────────────────────────────────────────────────────────────────────────── + +void UElevenLabsFacialExpressionComponent::InitializeEmotionPoses() +{ + EmotionCurveMap.Reset(); + + if (!PoseMap || PoseMap->EmotionPoses.Num() == 0) + { + UE_LOG(LogElevenLabsFacialExpr, Log, + TEXT("No emotion poses assigned in PoseMap — facial expressions disabled.")); + return; + } + + int32 EmotionCount = 0; + for (const auto& EmotionPair : PoseMap->EmotionPoses) + { + const EElevenLabsEmotion Emotion = EmotionPair.Key; + const FElevenLabsEmotionPoseSet& PoseSet = EmotionPair.Value; + + auto& IntensityMap = EmotionCurveMap.FindOrAdd(Emotion); + + if (PoseSet.Normal) + { + IntensityMap.Add(EElevenLabsEmotionIntensity::Low, ExtractCurvesFromAnim(PoseSet.Normal)); + ++EmotionCount; + } + if (PoseSet.Medium) + { + IntensityMap.Add(EElevenLabsEmotionIntensity::Medium, ExtractCurvesFromAnim(PoseSet.Medium)); + ++EmotionCount; + } + if (PoseSet.Extreme) + { + IntensityMap.Add(EElevenLabsEmotionIntensity::High, ExtractCurvesFromAnim(PoseSet.Extreme)); + ++EmotionCount; + } + } + + UE_LOG(LogElevenLabsFacialExpr, Log, + TEXT("=== Emotion poses: %d emotions, %d total anim slots loaded ==="), + PoseMap->EmotionPoses.Num(), EmotionCount); +} + +TMap UElevenLabsFacialExpressionComponent::ExtractCurvesFromAnim(UAnimSequence* AnimSeq) +{ + TMap CurveValues; + if (!AnimSeq) return CurveValues; + + const IAnimationDataModel* DataModel = AnimSeq->GetDataModel(); + if (!DataModel) return CurveValues; + + const TArray& FloatCurves = DataModel->GetFloatCurves(); + for (const FFloatCurve& Curve : FloatCurves) + { + const FName CurveName = Curve.GetName(); + const float Value = Curve.FloatCurve.Eval(0.0f); + if (FMath::Abs(Value) < 0.001f) continue; + CurveValues.Add(CurveName, Value); + } + + UE_LOG(LogElevenLabsFacialExpr, Log, + TEXT("Emotion anim '%s': Extracted %d non-zero curves."), + *AnimSeq->GetName(), CurveValues.Num()); + return CurveValues; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Emotion change handler +// ───────────────────────────────────────────────────────────────────────────── + +void UElevenLabsFacialExpressionComponent::OnEmotionChanged( + EElevenLabsEmotion Emotion, EElevenLabsEmotionIntensity Intensity) +{ + if (Emotion == ActiveEmotion && Intensity == ActiveEmotionIntensity) + return; // No change + + ActiveEmotion = Emotion; + ActiveEmotionIntensity = Intensity; + + // Look up target emotion curves + TargetEmotionCurves.Reset(); + const auto* IntensityMap = EmotionCurveMap.Find(Emotion); + if (IntensityMap) + { + const auto* Curves = IntensityMap->Find(Intensity); + if (Curves) + { + TargetEmotionCurves = *Curves; + } + else + { + // Fallback: try Medium, then Low, then High + static const EElevenLabsEmotionIntensity Fallbacks[] = { + EElevenLabsEmotionIntensity::Medium, + EElevenLabsEmotionIntensity::Low, + EElevenLabsEmotionIntensity::High + }; + for (EElevenLabsEmotionIntensity Fb : Fallbacks) + { + Curves = IntensityMap->Find(Fb); + if (Curves) { TargetEmotionCurves = *Curves; break; } + } + } + } + + // Start blending from current to target + EmotionBlendAlpha = 0.0f; + + UE_LOG(LogElevenLabsFacialExpr, Log, + TEXT("Emotion target set: %s (%s) — %d curves, blending over %.1fs..."), + *UEnum::GetValueAsString(Emotion), *UEnum::GetValueAsString(Intensity), + TargetEmotionCurves.Num(), EmotionBlendDuration); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Tick — smooth emotion blending +// ───────────────────────────────────────────────────────────────────────────── + +void UElevenLabsFacialExpressionComponent::TickComponent( + float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) +{ + Super::TickComponent(DeltaTime, TickType, ThisTickFunction); + + if (EmotionCurveMap.Num() == 0) + return; // No emotion data loaded + + // Advance blend alpha + if (EmotionBlendAlpha < 1.0f) + { + const float BlendSpeed = 1.0f / FMath::Max(0.05f, EmotionBlendDuration); + EmotionBlendAlpha = FMath::Min(1.0f, EmotionBlendAlpha + DeltaTime * BlendSpeed); + } + + // Blend CurrentEmotionCurves toward TargetEmotionCurves + { + TSet AllCurves; + for (const auto& P : CurrentEmotionCurves) AllCurves.Add(P.Key); + for (const auto& P : TargetEmotionCurves) AllCurves.Add(P.Key); + + for (const FName& CurveName : AllCurves) + { + const float Current = CurrentEmotionCurves.Contains(CurveName) + ? CurrentEmotionCurves[CurveName] : 0.0f; + const float Target = TargetEmotionCurves.Contains(CurveName) + ? TargetEmotionCurves[CurveName] : 0.0f; + const float Blended = FMath::Lerp(Current, Target, EmotionBlendAlpha); + + if (FMath::Abs(Blended) > 0.001f) + CurrentEmotionCurves.FindOrAdd(CurveName) = Blended; + else + CurrentEmotionCurves.Remove(CurveName); + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Mouth curve classification +// ───────────────────────────────────────────────────────────────────────────── + +bool UElevenLabsFacialExpressionComponent::IsMouthCurve(const FName& CurveName) +{ + const FString Name = CurveName.ToString().ToLower(); + return Name.Contains(TEXT("jaw")) + || Name.Contains(TEXT("mouth")) + || Name.Contains(TEXT("lips")) + || Name.Contains(TEXT("tongue")) + || Name.Contains(TEXT("cheekpuff")); +} diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp index a50f33d..b2cccf1 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsLipSyncComponent.cpp @@ -3,7 +3,7 @@ #include "ElevenLabsLipSyncComponent.h" #include "ElevenLabsLipSyncPoseMap.h" #include "ElevenLabsConversationalAgentComponent.h" -#include "ElevenLabsDefinitions.h" +#include "ElevenLabsFacialExpressionComponent.h" #include "Components/SkeletalMeshComponent.h" #include "Engine/SkeletalMesh.h" #include "Animation/MorphTarget.h" @@ -539,6 +539,7 @@ void UElevenLabsLipSyncComponent::InitializePoseMappings() UE_LOG(LogElevenLabsLipSync, Log, TEXT("No phoneme pose AnimSequences assigned — using hardcoded ARKit mapping.")); } + } void UElevenLabsLipSyncComponent::EndPlay(const EEndPlayReason::Type EndPlayReason) @@ -2274,6 +2275,44 @@ void UElevenLabsLipSyncComponent::MapVisemesToBlendshapes() } } + // ── Merge emotion base layer from FacialExpressionComponent ────────── + // Emotion provides the base expression (eyes, brows, cheeks). + // Lip sync overrides only mouth-area curves. + if (AActor* Owner = GetOwner()) + { + if (auto* FaceExpr = Owner->FindComponentByClass()) + { + const TMap& EmotionCurves = FaceExpr->GetCurrentEmotionCurves(); + if (EmotionCurves.Num() > 0) + { + // Collect which curves lip sync is actively driving (mouth area) + TSet LipSyncMouthCurves; + for (const auto& Pair : CurrentBlendshapes) + { + if (UElevenLabsFacialExpressionComponent::IsMouthCurve(Pair.Key) && Pair.Value > 0.01f) + LipSyncMouthCurves.Add(Pair.Key); + } + + // Add non-mouth emotion curves (eyes, brows, cheeks, nose) + for (const auto& Pair : EmotionCurves) + { + if (!UElevenLabsFacialExpressionComponent::IsMouthCurve(Pair.Key)) + { + // Emotion controls non-mouth curves exclusively + CurrentBlendshapes.FindOrAdd(Pair.Key) = Pair.Value; + } + else if (!LipSyncMouthCurves.Contains(Pair.Key)) + { + // Mouth curves from emotion only if lip sync has nothing active there + // (e.g. during silence, the emotion's mouth pose shows through) + CurrentBlendshapes.FindOrAdd(Pair.Key) = Pair.Value; + } + // Otherwise: lip sync already has a value for this mouth curve — keep it + } + } + } + } + // Clamp all values. Use wider range for pose data (CTRL curves can exceed 1.0). const float MaxClamp = bUsePoseMapping ? 2.0f : 1.0f; for (auto& Pair : CurrentBlendshapes) diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp index e0d32f0..cc8913f 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Private/ElevenLabsWebSocketProxy.cpp @@ -391,6 +391,10 @@ void UElevenLabsWebSocketProxy::OnWsMessage(const FString& Message) // Silently ignore — corrected text after interruption. UE_LOG(LogElevenLabsWS, Verbose, TEXT("agent_response_correction received (ignored).")); } + else if (MsgType == ElevenLabsMessageType::ClientToolCall) + { + HandleClientToolCall(Root); + } else if (MsgType == ElevenLabsMessageType::InterruptionEvent) { HandleInterruption(Root); @@ -658,6 +662,64 @@ void UElevenLabsWebSocketProxy::HandleInterruption(const TSharedPtr OnInterrupted.Broadcast(); } +void UElevenLabsWebSocketProxy::HandleClientToolCall(const TSharedPtr& Root) +{ + // Incoming: { "type": "client_tool_call", "client_tool_call": { + // "tool_name": "set_emotion", "tool_call_id": "abc123", + // "parameters": { "emotion": "surprise" } } } + const TSharedPtr* ToolCallObj = nullptr; + if (!Root->TryGetObjectField(TEXT("client_tool_call"), ToolCallObj) || !ToolCallObj) + { + UE_LOG(LogElevenLabsWS, Warning, TEXT("client_tool_call: missing client_tool_call object.")); + return; + } + + FElevenLabsClientToolCall ToolCall; + (*ToolCallObj)->TryGetStringField(TEXT("tool_name"), ToolCall.ToolName); + (*ToolCallObj)->TryGetStringField(TEXT("tool_call_id"), ToolCall.ToolCallId); + + // Extract parameters as string key-value pairs + const TSharedPtr* ParamsObj = nullptr; + if ((*ToolCallObj)->TryGetObjectField(TEXT("parameters"), ParamsObj) && ParamsObj) + { + for (const auto& Pair : (*ParamsObj)->Values) + { + FString Value; + if (Pair.Value->TryGetString(Value)) + { + ToolCall.Parameters.Add(Pair.Key, Value); + } + else + { + // For non-string values, serialize to string + ToolCall.Parameters.Add(Pair.Key, Pair.Value->AsString()); + } + } + } + + const double T = FPlatformTime::Seconds() - SessionStartTime; + UE_LOG(LogElevenLabsWS, Log, TEXT("[T+%.2fs] Client tool call: %s (id=%s, %d params)"), + T, *ToolCall.ToolName, *ToolCall.ToolCallId, ToolCall.Parameters.Num()); + + OnClientToolCall.Broadcast(ToolCall); +} + +void UElevenLabsWebSocketProxy::SendClientToolResult(const FString& ToolCallId, const FString& Result, bool bIsError) +{ + // Outgoing: { "type": "client_tool_result", "tool_call_id": "abc123", + // "result": "emotion set to surprise", "is_error": false } + TSharedPtr Msg = MakeShareable(new FJsonObject()); + Msg->SetStringField(TEXT("type"), ElevenLabsMessageType::ClientToolResult); + Msg->SetStringField(TEXT("tool_call_id"), ToolCallId); + Msg->SetStringField(TEXT("result"), Result); + Msg->SetBoolField(TEXT("is_error"), bIsError); + SendJsonMessage(Msg); + + const double T = FPlatformTime::Seconds() - SessionStartTime; + UE_LOG(LogElevenLabsWS, Log, TEXT("[T+%.2fs] Sent client_tool_result for %s: %s (error=%s)"), + T, *ToolCallId, *Result, bIsError ? TEXT("true") : TEXT("false")); +} + void UElevenLabsWebSocketProxy::HandlePing(const TSharedPtr& Root) { // Reply with a pong to keep the connection alive. diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsConversationalAgentComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsConversationalAgentComponent.h index a1020ca..a43fe2e 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsConversationalAgentComponent.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsConversationalAgentComponent.h @@ -62,6 +62,23 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentPartialResponse, */ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnAgentResponseTimeout); +/** + * Fired when the agent sets an emotion via the "set_emotion" client tool. + * Use this to drive facial expressions on your character (MetaHuman blendshapes, etc.). + * The emotion changes BEFORE the corresponding audio arrives, giving time to blend. + */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FOnAgentEmotionChanged, + EElevenLabsEmotion, Emotion, + EElevenLabsEmotionIntensity, Intensity); + +/** + * Fired for any client tool call that is NOT automatically handled (i.e. not "set_emotion"). + * Use this to implement custom client tools in Blueprint. + * You MUST call SendClientToolResult on the WebSocketProxy to acknowledge the call. + */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnAgentClientToolCall, + const FElevenLabsClientToolCall&, ToolCall); + // Non-dynamic delegate for raw agent audio (high-frequency, C++ consumers only). // Delivers PCM chunks as int16, 16kHz mono, little-endian. DECLARE_MULTICAST_DELEGATE_OneParam(FOnAgentAudioData, const TArray& /*PCMData*/); @@ -208,6 +225,24 @@ public: meta = (ToolTip = "Fires if the server doesn't respond within ResponseTimeoutSeconds.\nUse to show 'try again' or re-open the mic automatically.")) FOnAgentResponseTimeout OnAgentResponseTimeout; + /** Fired when the agent changes emotion via the "set_emotion" client tool. The emotion is set BEFORE the corresponding audio arrives, giving you time to smoothly blend facial expressions. */ + UPROPERTY(BlueprintAssignable, Category = "ElevenLabs|Events", + meta = (ToolTip = "Fires when the agent sets an emotion (joy, sadness, surprise, fear, anger, disgust).\nDriven by the 'set_emotion' client tool. Arrives before the audio.")) + FOnAgentEmotionChanged OnAgentEmotionChanged; + + /** Fired for client tool calls that are NOT automatically handled (i.e. not "set_emotion"). You must call GetWebSocketProxy()->SendClientToolResult() to respond. */ + UPROPERTY(BlueprintAssignable, Category = "ElevenLabs|Events", + meta = (ToolTip = "Fires for custom client tool calls (not set_emotion).\nYou must respond via GetWebSocketProxy()->SendClientToolResult().")) + FOnAgentClientToolCall OnAgentClientToolCall; + + /** The current emotion of the agent, as set by the "set_emotion" client tool. Defaults to Neutral. */ + UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") + EElevenLabsEmotion CurrentEmotion = EElevenLabsEmotion::Neutral; + + /** The current emotion intensity. Defaults to Medium. */ + UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") + EElevenLabsEmotionIntensity CurrentEmotionIntensity = EElevenLabsEmotionIntensity::Medium; + // ── Raw audio data (C++ only, used by LipSync component) ──────────────── /** Raw PCM audio from the agent (int16, 16kHz mono). Fires for each WebSocket audio chunk. * Used internally by UElevenLabsLipSyncComponent for spectral analysis. */ @@ -312,6 +347,9 @@ private: UFUNCTION() void HandleAgentResponsePart(const FString& PartialText); + UFUNCTION() + void HandleClientToolCall(const FElevenLabsClientToolCall& ToolCall); + // ── Audio playback ──────────────────────────────────────────────────────── void InitAudioPlayback(); void EnqueueAgentAudio(const TArray& PCMData); diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsDefinitions.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsDefinitions.h index 09d7ba1..6cc796b 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsDefinitions.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsDefinitions.h @@ -108,3 +108,50 @@ struct PS_AI_AGENT_ELEVENLABS_API FElevenLabsTranscriptSegment UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") bool bIsFinal = false; }; + +// ───────────────────────────────────────────────────────────────────────────── +// Agent emotion (driven by client tool "set_emotion" from the LLM) +// ───────────────────────────────────────────────────────────────────────────── +UENUM(BlueprintType) +enum class EElevenLabsEmotion : uint8 +{ + Neutral UMETA(DisplayName = "Neutral"), + Joy UMETA(DisplayName = "Joy"), + Sadness UMETA(DisplayName = "Sadness"), + Anger UMETA(DisplayName = "Anger"), + Surprise UMETA(DisplayName = "Surprise"), + Fear UMETA(DisplayName = "Fear"), + Disgust UMETA(DisplayName = "Disgust"), +}; + +// ───────────────────────────────────────────────────────────────────────────── +// Emotion intensity (maps to Normal/Medium/Extreme pose variants) +// ───────────────────────────────────────────────────────────────────────────── +UENUM(BlueprintType) +enum class EElevenLabsEmotionIntensity : uint8 +{ + Low UMETA(DisplayName = "Low (Normal)"), + Medium UMETA(DisplayName = "Medium"), + High UMETA(DisplayName = "High (Extreme)"), +}; + +// ───────────────────────────────────────────────────────────────────────────── +// Client tool call received from ElevenLabs server +// ───────────────────────────────────────────────────────────────────────────── +USTRUCT(BlueprintType) +struct PS_AI_AGENT_ELEVENLABS_API FElevenLabsClientToolCall +{ + GENERATED_BODY() + + /** Name of the tool the agent wants to invoke (e.g. "set_emotion"). */ + UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") + FString ToolName; + + /** Unique ID for this tool invocation — must be echoed back in client_tool_result. */ + UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") + FString ToolCallId; + + /** Raw JSON parameters as key-value string pairs. */ + UPROPERTY(BlueprintReadOnly, Category = "ElevenLabs") + TMap Parameters; +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsFacialExpressionComponent.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsFacialExpressionComponent.h new file mode 100644 index 0000000..f01fa12 --- /dev/null +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsFacialExpressionComponent.h @@ -0,0 +1,116 @@ +// Copyright ASTERION. All Rights Reserved. + +#pragma once + +#include "CoreMinimal.h" +#include "Components/ActorComponent.h" +#include "ElevenLabsDefinitions.h" +#include "ElevenLabsFacialExpressionComponent.generated.h" + +class UElevenLabsConversationalAgentComponent; +class UElevenLabsLipSyncPoseMap; +class USkeletalMeshComponent; + +// ───────────────────────────────────────────────────────────────────────────── +// UElevenLabsFacialExpressionComponent +// +// Drives emotion-based facial expressions on a MetaHuman (or any skeletal mesh) +// as a BASE layer. Lip sync (from ElevenLabsLipSyncComponent) modulates on top, +// overriding only mouth-area curves. +// +// Workflow: +// 1. Assign a PoseMap data asset with Emotion Poses filled in. +// 2. Assign the TargetMesh (same mesh as the LipSync component). +// 3. The component listens to OnAgentEmotionChanged from the agent component. +// 4. Emotion curves are smoothly blended (~500ms transitions). +// 5. The LipSync component reads GetCurrentEmotionCurves() to merge as base layer. +// ───────────────────────────────────────────────────────────────────────────── +UCLASS(ClassGroup = "ElevenLabs", meta = (BlueprintSpawnableComponent), + DisplayName = "ElevenLabs Facial Expression") +class PS_AI_AGENT_ELEVENLABS_API UElevenLabsFacialExpressionComponent : public UActorComponent +{ + GENERATED_BODY() + +public: + UElevenLabsFacialExpressionComponent(); + + // ── Configuration ───────────────────────────────────────────────────────── + + /** Pose map asset containing emotion AnimSequences (Normal / Medium / Extreme per emotion). + * Can be the same PoseMap asset used by the LipSync component. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|FacialExpression", + meta = (ToolTip = "Pose map with Emotion Poses filled in.\nCan be the same asset as the LipSync component.")) + TObjectPtr PoseMap; + + /** Skeletal mesh to apply emotion curves to. + * Should be the same mesh as the LipSync component's TargetMesh. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|FacialExpression", + meta = (ToolTip = "Skeletal mesh for emotion curves.\nShould match the LipSync component's TargetMesh.")) + TObjectPtr TargetMesh; + + /** Emotion transition duration in seconds. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "ElevenLabs|FacialExpression", + meta = (ClampMin = "0.1", ClampMax = "3.0", + ToolTip = "How long (seconds) to blend between emotions.\n0.5 = snappy, 1.5 = smooth.")) + float EmotionBlendDuration = 0.5f; + + // ── Getters ─────────────────────────────────────────────────────────────── + + /** Get the current smoothed emotion curves (for the LipSync component to merge). */ + UFUNCTION(BlueprintCallable, Category = "ElevenLabs|FacialExpression") + const TMap& GetCurrentEmotionCurves() const { return CurrentEmotionCurves; } + + /** Get the active emotion. */ + UFUNCTION(BlueprintPure, Category = "ElevenLabs|FacialExpression") + EElevenLabsEmotion GetActiveEmotion() const { return ActiveEmotion; } + + /** Get the active emotion intensity. */ + UFUNCTION(BlueprintPure, Category = "ElevenLabs|FacialExpression") + EElevenLabsEmotionIntensity GetActiveIntensity() const { return ActiveEmotionIntensity; } + + /** Check if a curve name belongs to the mouth area (overridden by lip sync). */ + UFUNCTION(BlueprintPure, Category = "ElevenLabs|FacialExpression") + static bool IsMouthCurve(const FName& CurveName); + + // ── UActorComponent overrides ───────────────────────────────────────────── + virtual void BeginPlay() override; + virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override; + virtual void TickComponent(float DeltaTime, ELevelTick TickType, + FActorComponentTickFunction* ThisTickFunction) override; + +private: + // ── Event handlers ──────────────────────────────────────────────────────── + + /** Called when the agent changes emotion via client tool. */ + UFUNCTION() + void OnEmotionChanged(EElevenLabsEmotion Emotion, EElevenLabsEmotionIntensity Intensity); + + // ── Curve extraction ────────────────────────────────────────────────────── + + /** Extract curve values at t=0 from an AnimSequence. */ + TMap ExtractCurvesFromAnim(UAnimSequence* AnimSeq); + + /** Initialize emotion curve data from PoseMap at BeginPlay. */ + void InitializeEmotionPoses(); + + // ── State ───────────────────────────────────────────────────────────────── + + /** Extracted curve data: Emotion → Intensity → { CurveName → Value }. */ + TMap>> EmotionCurveMap; + + /** Current smoothed emotion curves (blended each tick). */ + TMap CurrentEmotionCurves; + + /** Target emotion curves (set when emotion changes, blended toward). */ + TMap TargetEmotionCurves; + + /** Current blend progress (0 = old emotion, 1 = target emotion). */ + float EmotionBlendAlpha = 1.0f; + + /** Active emotion (for change detection). */ + EElevenLabsEmotion ActiveEmotion = EElevenLabsEmotion::Neutral; + EElevenLabsEmotionIntensity ActiveEmotionIntensity = EElevenLabsEmotionIntensity::Medium; + + /** Cached reference to the agent component on the same Actor. */ + TWeakObjectPtr AgentComponent; +}; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncPoseMap.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncPoseMap.h index 6bacbd1..34fecb6 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncPoseMap.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsLipSyncPoseMap.h @@ -5,10 +5,35 @@ #include "CoreMinimal.h" #include "Engine/DataAsset.h" #include "Engine/AssetManager.h" +#include "ElevenLabsDefinitions.h" #include "ElevenLabsLipSyncPoseMap.generated.h" class UAnimSequence; +// ───────────────────────────────────────────────────────────────────────────── +// Emotion pose set: 3 intensity levels (Normal / Medium / Extreme) +// ───────────────────────────────────────────────────────────────────────────── +USTRUCT(BlueprintType) +struct PS_AI_AGENT_ELEVENLABS_API FElevenLabsEmotionPoseSet +{ + GENERATED_BODY() + + /** Low intensity expression (subtle). E.g. MHF_Happy_N */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, + meta = (ToolTip = "Low intensity (Normal). E.g. MHF_Happy_N")) + TObjectPtr Normal; + + /** Medium intensity expression. E.g. MHF_Happy_M */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, + meta = (ToolTip = "Medium intensity. E.g. MHF_Happy_M")) + TObjectPtr Medium; + + /** High intensity expression (extreme). E.g. MHF_Happy_E */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, + meta = (ToolTip = "High intensity (Extreme). E.g. MHF_Happy_E")) + TObjectPtr Extreme; +}; + /** * Reusable data asset that maps OVR visemes to phoneme pose AnimSequences. * @@ -103,4 +128,17 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Phoneme Poses", meta = (ToolTip = "Close back vowel (OO). E.g. MHF_OU")) TObjectPtr PoseOU; + + // ── Emotion Poses ──────────────────────────────────────────────────────── + // + // Facial expression animations for each emotion, with 3 intensity levels. + // These are applied as a BASE layer (eyes, eyebrows, cheeks). + // Lip sync MODULATES on top, overriding only mouth-area curves. + + /** Map of emotions to their pose sets (Normal / Medium / Extreme). + * Add entries for each emotion your agent uses (Joy, Sadness, Anger, Surprise, Fear, Disgust). + * Neutral is optional — absence means no base expression. */ + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Emotion Poses", + meta = (ToolTip = "Emotion → AnimSequence mapping with 3 intensity levels.\nThese drive the base facial expression (eyes, brows, cheeks).\nLip sync overrides the mouth area on top.")) + TMap EmotionPoses; }; diff --git a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h index c1a3cb7..e183c5d 100644 --- a/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h +++ b/Unreal/PS_AI_Agent/Plugins/PS_AI_Agent_ElevenLabs/Source/PS_AI_Agent_ElevenLabs/Public/ElevenLabsWebSocketProxy.h @@ -48,6 +48,10 @@ DECLARE_DYNAMIC_MULTICAST_DELEGATE(FOnElevenLabsAgentResponseStarted); DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnElevenLabsAgentResponsePart, const FString&, PartialText); +/** Fired when the server sends a client_tool_call — the agent wants the client to execute a tool. */ +DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FOnElevenLabsClientToolCall, + const FElevenLabsClientToolCall&, ToolCall); + // ───────────────────────────────────────────────────────────────────────────── // WebSocket Proxy @@ -103,6 +107,10 @@ public: UPROPERTY(BlueprintAssignable, Category = "ElevenLabs|Events") FOnElevenLabsAgentResponsePart OnAgentResponsePart; + /** Fired when the agent invokes a client tool. Handle the call and reply with SendClientToolResult. */ + UPROPERTY(BlueprintAssignable, Category = "ElevenLabs|Events") + FOnElevenLabsClientToolCall OnClientToolCall; + // ── Lifecycle ───────────────────────────────────────────────────────────── /** @@ -172,6 +180,17 @@ public: UFUNCTION(BlueprintCallable, Category = "ElevenLabs") void SendInterrupt(); + /** + * Send the result of a client tool call back to ElevenLabs. + * Must be called after receiving a OnClientToolCall event. + * + * @param ToolCallId The tool_call_id from the original client_tool_call. + * @param Result A string result to return to the agent. + * @param bIsError True if the tool execution failed. + */ + UFUNCTION(BlueprintCallable, Category = "ElevenLabs") + void SendClientToolResult(const FString& ToolCallId, const FString& Result, bool bIsError = false); + // ── Info ────────────────────────────────────────────────────────────────── UFUNCTION(BlueprintPure, Category = "ElevenLabs") @@ -193,6 +212,7 @@ private: void HandleAgentResponse(const TSharedPtr& Payload); void HandleAgentChatResponsePart(const TSharedPtr& Payload); void HandleInterruption(const TSharedPtr& Payload); + void HandleClientToolCall(const TSharedPtr& Payload); void HandlePing(const TSharedPtr& Payload); /** Build and send a JSON text frame to the server. */