From 4d30672adb5add7d2e9aafabd72a29252aaec231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Fri, 27 Feb 2026 15:22:15 +0000 Subject: [PATCH] browser relay: multi-session override, stale session recovery, better error feedback - relay accepts new extension connections by closing old one (last wins, code 4000) - extension recognizes code 4000 and stops auto-reconnect (shows "replaced" badge) - fix ping interval race where old WS close handler killed new connection's pings - retry CDP commands on "Session with given id not found" by re-attaching debugger - describeError() maps known errors to user-friendly badge tooltips - persisted relay tokens restored on server start - user-scoped token salts, token regeneration/deletion endpoints - extension download endpoint, integrations UI for browser relay setup - browser relay env vars passed to pi-bridge sandboxes - browser screen layout with chat panel and prompt prefix Co-Authored-By: Claude Opus 4.6 --- public/browser-relay-extension.zip | Bin 0 -> 13858 bytes seed/tools/browser/TOOL.md | 43 ++++ seed/tools/browser/index.ts | 222 ++++++++++++++++++ .../Dashboard/Browser/BrowserScreen.tsx | 3 + .../Screens/Dashboard/Browser/TabList.tsx | 65 +---- .../Dashboard/Browser/defaultLayout.ts | 13 +- .../IntegrationsSettings/BrowserRelay.tsx | 198 ++++++++++++++++ .../Settings/IntegrationsSettings/index.tsx | 10 +- src/databases/officer_db/src/index.ts | 1 + .../officer_db/src/queries/integrations.ts | 4 + src/extensions/browser-relay/background.js | 64 ++++- src/server.tsx | 2 +- src/servers/api/browser/relay-auth.ts | 19 +- src/servers/api/browser/relay.ts | 25 +- src/servers/api/browser/router.ts | 93 ++++++-- src/servers/api/pi/pi-bridge.ts | 23 ++ 16 files changed, 696 insertions(+), 89 deletions(-) create mode 100644 public/browser-relay-extension.zip create mode 100644 seed/tools/browser/TOOL.md create mode 100644 seed/tools/browser/index.ts create mode 100644 src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/BrowserRelay.tsx diff --git a/public/browser-relay-extension.zip b/public/browser-relay-extension.zip new file mode 100644 index 0000000000000000000000000000000000000000..ae3af8e36f00014a310f67721e866f4f7b54c122 GIT binary patch literal 13858 zcmbW81ymkOw(oJb5Fog_ySux)1b26LcZc8(A-LOzJLJP9xVyUrc$}Fxcg~r+?%a8= zdv&#})xXtUyZ%*Oz4um>0f&GC`D2L@(b4_e!T&9wKyX1UjqUB68Prr^LBPw@3M~KF zTs`1HKp_snK|uaiDE{UCE*G~nSWpmPf%S(f~9rAtawNR>0z*Z|X_CxYH!aPWm7 zFU)v@zvxhCOVjb{P2m2FWyjwix-yK>j<*L!1^lydU&+%-=Bq<2!;Nbo5*B%i=ILKF?1fZ0}M@9yE zdU_1_Jn0Ga>Kl$G;;tI`vMOXq{Vq-`SugU!2J2UxHfD8iq;P^A4Ce$C)i&7}73U&7uATMaI* zM-dQWj&tebh_GTuVNoliKZ|;0VrzM&nb>;V9@YrBzXXEg+#Gmgz{1z0I?ix-7@eRf z7k;OI72Vl*c|cOh!(F;c+=^3>T@oQT=V@-{JB73hqEX=Q;5)Go)ba>-`sQe!@2s9U zM?r|rRK@fTYb?KE{G>r7J$;MbfY;+oa4~JrqQ2?JD1n;-GlAuPeT9 zQ^3J!2A&%z-A~R&I`viFeYP)TE!v*TRvrPRw;?5rH(>J}cypSBJv|AYWrv%PIx1oY zgv=i8&ux{?(iUJKe>Iyj>1+DRhsn&qKtMkIRSC@hA9h3ho86FC$a0~SZKT%E=@H(A z9D3LHlCdzq74zxgics$ZI;~g8wnwHMlxAu#^RHs6Q&~zhaCsM4i!!O6eEM)ESgN7S zK!5HJ74{yWFYs0xEaOc#R~_p#y)BoljOWpPO}OLnTckzEcD4dT+wwL$n`r>D-|K;hxePbiD zOSKuvFq}&=YNM0U_rCOYV}9eHgP`nWGMVPhv1y~r>kSm2boN_-*E!myws_N`nt#>P zsDVDgOPBfDxbxy(2f^a_(#GR;-Rs(`1y;xNMr0_P&hO7tXCo`U`ifT%vvG@geHV2X zk0bY4LQgGEFC0G!t}_%>*Hd4-HwWJv!u46ib$zdNW)iRr`81WD%5F|EN^TyJL-LO+ zVgwwhZC}kFSR}1_QM;P<{BM4qm>SP8W#4kW0L40j7oDv4kc~a=qK_(&y>$kj-u3m{ z_NVj4H{$&OPg!J$Vu(|;4exA-pY3xG(pz6P)(;=}k&Myc1CHry%+o1j1Zv^ewDLV% zp8B;dHBL|ci%*#%PL7Tb>L;iQVE?6+aJ{jwaecHB{{JR!7UutVal`$skwjPg=mPnx zxCIj@|IqfQsz0a9~GAzUaI z7#P>UOa>*8jZL1t(S5)F1Wri!d>A4UKiCJ0${vLj-+dL zXv#@KL3AY~Xjm502P7Xf=z;_W^=^aF8ZvpmS?|LM7?`CQ8%4S?IgHGpjBN9IY=R0H zhn|nq=1D=GEd6I`oB7 z&=5#>m4}u47ZaG`XeMTSjAj`h0}tGXQ0*ODEI$T4dMoFDH&`{P+1jtLqr9E#JA%VW zg5h!TZ|e-v{!8Z{3vx?d5)Ol`E>j9*&W;P38!JD43Fx2D`Y zJ<#X|tnzq0T#ezAnKS6`mEF3FZ+3{METopyYWfjZQ$5);1_NPwb|3@aY%p5BUz4d| zGlWRh#B6V`uk^LpCy*{-PRO%>UGYfz!`$n3?*-w=ilqe#&r(hsWB-Kwt=&_8N3zNG z+r{r~=PAMS7kZJcXqT3+LZ>`+l&^YX;Ki!-}WN(a6!N!2?v!gvZ3ske`NWCKk$}tkJ%hrg&=1>CA zs94fR#<)7Fn554!V{8K-=~KEXlyAgjn6wIJc3(;d6Ou6CD}8v-QBwAFp-uM|NNsBf zz(KkO*{QQJEj?jq<<6NA^t4-U?dP~dE^X8n zpCM}{hJd}YlEy(o;e5#Lp=r)mc0bif5X$sp3!gWwlb-WM4LS#?Jbc@y6Jq*d$-Ezq zs+nU)o`TQ5u9h~d5#+K{EyWO75kR*{J#1!s>Y+i#eiB(FNJdy?fbj{*0REnsOC?&N zk>|YWM8W7ttA!0u%5XxW3T)_XL{73l_%4Q5|Ngcqft{aLDg6SZmJ)lpYv|Y|t1%t5 zTIo8{?=o=9*QRP8B9;ML+bIot^9tMeXP{mROWeB};_6T;Q?X`%|2k@w}uL8IgX%UEKp#!~%u z%wFk>(bBiFp{7ZRn%YStALZvkPYg6==KSb1Yy9q+$3KcMJ*c zFfZ0M!HP{wi*5DiS;g=2IkPD1a;>jf&Dg6ue&3tHXWo;m!}8peG07s;=555XF4{`V zu+%bB*jkG4zpD;fYf62EWHTdaxlFdaCHU(ooEpq(Z~oD@;(iDm@k89UhIW=_rp_)O z>bAH0clr)hmb2~?MCp8?Nz2xzoc%%AO@v@7jQKFY@WC+ubZyJv2!s zEQ905ZNC#&Jcf*Ju^#N;hE9_rR$kraUNm5z&L9b(k&0%pRKdm*eg%%UK~cA7PnuDz z#m%KAkCjCtm87yw(k6a-`BD7#^k=++)r z6){oTa%kffJCWXU=;Ik{Hp_zv|4HHzQ{$+od1l(Paq7!gwUG8wF|BRSElQ`YJq&(N zp~#z|YiuJu$CaTtJT!YB8f*Od$GD%dLQNM(d2Py`wPT&PdXQhTmNWBZX26en?UcUz zn4dzmXzIBa3tTaEUlbZwQ3VGT3xuS_$ln&GDPHCKoR?_47$Q^j z;HB{>CAy;D70E?xW8Yl~a6Vic6!`2fm=5hZQ$u2}p6<2$sGew13Z5gAQ&a}yjRPVT zSET<}Ye~qRd|UYVUW$MQ0YU#~S4`(-Xk%$&_(!+=zxK)(*4u(8ueseuv@@a>$vmwJ zL1@&WJcoKRG>bLWqXWft9fs~zQFs2`O9YYHIMokZf&*(yzAyVg0F%^aCe=?tht`V3 z_%Ak)i^`Q~(ydz7yFZGj!0CTM73K+$JLr@)Mg|i~7|6oEPTKT7aUIIV5<_?3#)_jv zEwPkxv(`B3x;OcA3;%e;%?Ai4K zCI#y>AlS{x;sp$hl>~C^ymv51Br&3oXBlTNl1KS&TfDj4X8DYyjjV3ue9I~32pNMU z=w$d?Vvxb@?NP#l!v=*&>B||?v3^jRxOP&in0+IijKVg%%q`RFDZlo6*ZeSny5Tb^ zWFGi+B?j4|NIrgJYCs2XnRo)@jd*giu52*9G$idfB@Vg*-gaFs!MBlOyTg{#99hIi zm`rdD$DPv(8s*_!wtS!zv(n@{`?gF~?#M5t!y|HUUwO3Sh3_IIDmj%fA|r!I+BcQm zcd~BgCdMU+(=lT>d#D6}#&qnGY<1v+e|14c##z?xd}`6%i`3EExuFssw#P97ZDI*z z7TwfxDxIs4oDFx{dYK*5PI{@mzjfHW-LH*I$0nH=Lw>d^y9L$?IoiTg1&Lg1l1a-8 zR_>EEbZbW%b;{veWK_1VHwVZz`<15W*+X+-*3IXv>jD_V%i;XG76x6ya*Yk-Lc<~# z(T0M^a5Gf?2)pt#8Nc##o_1;NEp^1(@_=#4mYme7HG@v~6U2_!1$SxgT~zf>N}o)R+WmnJR2Bw(yeIjKs28uZ#xX?K8w1^iY;erX>sPT=@T$AD3A zeYoYYbMr9xX%z<-BENqbM;imhQ9&u0jp8s~f2$Ls$|nF^PNzf=zciqr&R`3EHqIk!~yx8c)%0c|HGmSH2!RewV%^PhuZnd0dv1g}uO)oJtvwAkT zvnO=tPNLz8gcT>a8Cl}0wNr&%HK|1gQXX||0j_f=vfdd=;JAuL&Xj88Q&sCgK83KQ zJgpYyHOLJWNFBspuOjjCAR9g5?;Ybc0|uGVmRe3847^<*Al+zFVsa#~3^bj`S~;Y~ zNGp(^#Jvu3Jw2p>2J%<%g>>%hF1D)tWUV(ZDf&$!mKG`WK@Fr_rNusG{X6Wz$xr2( zZX%5bC=ZjiU!`ZENli`8#6oe%ZUfj-ewZO7^?XrAT2(HVM^b13FdzmN&Q)9Xr;A$c zS){REMJxC{lr^_P^LJ_3Pn$HF$pg@V^J!EG#S#%`jv6S&hU2xpCMgSdXSC7?IT-@P zHbUO!(3`b}Ye&)F9PcKEe2?0x3=O->a)7h3&yo4omSXX>OB_gCw~QOaTtFL~`Qfgy z`cKN;dKtA9QtK2hsW%?8HSNhS6WFU#G48|cD(b3k0t><-kkjb%gQ}C;Y$JdKrkbM_ z`31HR4>Gl~PeemlM@{U~p1P^`%`9$gvJ9nbEIHYop{&f)yP^~a^|*OPM;7=dfD}It z6IUu6g}%>Qh^krs#|=sz*DRwPAvw=TnRQv^V<+H)+k=UH6>yBh@+2oVk(~jR<8Rry z}9GkM1>GzDR(!vd48F;0Gk4RHLVvH@M-~iFf2xX>9MZeP}r-*y=rmk zpPy&Ws4A0zX)M*cf>4KehFsy$=A2;%))OGKt*pbC-}W5nxf1kMs}tip%;&992W*b_ ztEX>0{2@vPW~4;)&wFT8(n_XnEy`354CYlxt@PeyYX{!K>P@crHhkWnxEFZ|*0xq) z)y>f*La!{#eg#FK=<3o%7k_?q53PK|4r$cQQVcIOy_4r}u4TKz8 z#WhNUcD0dq9l1ooKzqTGL+!)JMMJ+zW5^STn^{qMWF!D`TRy7zTr^U{Y zlR2f+KVV%a%=kpgHL0P=DuwyRnB)O@2b0czoS-pjlsOOlL$Sec zGhZ4Yj_?(nW7|+j(-8H)TLh&f7^w#0S!k1JXvjn4;BBnbmuurQPp}JGrt$3xM&m9a91xL+t-4hm!d!X{ET9S`(s=fiXOy13v*etG$Zu5 z?ozdEn@)h+FB6$9uzwL1F3P)FWWV$wUkX@LV6#FviL1eq8;p#B54#X(B@xsig#vYE zYT7+mJ+x1+Y91Q82pn9$=A-qG1-(9$hjh5`^#RXde)*r-yZPM;3HIOb@A-QEyo%tL7rhQjxM;HRYXMNOM4i>qlSjSuZ8DstT*t#ex-_+v9(Zs%qNaN z{!4%U7=VooKU#^Cy{nxGovVwb&Hwi2z*lYKk3R=~5BiRuuLVt!_Dn38S}&tm$ES>( zp?9J0713bG@@&~rsS*yj!%p6PB_fkj-Ij1hdHf#M9&*0Z2D^{#fK>)|TeE7EEcfx4 zx8URH3Vgz@;c^KsSq}W>-x}QQ(i{FQt{!1Qv?9B<)0Ad=#JA|DLm7myUKp*UAn@`?<%bUJ#9BZmtyu+rUqkjn@N&d1pII7p28t)3A zy=`g@9?VX)8f@U$O@R%BL)%GWqBzP5oEt3CqxZ(N6-B#6qu;!B(WK#CS#^`dF@HW0 z5HNOB-Dkb0f1D}EW7u;q<&<9n%Xy0euC6ahuo{)Xv^cVaIhbR9WpG4{OoJ6u@; zAW#1*nXx8K_x{@1%XmYyMoltfH<4|*bo|-1%J$tDX66`pLxg{fdCIP+LRI!ECax#l>a9y;_l6UU7<0t+(yrqRhdZyhGrf@~6Nsy;2UMAGlPpr%U)K9xcw|z z&~D0{21-U}hS)wYP`Zjv#a4(5GAIJmpTGhZDur1P7bmn9rALS-njT`HV>#<0*aKd9 zjuIn2^F{;_QM(0Hs?emA(5X`C3B+0677{sd`px+YUmPL_%Lc>Z|GKJ0WVjmTA8|g^ z9rv=b&$4bv_O*U|1%}g%2U|}iU`}!KBIF$MO}#;lo~eFiPJZ-hnaOt+*`m44eKEbq zHL&HSQVBUIAKm{F9B-P=diEQ(t%@#(fJw~iEz$<#aM0g!5*!UN8k==Py3B%?q@{Oi zw)?N5!>k}_iv9uu@?!k=iTpoB_rFXD_w-JjR)&(^Cd&e>^*~Wn63}X#x9TdJOt}{9 ze5hp1Z~FJ@1&t_)!9ejr(5h0kniLw4n-%KU_}mRZl$#8n?&%?9UbeH)nDc< z+#iUm%*P#BsrEa#SQ+RU+~5h>L5+fixb@LY&(=J4Of$(uj~z4(S4SDY@9mB^QW}|qnY1*hp*l5uLAo4 zU*i3^*1gH07}0{KA99y&mKa_nUL@w6`7lrY-a8-c@9u&;Il3FJEScI&?hbr1Osy|o z)`W(vP{>Dd)579`p!PzgmV@~E=$6+5OotISbnQ}LKb{s_%_<(BD_c#t#^yLIHC$m& z&d8tRUX4e8Jd-AxluIC;@R_1p;@RXk0W_|*9lNsQu_$2@!bc^#(kkDz-g;3J)CVK8 zNyAAdMyD!vu7l?HlUu8IMo;B3YOfszq-yd|T6D>dJ!FlHX^_d0`^Y(xkO-DBpH8(I zc9GD?Ss1?vZB=qc4`XD=x2054*VC}WRf{|N;oReai3%Pwpc8r2F>*n!6|9VDF~#>53Y z$Z)N{qx{$+;~WDQB30&7WwCp<*0yPk7>~-?b#;*T#@ig3%lj1B%8#d~kvSd-!XEb?KeNCKH4wD#IdYd@_8oO;_bPQE|{h9UYmIwW96j_Bv6 zJti%N863EGl$!VyH?lBrOSUnCzAfC+i9x@Q?Gxitb!flSCD2z3dZ)=w)DOK3y&`XL z6cWA%Q@-aH{VXrjQ;w`s4OJIlxEXmWXrF>BK2nx+G&>L(!H!obMgoNJ)0X`@_bI_| z+0bENw%ZVO7*}<(??ePx;-%a6op#V>vDPp{DAWZ4ModzBQ2oWHu>?Ef!lpHRRkO;|- zzHRzGU3TX;3D<)RPr|7AJlNnG%VdOY5QW1 zb&hjvj84QiT$*m`R&0XgV@Z zlf6v>0{&$6`)8;=_JzHqBQx9gG&$-fNz7vIpPks&IF)f$6v84(tJ`_eSY#4!zqN5#5eXzCU&hLLxQ}2#mP3}YxfUEsG5vjr>QJKpBInC+pHz3G8#ERh7K-;yh>CFGyIKfhB{m0 zJ}ETV$yH(gF$sOYemrunteA=Boe*E2WkTUA5g=xbG17t-5FEsjPEF%R3n;`ag>P;t zM9Vu)*vuAzN2jSF#5Sr0W&8G%a%P~(ymklj!Ctg43~h0%?G`7ERTQ+Kk%AEPs=6KJ ztJW=uPA|(uT^HW{^4VE^~JW!sz@YgMPRTG;R=)9&|b(sWVEoV z9!3&@kp&Mh?rI5SiG!S5)xecAaa1E>NvFLu0t=VA)L&q^5CT3VsM`HP@WV)%P(9O- zFH6DDNs5U?*^ZyG^RCqh1xvk*7v$MP;EYu?jZ@v;!{BiHz`%?sbtfxs92gyFdm~WK zbiZ5oR`V8;T;WLSA2<4=cGP{*a*%YsG(3?}G#g+Fp8<%s7dy0fX)D>U_R)(T=yCDM>uVJB}rJk8P+A71Ei0DnkBh1WkkK4$|Vf|-1<5tZbhPC(hl zwd^v{^Sr?r+}(e!Yw$WP$HTE~h9r*|F`kc>M*G3;f|1~sUae@57OO7+)q`(-w2P|j z7$|#6j`04Oy(plpCEVJ}I=QvJ_^A=Zmj#Jr`!_Z+5n%KF7l`cv78MLJ;_sHv--M({ z8_8?bpbbE|xMFUL0+Sqav(X#9L@a(hf(=3|B;)e=d&igt6%a+Y#vII#fRW5G_L|bt zm2V=`di06)oP=>zef7fgP{$G*>LWdmL_(fV%dCB6hKA2V)!UQ+bjNf5FgA?})p`o(RWKc-O(=!@LY`GnS1 zpP%ZDWD(l1L<{t_i`GoWIV!l^uZWT*d3>1Vs3Gx-$dG3zK*%8+xmj9J+)CUrGAG5?+=7bq}aCpq6&mZDEJ%%&b7RPZs`8K&!<|R?- zbMOXqHXsxlFnLFbN+u`a zzyWA2HEfEYKa4z4JMRi+FsNlNEQkPjD?Zrq1-Z`>ow@yHp1 zFynUM*iwgS01}2?#kAGE`oKheX2F9;F0}C}?GDFVr%{!becsbsRno~q-Se* z=);W2xFP^K*3nJ>{iQOga0GFu*5+763rsKbnl<1}1q6B-;jM8aG!VmYtTC#Prfqh8ihaZYTY6;rBt12E@#j(utu_Q#E z!~O6OVe>uJltk}YVC(tG>@=ohQ{9HPNXDuvinTg=8EZjIRJ1!i2 zL~j(ci4PakCyyB5(;|Ds%&^88z6F{8%sE7^kyH-*h3HyGZSe{(FDvweMG766LTlK@sH_H~n`({2+ z-P*R0r=2Xax!F0J{r3?XlBv67RjUm#&bP!;4?+lF_@}L=a4QlrwK+GstMj}Tn5N)rXI=N=(nJpBX zOx0DGYp&tzI*gpF$V0;;ROEg^9l0{8Y_c4!Zm#!5y&*!comTB5=%KVsL9a<2LIf+v zKi^=Md_nh%bjZy5eqyFHz~ipEE=rm)27qhgF=l`WG$B#~`9|zw1x7xVT~$y#?~S(1 zr@i{tS!Lf;=g6W@{7u-Hk3z~IUy$2WdIq)Xpa&{UQG`|$#y!|w&HVg$8Nz>}rYBJ8 z=ML*bB+-sC&Auscyy{%*yN)-iSLJ?ww@)C`H3IT$vGprhg%sG!*kX>Mm{gcKFsc+l zA5jdI8{Hr-+?NPxpacdnSSYUV+0fh^4o;?dOHFAt1*z+p6o|_*zFn|v@Q=e@!WZT&{V@O*#Og*H@ZJ#YzZ``L-aPkF?2hgbw3>NP-mGgb&J2BIFDxXLG3L2R2W{$!TPi+U!gS`O+5ey*S)Akaie7d&+z(Z~y+h!N- z8K)Dq&?$P(rM=i*X$U60j!=n}znl(grQ2(vF@;}v%&a#e5@_uAC3x}>Oo2nV%y#t# zqqZEL)zzrjU0kxIM`96i%A^Xd8s6R|C}x>wtrcz1voqR0d8_tku)AwXx&*2pfdZvs z|4ODK$Y_ABVf;ibL)(U1w!^cL@i&J-;~VZ8U;Qb>CVR+c_kuk+;MTSwEEpQn$Qq$B z*qNN6Lxz!+)c6Z!`EzsWH$u+`Tj3EwuD5`6ui>9EQEK>FPjv?g&5}{B9L#_^C&6^A~p;*qHnG6(cF2a9$g%0H9cfreMM4Gr~%se`|S z{;6~u^yiIk7O zyOm>uq(8&~8JE49HeCy7WGlrJbPCKoAd&sGZcY-Q?51T?P438wDJMd_K(KmKBZYr- z=z~c(n)J5_OObYxR&feNNBDM0y_jZyP4g%aIF(i9Q9SOrqFCTTldc$>E%gzd;qOHV ziF8>LuS~ydESTl4g#>u;hM~s~X=Up_Lot2LXl~%)7btd@`OO{r^XmIynw{=3 z4%nux#UbRA7D8G!+Qe>_#3oLQnZLq$Y~&MW*{k}dDkA_iVXROrE+&TCFZl6Fbdbtd zep|*T#eTG#6SamLrY{E{<~3easpRs>02~eDA$VT?ZUy7aWpsZK9fD10+!EpSb`jj2 zAosU@0&BT?`VW6sLjm-LHMMmo0LGISym29~PJC0oF&tGIADiVkV_jk9*nm@Ih6l^9 z7ve}US6T@Jc0M}tap`_*+uzz3Cf=Y)rP|=itnl6RLHp7Ok6$99isU4Sh*66XsRZLMG+H9{PJ3J!f>5%=+O&BQ6oH@ z({wMBUu2gQ-=j>q46B&fmnDgJ_(;kJ;+Q)K^5#TnX@}^i^~l!-qLkLFOjNYI;4S*8 z6^jRfr!1^f5BxRTJpjAWkdeENM@0mSgXh!)cks~9I2L-N7p;M`jY6-$BW|H9^5(sP z9o>@Hg@FTHmgFCY5P%kSp{FU>a|ATAU_p|}L|8$C(c!B*2?tBE&JF`N$|d4xiYdd? zg@$<8jvvv)6{;2m+YRuI9DuY{L%>Z)&SQDzy8Tt<$_|ZK6RF?IQRvHXIb%ABL1|^h zhM{d>t19>M+MP%NuJ7yZo(i_3MxYJY^zDM9DJ8RyeaZ0R7pQ!sH4p#vqB0CMvN1pA zRhe~JC%eN|3&KZv-My6@;Ocgz;h8pRJG%(0f8^(|zNE5=`B+!XGJa2=)UPP04J2%n z?_k5xGuM*zg z1jVRD(9w5UX8tPvycotL07b1y)&5{;W6FW&~Rr!r@mh+k-elF?7C0d zIBp{8%l6H3&l{U26O3A*3^?X&BezZ5Mc`?2B}3zjn!Xf=wtoHbJMr7InEEVXaLHD={Km8v$Gnr1wk6>L zhp%Q6)LMf}W;_nm^=v*<;_BF$%coz*izJiDaQPV-U(6u2I2dBs+{9*+jvo+D`_hWq zw@)ftvuJ-IoXDvo&8NfxRGXy0(QuV_P4JrGM3o?xwOubfKEo%18-(3M03i5>i5|1Ws9eIvR9D|-!cj+@Mn#UV z<82FV-v#)kr z%Zhun^asa{;r_&K2T65e*?~se?pRZqv&3AZ`a($RJ1eE$(tT{PctbfY_=d*VC2hmfuJ^MqhXMy~}vMu;-m7 zh)Sy6UN{}Dv6BG!Ygc*0IsdcRUdCXrGu>A_0Tlb3YjH*KZ`DZzl*P%|@3V0}hI!tf zt*#}a1|sr=uk3kH9d|;T^y4RcmSEdk9&1T9T(=IS)w#NOBKBy=O>eD=0q5X)BuqGB z`1-fYLRgNpiyG>omp}sacegG-vXph$nZ_d4Jd<$!%XcC}T_tC|ynEbL%uA^#dS_-CXa!hb#DzXlQhXXw8MTm2bbg#2IV{~LPspArA+ zJ^nM1^uxXNPq)^ee98YA_^%$aKZ7%{{|x?{zwCcPLjI9!1p@!E7JM{CYvw=B{tv@4 BG$sH5 literal 0 HcmV?d00001 diff --git a/seed/tools/browser/TOOL.md b/seed/tools/browser/TOOL.md new file mode 100644 index 00000000..0d2c65e6 --- /dev/null +++ b/seed/tools/browser/TOOL.md @@ -0,0 +1,43 @@ +--- +name: browser +label: Browser Control +description: Control connected Chrome browser tabs via the Officer Browser Relay. Use this tool to list tabs, take screenshots, navigate to URLs, evaluate JavaScript, get page info, activate (focus) tabs, or close tabs. Requires the user to have connected the Browser Relay extension in Settings → Integrations. +language: typescript +inputs: + action: + type: string + description: "Action to perform: list_tabs, screenshot, navigate, evaluate, page_info, activate, close" + tab_id: + type: string + description: Target tab ID. Optional — defaults to the first connected tab. + optional: true + url: + type: string + description: URL to navigate to (required for navigate action) + optional: true + expression: + type: string + description: JavaScript expression to evaluate in the tab (required for evaluate action) + optional: true +--- + +# Browser Tool + +Control the user's Chrome browser tabs through the Officer Browser Relay and Chrome DevTools Protocol. + +## Available Actions + +- **list_tabs**: List all connected tabs with their title, URL, and ID. +- **screenshot**: Capture a screenshot of a tab. Returns the image directly. Use this when asked about page content. +- **navigate**: Navigate a tab to a URL. Requires `url`. +- **evaluate**: Run JavaScript in a tab and return the result. Requires `expression`. +- **page_info**: Get the title and URL of a tab. +- **activate**: Bring a tab to the foreground (focus it). +- **close**: Close a tab. + +## Tips + +- When asked about what's on a page, take a screenshot first. +- Use `evaluate` for extracting structured data from pages (DOM queries, reading text content, etc.). +- If no `tab_id` is provided, the first connected tab is used. +- Tab IDs can be obtained from `list_tabs`. diff --git a/seed/tools/browser/index.ts b/seed/tools/browser/index.ts new file mode 100644 index 00000000..55e9a33a --- /dev/null +++ b/seed/tools/browser/index.ts @@ -0,0 +1,222 @@ +type ToolResult = { + content: Array<{ type: string; text?: string; source?: { type: string; media_type: string; data: string } }>; + isError?: boolean; +}; + +type Params = { + action: string; + tab_id?: string; + url?: string; + expression?: string; +}; + +type TabInfo = { + id: string; + title: string; + url: string; +}; + +const RELAY_AUTH_HEADER = 'x-officer-relay-token'; + +function getConfig(): { port: number; token: string } | null { + const port = process.env.OFFICER_BROWSER_RELAY_PORT; + const token = process.env.OFFICER_BROWSER_RELAY_TOKEN; + if (!port || !token) return null; + return { port: Number(port), token }; +} + +async function listTabs(port: number, token: string): Promise { + const res = await fetch(`http://127.0.0.1:${port}/json/list`, { + headers: { [RELAY_AUTH_HEADER]: token }, + }); + if (!res.ok) throw new Error(`Failed to list tabs (${res.status})`); + const tabs = (await res.json()) as Array<{ id: string; title: string; url: string }>; + return tabs.map((t) => ({ id: t.id, title: t.title, url: t.url })); +} + +async function resolveTab(port: number, token: string, tabId?: string): Promise { + const tabs = await listTabs(port, token); + if (tabs.length === 0) throw new Error('No browser tabs connected. The user needs to attach tabs via the Browser Relay extension.'); + if (tabId) { + const tab = tabs.find((t) => t.id === tabId); + if (!tab) throw new Error(`Tab ${tabId} not found. Available tabs: ${tabs.map((t) => t.id).join(', ')}`); + return tab; + } + return tabs[0]!; +} + +async function sendCdpCommand(port: number, token: string, tabId: string, method: string, params?: unknown): Promise { + const url = `ws://127.0.0.1:${port}/cdp?token=${encodeURIComponent(token)}`; + + return await new Promise((resolve, reject) => { + const ws = new WebSocket(url); + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + settled = true; + ws.close(); + reject(new Error(`CDP command timeout: ${method}`)); + }, 30_000); + + ws.addEventListener('open', () => { + const cmd: Record = { id: 1, method }; + if (params) cmd.params = params; + cmd.sessionId = tabId; + ws.send(JSON.stringify(cmd)); + }); + + ws.addEventListener('message', (event) => { + if (settled) return; + try { + const msg = JSON.parse(String(event.data)) as { id?: number; result?: unknown; error?: { message: string } }; + if (msg.id === 1) { + settled = true; + clearTimeout(timeout); + ws.close(); + if (msg.error) reject(new Error(msg.error.message)); + else resolve(msg.result); + } + } catch { + // ignore parse errors, wait for correct message + } + }); + + ws.addEventListener('error', () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + reject(new Error('CDP WebSocket connection failed — is the Browser Relay running?')); + }); + + ws.addEventListener('close', () => { + if (settled) return; + settled = true; + clearTimeout(timeout); + reject(new Error('CDP WebSocket closed before response')); + }); + }); +} + +// --- Actions --- + +async function actionListTabs(port: number, token: string): Promise { + const tabs = await listTabs(port, token); + if (tabs.length === 0) { + return { content: [{ type: 'text', text: 'No browser tabs connected. The user needs to attach tabs via the Browser Relay extension.' }] }; + } + const lines = tabs.map((t, i) => `${i + 1}. ${t.title}\n URL: ${t.url}\n ID: ${t.id}`); + return { content: [{ type: 'text', text: `Connected tabs (${tabs.length}):\n\n${lines.join('\n\n')}` }] }; +} + +async function actionScreenshot(port: number, token: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + const result = (await sendCdpCommand(port, token, tab.id, 'Page.captureScreenshot', { format: 'png' })) as { data: string }; + return { + content: [ + { type: 'text', text: `Screenshot of "${tab.title}" (${tab.url})` }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: result.data } }, + ], + }; +} + +async function actionNavigate(port: number, token: string, url: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + await sendCdpCommand(port, token, tab.id, 'Page.navigate', { url }); + return { content: [{ type: 'text', text: `Navigated tab "${tab.title}" to ${url}` }] }; +} + +async function actionEvaluate(port: number, token: string, expression: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + const result = (await sendCdpCommand(port, token, tab.id, 'Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true, + })) as { result?: { value?: unknown; description?: string }; exceptionDetails?: { text?: string } }; + + if (result.exceptionDetails) { + return { content: [{ type: 'text', text: `Error evaluating JS: ${result.exceptionDetails.text ?? 'Evaluation failed'}` }], isError: true }; + } + + const value = result.result?.value; + const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2); + return { content: [{ type: 'text', text: `Result from "${tab.title}":\n${text}` }] }; +} + +async function actionPageInfo(port: number, token: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + const result = (await sendCdpCommand(port, token, tab.id, 'Runtime.evaluate', { + expression: 'JSON.stringify({ title: document.title, url: location.href })', + returnByValue: true, + })) as { result?: { value?: string } }; + + let info: { title: string; url: string }; + try { + info = JSON.parse(result.result?.value ?? '{}'); + } catch { + info = { title: tab.title, url: tab.url }; + } + + return { content: [{ type: 'text', text: `Title: ${info.title}\nURL: ${info.url}\nTab ID: ${tab.id}` }] }; +} + +async function actionActivate(port: number, token: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + const res = await fetch(`http://127.0.0.1:${port}/json/activate/${encodeURIComponent(tab.id)}`, { + headers: { [RELAY_AUTH_HEADER]: token }, + }); + if (!res.ok) throw new Error(`Failed to activate tab (${res.status})`); + return { content: [{ type: 'text', text: `Activated tab "${tab.title}"` }] }; +} + +async function actionClose(port: number, token: string, tabId?: string): Promise { + const tab = await resolveTab(port, token, tabId); + const res = await fetch(`http://127.0.0.1:${port}/json/close/${encodeURIComponent(tab.id)}`, { + headers: { [RELAY_AUTH_HEADER]: token }, + }); + if (!res.ok) throw new Error(`Failed to close tab (${res.status})`); + return { content: [{ type: 'text', text: `Closed tab "${tab.title}"` }] }; +} + +// --- Main --- + +export async function execute(_toolCallId: string, params: Params): Promise { + const config = getConfig(); + if (!config) { + return { + content: [{ type: 'text', text: 'Browser Relay is not available. The user needs to connect the Browser Relay extension in Settings → Integrations.' }], + isError: true, + }; + } + + const { port, token } = config; + const { action, tab_id, url, expression } = params; + + try { + switch (action) { + case 'list_tabs': + return await actionListTabs(port, token); + case 'screenshot': + return await actionScreenshot(port, token, tab_id); + case 'navigate': + if (!url) return { content: [{ type: 'text', text: 'url is required for navigate action' }], isError: true }; + return await actionNavigate(port, token, url, tab_id); + case 'evaluate': + if (!expression) return { content: [{ type: 'text', text: 'expression is required for evaluate action' }], isError: true }; + return await actionEvaluate(port, token, expression, tab_id); + case 'page_info': + return await actionPageInfo(port, token, tab_id); + case 'activate': + return await actionActivate(port, token, tab_id); + case 'close': + return await actionClose(port, token, tab_id); + default: + return { + content: [{ type: 'text', text: `Unknown action: ${action}. Use list_tabs, screenshot, navigate, evaluate, page_info, activate, or close.` }], + isError: true, + }; + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { content: [{ type: 'text', text: `Browser error: ${message}` }], isError: true }; + } +} diff --git a/src/apps/officer-web/Screens/Dashboard/Browser/BrowserScreen.tsx b/src/apps/officer-web/Screens/Dashboard/Browser/BrowserScreen.tsx index 9535c6ae..5e1c24b8 100644 --- a/src/apps/officer-web/Screens/Dashboard/Browser/BrowserScreen.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Browser/BrowserScreen.tsx @@ -8,6 +8,8 @@ import { defaultLayout } from './defaultLayout'; import { TabList } from './TabList'; import { TabPreview } from './TabPreview'; +const PROMPT_PREFIX = `You are a browser assistant. The user has connected Chrome tabs via the Officer Browser Relay. You have a 'browser' tool — use it to list tabs, take screenshots, navigate, evaluate JavaScript, and get page info. When asked about a page, take a screenshot first.`; + export const BrowserScreen = () => { const isMobile = useIsMobile(); const [selectedId, setSelectedId] = useGlobal('BROWSER_SELECTED_TAB', null); @@ -30,6 +32,7 @@ export const BrowserScreen = () => { workspace={workspace} locked components={components} + promptPrefix={PROMPT_PREFIX} mobilePanelId={mobilePanelId} onMobilePanelChange={mobilePanelId ? () => onMobileBack() : undefined} /> diff --git a/src/apps/officer-web/Screens/Dashboard/Browser/TabList.tsx b/src/apps/officer-web/Screens/Dashboard/Browser/TabList.tsx index 3df4b217..7ce4a58e 100644 --- a/src/apps/officer-web/Screens/Dashboard/Browser/TabList.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Browser/TabList.tsx @@ -1,16 +1,9 @@ -import { useState } from 'react'; import { useQuery } from '@tanstack/react-query'; -import { Globe, Copy, Check, Loader2, X, Eye } from 'lucide-react'; +import { Globe, Loader2, X, Eye } from 'lucide-react'; import { toast } from 'sonner'; -import { Button } from '@/components/ui/button'; import { useClient } from 'hooks/useClient'; import { useGlobal } from 'hooks/useGlobal'; -type RelayToken = { - token: string; - port: number; -}; - type Target = { id: string; sessionId: string; @@ -22,7 +15,6 @@ type Target = { export const TabList = () => { const client = useClient(); const [selectedId, setSelectedId] = useGlobal('BROWSER_SELECTED_TAB', null); - const [copied, setCopied] = useState(false); const { data: status } = useQuery({ queryKey: ['browser-status'], @@ -30,11 +22,6 @@ export const TabList = () => { refetchInterval: 5000, }); - const { data: tokenData } = useQuery({ - queryKey: ['browser-relay-token'], - queryFn: () => client.get('/browser/relay-token'), - }); - const { data: targets, isLoading } = useQuery({ queryKey: ['browser-targets'], queryFn: () => client.get('/browser/targets'), @@ -44,18 +31,6 @@ export const TabList = () => { const targetList: Target[] = targets ?? []; - const handleCopyToken = async () => { - if (!tokenData?.token) return; - try { - await navigator.clipboard.writeText(tokenData.token); - setCopied(true); - toast.success('Token copied to clipboard'); - setTimeout(() => setCopied(false), 2000); - } catch { - toast.error('Failed to copy'); - } - }; - const handleClose = async (targetId: string) => { try { await client.post(`/browser/targets/${targetId}/close`); @@ -86,35 +61,15 @@ export const TabList = () => { {!isConnected && ( -
-

Connect the Officer Browser Relay extension to start.

- - {tokenData && ( -
-
- Server address - {window.location.hostname} -
-
- Relay port - {tokenData.port} -
-
- Relay token - -
-
- )} - -
    -
  1. Load the extension from src/extensions/browser-relay/
    2. -
  2. Open extension options
    3. -
  3. Enter the server address, port, and token
    4. -
  4. Click the extension icon on a tab
    5. -
+
+ +

Extension not connected

+ + Set up in Integrations +
)} diff --git a/src/apps/officer-web/Screens/Dashboard/Browser/defaultLayout.ts b/src/apps/officer-web/Screens/Dashboard/Browser/defaultLayout.ts index 5cfd4b95..91243c09 100644 --- a/src/apps/officer-web/Screens/Dashboard/Browser/defaultLayout.ts +++ b/src/apps/officer-web/Screens/Dashboard/Browser/defaultLayout.ts @@ -6,6 +6,17 @@ export const defaultLayout: LayoutNode = { direction: 'horizontal', children: [ { node: { type: 'panel', id: 'browser-tabs', appType: null }, size: 25 }, - { node: { type: 'panel', id: 'browser-preview', appType: null }, size: 75 }, + { + node: { + type: 'group', + id: 'browser-right', + direction: 'vertical', + children: [ + { node: { type: 'panel', id: 'browser-preview', appType: null }, size: 60 }, + { node: { type: 'panel', id: 'browser-chat', appType: 'officerdev/chat' }, size: 40 }, + ], + }, + size: 75, + }, ], }; diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/BrowserRelay.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/BrowserRelay.tsx new file mode 100644 index 00000000..06a6b35f --- /dev/null +++ b/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/BrowserRelay.tsx @@ -0,0 +1,198 @@ +import { useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { Copy, Check, Download, ExternalLink, RefreshCw, Trash2 } from 'lucide-react'; +import { toast } from 'sonner'; +import { Button } from '@/components/ui/button'; +import { useClient } from 'hooks/useClient'; + +type RelayToken = { + token: string; + port: number; +}; + +export const BrowserRelay = () => { + const client = useClient(); + const queryClient = useQueryClient(); + const [copiedField, setCopiedField] = useState(null); + + const { data: status } = useQuery({ + queryKey: ['browser-status'], + queryFn: () => client.get<{ extensionConnected: boolean; targetCount: number }>('/browser/status'), + refetchInterval: 5000, + }); + + const { data: tokenData } = useQuery({ + queryKey: ['browser-relay-token'], + queryFn: () => client.get('/browser/relay-token'), + }); + + const regenerate = useMutation({ + mutationFn: () => client.post('/browser/relay-token/regenerate'), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['browser-relay-token'] }); + toast.success('Token regenerated — update the extension with the new token'); + }, + onError: () => toast.error('Failed to regenerate token'), + }); + + const revoke = useMutation({ + mutationFn: () => client.delete('/browser/relay-token'), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['browser-relay-token'] }); + toast.success('Token revoked'); + }, + onError: () => toast.error('Failed to revoke token'), + }); + + const handleCopy = async (value: string, field: string) => { + try { + await navigator.clipboard.writeText(value); + setCopiedField(field); + toast.success('Copied to clipboard'); + setTimeout(() => setCopiedField(null), 2000); + } catch { + toast.error('Failed to copy'); + } + }; + + const isConnected = status?.extensionConnected ?? false; + const serverAddress = window.location.hostname; + + return ( +
+ {isConnected ? ( +
+
+
+

Connected

+

+ {status?.targetCount ?? 0} tab{(status?.targetCount ?? 0) !== 1 ? 's' : ''} attached + {' — '} + view tabs +

+
+
+ ) : ( +

+ Connect your Chrome browser to Officer so AI agents can view and interact with your tabs. +

+ )} + + {/* Step 1: Install */} +
+

1. Install the extension

+

+ Download and unzip the extension, then load it in Chrome: +

+ + + Download Extension + +
    +
  1. + Open{' '} + chrome://extensions{' '} + in Chrome +
    2. +
  2. + Enable Developer mode (top-right toggle) +
    3. +
  3. + Click Load unpacked and select the unzipped folder +
    4. +
+
+ + {/* Step 2: Configure */} + {tokenData && ( +
+

2. Configure the extension

+

+ Open the extension options (right-click icon → Options) and enter these values: +

+ +
+ handleCopy(serverAddress, 'host')} + /> + handleCopy(String(tokenData.port), 'port')} + /> + handleCopy(tokenData.token, 'token')} + /> +
+ +
+ + +
+
+ )} + + {/* Step 3: Attach */} +
+

3. Attach a tab

+

+ Navigate to any webpage and click the Officer extension icon in the toolbar. A cyan ON badge means + the tab is connected. Then go to{' '} + + /browser + {' '} + to view your tabs. +

+
+
+ ); +}; + +type CredentialRowProps = { + label: string; + value: string; + masked?: boolean; + copied: boolean; + onCopy: () => void; +}; + +const CredentialRow = ({ label, value, masked, copied, onCopy }: CredentialRowProps) => ( +
+ {label} + + {masked ? `${value.slice(0, 8)}${'•'.repeat(16)}` : value} + + +
+); diff --git a/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/index.tsx b/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/index.tsx index dd6d655b..5cdfb32a 100644 --- a/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/index.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Settings/IntegrationsSettings/index.tsx @@ -1,5 +1,5 @@ import { useMemo } from 'react'; -import { Puzzle, KeyRound, UserCircle, MessageCircle } from 'lucide-react'; +import { Puzzle, KeyRound, UserCircle, MessageCircle, Globe } from 'lucide-react'; import type { LayoutNode, PanelComponents } from 'officerdev'; import { WorkspaceLayout } from 'officerdev'; import { useAuth } from 'hooks/useAuth'; @@ -15,6 +15,7 @@ import { TelegramBotConfig } from './TelegramBotConfig'; import { TelegramAccount } from './TelegramAccount'; import { WhatsAppBotConfig } from './WhatsAppBotConfig'; import { WhatsAppAccount } from './WhatsAppAccount'; +import { BrowserRelay } from './BrowserRelay'; const GLOBAL_KEY = 'INTEGRATIONS_SETTINGS_SELECTED'; const TAB_KEY = 'INTEGRATIONS_SETTINGS_TAB'; @@ -79,6 +80,13 @@ const personalSections: SettingsSection[] = [ description: 'Link your WhatsApp account', content: , }, + { + key: 'browser-relay', + icon: Globe, + title: 'Browser Relay', + description: 'Connect your Chrome browser', + content: , + }, ]; const IntegrationsSidebar = () => { diff --git a/src/databases/officer_db/src/index.ts b/src/databases/officer_db/src/index.ts index 017fd82c..ae1f895f 100644 --- a/src/databases/officer_db/src/index.ts +++ b/src/databases/officer_db/src/index.ts @@ -29,6 +29,7 @@ export { deleteServerIntegration, getUserIntegrations, getUserIntegration, + getIntegrationsByProvider, upsertUserIntegration, deleteUserIntegration, findUserByIntegrationConfig, diff --git a/src/databases/officer_db/src/queries/integrations.ts b/src/databases/officer_db/src/queries/integrations.ts index 6dca03f0..c0912137 100644 --- a/src/databases/officer_db/src/queries/integrations.ts +++ b/src/databases/officer_db/src/queries/integrations.ts @@ -38,6 +38,10 @@ export async function getUserIntegrations(userId: number): Promise { + return db.select().from(userIntegrations).where(eq(userIntegrations.provider, provider)); +} + export async function getUserIntegration(userId: number, provider: string): Promise { const [row] = await db .select() diff --git a/src/extensions/browser-relay/background.js b/src/extensions/browser-relay/background.js index 1418aeea..5ecf1484 100644 --- a/src/extensions/browser-relay/background.js +++ b/src/extensions/browser-relay/background.js @@ -39,6 +39,9 @@ const tabOperationLocks = new Set() /** @type {Set} */ const reattachPending = new Set() +/** @type {Set} */ +const reattachingTabs = new Set() + let reconnectAttempt = 0 let reconnectTimer = null @@ -162,8 +165,12 @@ async function ensureRelayConnection() { } }) - ws.onclose = () => { + ws.onclose = (ev) => { if (ws !== relayWs) return + if (ev.code === 4000) { + onRelayReplaced() + return + } onRelayClosed('closed') } ws.onerror = () => { @@ -205,6 +212,30 @@ function onRelayClosed(reason) { scheduleReconnect() } +function onRelayReplaced() { + relayWs = null + relayGatewayToken = '' + relayConnectRequestId = null + + for (const [id, p] of pending.entries()) { + pending.delete(id) + p.reject(new Error('Replaced by another browser session')) + } + + reattachPending.clear() + + for (const [tabId, tab] of tabs.entries()) { + if (tab.state === 'connected') { + setBadge(tabId, 'error') + void chrome.action.setTitle({ + tabId, + title: 'Officer Browser Relay: replaced by another browser — click to reconnect', + }) + } + } + // Do NOT schedule reconnect — user must click to take over again +} + function scheduleReconnect() { if (reconnectTimer) { clearTimeout(reconnectTimer) @@ -534,6 +565,15 @@ async function detachTab(tabId, reason) { await persistState() } +function describeError(err) { + const msg = err instanceof Error ? err.message : String(err) + if (msg.includes('not reachable')) return 'relay not reachable — check server address in options' + if (msg.includes('Missing relay token')) return 'no token configured — open options to set up' + if (msg.includes('401') || msg.includes('Unauthorized')) return 'token rejected — check token in options' + if (msg.toLowerCase().includes('timeout')) return 'connection timed out' + return msg.length > 80 ? msg.slice(0, 80) + '…' : msg +} + async function connectOrToggleForActiveTab() { const [active] = await chrome.tabs.query({ active: true, currentWindow: true }) const tabId = active?.id @@ -576,7 +616,7 @@ async function connectOrToggleForActiveTab() { setBadge(tabId, 'error') void chrome.action.setTitle({ tabId, - title: 'Officer Browser Relay: relay not running (open options for setup)', + title: `Officer Browser Relay: ${describeError(err)}`, }) void maybeOpenHelpOnce() const message = err instanceof Error ? err.message : String(err) @@ -660,7 +700,24 @@ async function handleForwardCdpCommand(msg) { ? { ...debuggee, sessionId } : debuggee - return await chrome.debugger.sendCommand(debuggerSession, method, params) + try { + return await chrome.debugger.sendCommand(debuggerSession, method, params) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + if (!message.includes('Session with given id not found')) throw err + + // Chrome's internal debugger session became stale — re-attach and retry once + console.warn(`Stale session for tab ${tabId}, re-attaching debugger`) + reattachingTabs.add(tabId) + try { + await chrome.debugger.detach(debuggee).catch(() => {}) + await chrome.debugger.attach(debuggee, '1.3') + await chrome.debugger.sendCommand(debuggee, 'Page.enable').catch(() => {}) + return await chrome.debugger.sendCommand(debuggee, method, params) + } finally { + reattachingTabs.delete(tabId) + } + } } function onDebuggerEvent(source, method, params) { @@ -695,6 +752,7 @@ async function onDebuggerDetach(source, reason) { const tabId = source.tabId if (!tabId) return if (!tabs.has(tabId)) return + if (reattachingTabs.has(tabId)) return if (reason === 'canceled_by_user' || reason === 'replaced_with_devtools') { void detachTab(tabId, reason) diff --git a/src/server.tsx b/src/server.tsx index 8c32df2e..c64e0ee5 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -202,7 +202,7 @@ console.log(`🚀 Server running at ${server.url}`); const BROWSER_RELAY_PORT = Number(process.env.BROWSER_RELAY_PORT ?? '18792'); try { - startBrowserRelay(BROWSER_RELAY_PORT); + await startBrowserRelay(BROWSER_RELAY_PORT); console.log(`[browser-relay] listening on port ${BROWSER_RELAY_PORT}`); } catch (err) { console.error('[browser-relay] failed to start:', err instanceof Error ? err.message : err); diff --git a/src/servers/api/browser/relay-auth.ts b/src/servers/api/browser/relay-auth.ts index 923ad637..52282734 100644 --- a/src/servers/api/browser/relay-auth.ts +++ b/src/servers/api/browser/relay-auth.ts @@ -8,20 +8,31 @@ if (!JWT_SECRET) { throw new Error('JWT_SECRET is required for browser relay auth'); } -export function deriveRelayToken(userId: number, port: number): string { +export function deriveRelayToken(userId: number, port: number, salt: string): string { return createHmac('sha256', JWT_SECRET!) - .update(`${RELAY_TOKEN_CONTEXT}:${port}:${userId}`) + .update(`${RELAY_TOKEN_CONTEXT}:${port}:${userId}:${salt}`) .digest('hex'); } const tokenToUser = new Map(); -export function registerUserToken(userId: number, port: number): string { - const token = deriveRelayToken(userId, port); +export function registerUserToken(userId: number, port: number, salt: string): string { + const token = deriveRelayToken(userId, port, salt); tokenToUser.set(token, userId); return token; } +export function unregisterUserToken(token: string): void { + tokenToUser.delete(token); +} + +export function restoreUserTokens(entries: Array<{ userId: number; salt: string }>, port: number): void { + for (const { userId, salt } of entries) { + const token = deriveRelayToken(userId, port, salt); + tokenToUser.set(token, userId); + } +} + export function resolveUserFromToken(token: string): number | null { return tokenToUser.get(token) ?? null; } diff --git a/src/servers/api/browser/relay.ts b/src/servers/api/browser/relay.ts index c35b2cf1..27b21886 100644 --- a/src/servers/api/browser/relay.ts +++ b/src/servers/api/browser/relay.ts @@ -16,7 +16,8 @@ import type { ExtensionMessage, ExtensionPongMessage, } from './relay-types'; -import { getRelayAuthHeader, resolveUserFromToken } from './relay-auth'; +import { getIntegrationsByProvider } from 'officerdb'; +import { getRelayAuthHeader, resolveUserFromToken, restoreUserTokens } from './relay-auth'; const RELAY_AUTH_HEADER = getRelayAuthHeader(); const DEFAULT_EXTENSION_RECONNECT_GRACE_MS = 5_000; @@ -315,11 +316,11 @@ function handleExtensionMessage(relay: UserRelay, data: string) { } function handleExtensionClose(relay: UserRelay, ws: ServerWebSocket) { + if (relay.extensionWs !== ws) return; if (relay.pingInterval) { clearInterval(relay.pingInterval); relay.pingInterval = null; } - if (relay.extensionWs !== ws) return; relay.extensionWs = null; for (const [, pending] of relay.pendingExtension) { clearTimeout(pending.timer); @@ -449,9 +450,18 @@ export function getUserTargets(userId: number): ConnectedTarget[] { return Array.from(relay.connectedTargets.values()); } -export function startBrowserRelay(port: number) { +export async function startBrowserRelay(port: number) { relayPort = port; + // Restore persisted relay tokens from DB + const integrations = await getIntegrationsByProvider('browser-relay'); + const entries = integrations + .map((i) => ({ userId: i.userId, salt: (i.config as { tokenSalt?: string })?.tokenSalt })) + .filter((e): e is { userId: number; salt: string } => Boolean(e.salt)); + if (entries.length > 0) { + restoreUserTokens(entries, port); + } + const server = Bun.serve({ port, hostname: '0.0.0.0', @@ -566,18 +576,15 @@ export function startBrowserRelay(port: number) { if (path === '/extension') { if (userId === null) return new Response('Unauthorized', { status: 401, headers: corsHeaders }); const relay = getUserRelay(userId); - // Close stale extension WS - if (relay.extensionWs && relay.extensionWs.readyState !== WebSocket.OPEN) { + // Close existing extension WS (stale or active) — last connection wins + if (relay.extensionWs) { try { - (relay.extensionWs as ServerWebSocket).close(); + relay.extensionWs.close(4000, 'replaced by new connection'); } catch { // ignore } relay.extensionWs = null; } - if (extensionConnected(relay)) { - return new Response('Extension already connected', { status: 409, headers: corsHeaders }); - } const ok = server.upgrade(req, { data: { kind: 'extension', userId, token: token! } }); if (!ok) return new Response('Upgrade failed', { status: 500, headers: corsHeaders }); return undefined as unknown as Response; diff --git a/src/servers/api/browser/router.ts b/src/servers/api/browser/router.ts index 6521463c..e632981a 100644 --- a/src/servers/api/browser/router.ts +++ b/src/servers/api/browser/router.ts @@ -1,17 +1,44 @@ +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; import { createRouter } from '@@/create-router'; import * as errors from '@@/custom-errors'; -import { registerUserToken } from './relay-auth'; +import { getUserIntegration, upsertUserIntegration, deleteUserIntegration } from 'officerdb'; +import { deriveRelayToken, registerUserToken, unregisterUserToken } from './relay-auth'; import { getRelayPort, getUserRelayStatus, getUserTargets } from './relay'; import { captureScreenshot, evaluateJS, navigateTo } from './cdp'; export const browserRouter = createRouter(); -function getOpts(userId: number) { +const EXTENSION_DIR = join(import.meta.dir, '../../../extensions/browser-relay'); + +async function getOpts(userId: number) { const port = getRelayPort(); - const token = registerUserToken(userId, port); + const integration = await getUserIntegration(userId, 'browser-relay'); + const salt = (integration?.config as { tokenSalt?: string } | null)?.tokenSalt ?? randomUUID(); + if (!integration) { + await upsertUserIntegration({ userId, provider: 'browser-relay', config: { tokenSalt: salt } }); + } + const token = registerUserToken(userId, port, salt); return { relayPort: port, userToken: token }; } +browserRouter.get('/extension-download', async (ctx) => { + const proc = Bun.spawn(['zip', '-r', '-', '.'], { + cwd: EXTENSION_DIR, + stdout: 'pipe', + stderr: 'pipe', + }); + const blob = await new Response(proc.stdout).blob(); + await proc.exited; + if (proc.exitCode !== 0) throw errors.INTERNAL_SERVER_ERROR('Failed to create zip'); + return new Response(blob, { + headers: { + 'Content-Type': 'application/zip', + 'Content-Disposition': 'attachment; filename="officer-browser-relay.zip"', + }, + }); +}); + browserRouter.get('/status', async (ctx) => { const user = ctx.get('user'); const status = getUserRelayStatus(user.id); @@ -21,10 +48,48 @@ browserRouter.get('/status', async (ctx) => { browserRouter.get('/relay-token', async (ctx) => { const user = ctx.get('user'); const port = getRelayPort(); - const token = registerUserToken(user.id, port); + let integration = await getUserIntegration(user.id, 'browser-relay'); + if (!integration) { + const salt = randomUUID(); + integration = await upsertUserIntegration({ userId: user.id, provider: 'browser-relay', config: { tokenSalt: salt } }); + } + const salt = (integration.config as { tokenSalt: string }).tokenSalt; + const token = registerUserToken(user.id, port, salt); return ctx.json({ token, port }); }); +browserRouter.post('/relay-token/regenerate', async (ctx) => { + const user = ctx.get('user'); + const port = getRelayPort(); + const oldIntegration = await getUserIntegration(user.id, 'browser-relay'); + if (oldIntegration) { + const oldSalt = (oldIntegration.config as { tokenSalt?: string })?.tokenSalt; + if (oldSalt) { + const oldToken = deriveRelayToken(user.id, port, oldSalt); + unregisterUserToken(oldToken); + } + } + const newSalt = randomUUID(); + await upsertUserIntegration({ userId: user.id, provider: 'browser-relay', config: { tokenSalt: newSalt } }); + const token = registerUserToken(user.id, port, newSalt); + return ctx.json({ token, port }); +}); + +browserRouter.delete('/relay-token', async (ctx) => { + const user = ctx.get('user'); + const port = getRelayPort(); + const integration = await getUserIntegration(user.id, 'browser-relay'); + if (integration) { + const salt = (integration.config as { tokenSalt?: string })?.tokenSalt; + if (salt) { + const oldToken = deriveRelayToken(user.id, port, salt); + unregisterUserToken(oldToken); + } + await deleteUserIntegration(user.id, 'browser-relay'); + } + return ctx.json({ ok: true }); +}); + browserRouter.get('/targets', async (ctx) => { const user = ctx.get('user'); const targets = getUserTargets(user.id); @@ -44,7 +109,7 @@ browserRouter.get('/targets/:id/screenshot', async (ctx) => { const targetId = ctx.req.param('id'); const format = (ctx.req.query('format') as 'png' | 'jpeg') || 'png'; - const opts = getOpts(user.id); + const opts = await getOpts(user.id); // Find session ID for the target const targets = getUserTargets(user.id); const target = targets.find((t) => t.targetId === targetId); @@ -65,7 +130,7 @@ browserRouter.post('/targets/:id/evaluate', async (ctx) => { if (!body.expression) throw errors.BAD_REQUEST('expression is required'); - const opts = getOpts(user.id); + const opts = await getOpts(user.id); const targets = getUserTargets(user.id); const target = targets.find((t) => t.targetId === targetId); if (!target) throw errors.NOT_FOUND('Target not found'); @@ -85,7 +150,7 @@ browserRouter.post('/targets/:id/navigate', async (ctx) => { if (!body.url) throw errors.BAD_REQUEST('url is required'); - const opts = getOpts(user.id); + const opts = await getOpts(user.id); const targets = getUserTargets(user.id); const target = targets.find((t) => t.targetId === targetId); if (!target) throw errors.NOT_FOUND('Target not found'); @@ -102,12 +167,11 @@ browserRouter.post('/targets/:id/activate', async (ctx) => { const user = ctx.get('user'); const targetId = ctx.req.param('id'); - const port = getRelayPort(); - const token = registerUserToken(user.id, port); + const { relayPort, userToken } = await getOpts(user.id); try { - const res = await fetch(`http://127.0.0.1:${port}/json/activate/${encodeURIComponent(targetId)}`, { - headers: { 'x-officer-relay-token': token }, + const res = await fetch(`http://127.0.0.1:${relayPort}/json/activate/${encodeURIComponent(targetId)}`, { + headers: { 'x-officer-relay-token': userToken }, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return ctx.json({ ok: true }); @@ -120,12 +184,11 @@ browserRouter.post('/targets/:id/close', async (ctx) => { const user = ctx.get('user'); const targetId = ctx.req.param('id'); - const port = getRelayPort(); - const token = registerUserToken(user.id, port); + const { relayPort, userToken } = await getOpts(user.id); try { - const res = await fetch(`http://127.0.0.1:${port}/json/close/${encodeURIComponent(targetId)}`, { - headers: { 'x-officer-relay-token': token }, + const res = await fetch(`http://127.0.0.1:${relayPort}/json/close/${encodeURIComponent(targetId)}`, { + headers: { 'x-officer-relay-token': userToken }, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); return ctx.json({ ok: true }); diff --git a/src/servers/api/pi/pi-bridge.ts b/src/servers/api/pi/pi-bridge.ts index b0c2d66d..bb091d38 100644 --- a/src/servers/api/pi/pi-bridge.ts +++ b/src/servers/api/pi/pi-bridge.ts @@ -9,6 +9,8 @@ import { ensureDockerContainer } from "../terminal/websocket"; import { getServerIntegration, getUserIntegration } from "officerdb"; import { logger } from "./logger"; import { parseFrontmatter } from "../skills/skills"; +import { getRelayPort } from "../browser/relay"; +import { registerUserToken } from "../browser/relay-auth"; export type PiEventHandler = (event: PiEvent) => void; @@ -188,6 +190,23 @@ async function ensureGoogleTokenFile(userId: number, email: string): Promise> { + const port = getRelayPort(); + if (!port) return {}; + try { + const integration = await getUserIntegration(userId, 'browser-relay'); + const salt = (integration?.config as { tokenSalt?: string })?.tokenSalt; + if (!salt) return {}; + const token = registerUserToken(userId, port, salt); + return { + OFFICER_BROWSER_RELAY_PORT: String(port), + OFFICER_BROWSER_RELAY_TOKEN: token, + }; + } catch { + return {}; + } +} + type SandboxOptions = { userId: number; username: string; @@ -245,6 +264,7 @@ export async function spawnPi( const googleConfigHost = await ensureGoogleConfigFile(); await ensureGoogleTokenFile(sandbox.userId, sandbox.email); + const browserRelayEnv = await getBrowserRelayEnv(sandbox.userId); const envFlags = [ '-e', `HOME=${containerHome}`, @@ -256,6 +276,7 @@ export async function spawnPi( '-e', `OFFICER_GOOGLE_CONFIG_PATH=/officer/google-oauth.json`, '-e', `OFFICER_GOOGLE_TOKEN_PATH=/officer/user/integrations/google.json`, '-e', `OFFICER_EMAIL_DB=/officer/emails.db`, + ...Object.entries(browserRelayEnv).flatMap(([k, v]) => ['-e', `${k}=${v}`]), ]; const rel = relative(sandbox.homeDir, cwd); @@ -299,6 +320,7 @@ export async function spawnPi( const toolsDirs = [getGlobalToolsDir(), getUserToolsDir(email)].join(':'); const googleConfigPath = await ensureGoogleConfigFile(); const googleTokenPath = await ensureGoogleTokenFile(userId, email); + const browserRelayEnv = await getBrowserRelayEnv(userId); proc = Bun.spawn(args, { cwd, @@ -317,6 +339,7 @@ export async function spawnPi( OFFICER_GOOGLE_CONFIG_PATH: googleConfigPath, OFFICER_GOOGLE_TOKEN_PATH: googleTokenPath, OFFICER_EMAIL_DB: join(DATA_PATH, email, 'emails.db'), + ...browserRelayEnv, }, });