From 538e23dd65d20dd066583b15d6397c9e87d31ce6 Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Thu, 5 Dec 2024 11:41:01 +0300 Subject: [PATCH 1/6] fix nested functions in assistant Signed-off-by: Hunter Achieng --- bun.lockb | Bin 114599 -> 115149 bytes package.json | 2 +- .../evaluation/test_prompts_issue_97_a.ipynb | 508 ++++++++++++++++++ 3 files changed, 509 insertions(+), 1 deletion(-) mode change 100644 => 100755 bun.lockb create mode 100644 services/job_chat/evaluation/test_prompts_issue_97_a.ipynb diff --git a/bun.lockb b/bun.lockb old mode 100644 new mode 100755 index ed46ec1c1266ad607026fd5bb495eef0631243ab..3231c4d68b8c47a06755a1aa7ea95ab2d2251832 GIT binary patch delta 22040 zcmeI4d3=r6_W#d5a&Tw_86*-S#1IuEk%WeGgc=VjDx#$b4iX_Un5vbiR#ivc)u&o> zCsI?2R!aw06{W@Pq}AKf5tn=EpxS%ee&6dE(EDxs{r>*=-hY1gdF7L}KWneO_gZ^; zrsR40mzr{b9 zJ-A~($nA2~hi4(PHAsb=01X&lk13PhaJu(a#EE|_!ls`T=uBc#o0recI<#Kt^N7T0L zZ$hSvr|U_GN9AQj1!FNBHz~hpGIAK!65*u$vg8Rx`4c11NyS_BQWbwgI6g{s|LXQs zs+8KkM4I__7)#+2*1fizg^YwxD404vdHOWhTB?>Xmg@%Y))kJkZLyM|5|C0`C=x#^ zJ2iBb7_P2xv7d*Ode2aq$O*-z z$-(?_6_cv%0%24r8IMF;4Hlp$S@|Ia$g2DSf~s;6T!uHdvCG9zWr*EyT&xx52Bg&U zH@m^*aG5UmAjQ%7c76k7T~~#x@@qOr8u*?9G787wGTg7(@>x6o5u`Nu9S%sO(G-#j zo1mA7E0N;xWF$Y8Bl#tE1!c*l!HL0H0l3%~l;xL}=FfJe!=q$gewJWm_?lZ0wjo2h zEDR|Qd5|*DM{!Uh--nclb|Ph<*C3^#B}gf^Khe^UhD*7ANGaD6DS<^H#Xh^0%T*x_ zgrJZf*h6HrRk_)gtB}%AF_PY@%tM+H76qr5GPdK&X2=k`Y9v{9QwnBH3BoV5u^c&z zl%Y6*6i1(-n}n}IN{_wO)-}~k>~3o*W=<*?H;D=g%7U|`qES~{o;Gi1RhSEx3TEdQ z6%~|~yPiQWyev3nMi7r4qE=~Wp)Jk&lBZ3{Xm9nT)(M{0H+>X(IRF|7O|+1h}n zO`e!MzM#~#x1*KcGsUWXY-#?uV42JHB3$~@m1@;C1SzdM3riMMql%enR;|U;%f|<& zmb<0|i)RvwYegr^dNEQ)sWfP2yMpPK&(%n&eK=C`7hoseIps=&`Qwvu(cv{SE&HuV znN|j}2?JUgEG?C_HEUgGtAYi#LGLbBfefUK>N%vi|9iVo@EU76Ub&5lW6m!27Uh|- zmD6xkrq_pEU9LD;P-DAU)95Ht8rqMP`8EtGqjLa5;ibW{;-VQrSDxSHiiZzGN)Pyv zGA*4^J(XcbRyl@-q`t|rkrhvH`LeD0SWX2KX1iQN$gfv{K~ECWz?A%H$;D#}UFnhm zzu3cy*wMGL?Ium1l0P-MVElwKx~;r4zuC1`#f4?XQzsM@1!pr3QeXL$X)=|9?yPsvwjxn*Qc&uw4Tt6VMpyGo23?eH8}9SmLa|PILqwK)v(i-&K6RBo72$V}a_hoy zpZ6)ZRotVi!!z9{-MVdr&mC7o7h*V5Cq??zAzcvZ_tp(@x!4&hYv_{jOm9yZ%L{fs zuCq>x@~h8vL6qO!(xa=QeC}M2J{{%r?(%RV$Ev2(?X6YQVrEO;p0Ht74QZL)r7$0? zwmuJgAJ!k{)+JGy?u1&pFxuyx$mP*x2Ufz^M=2mxe+XmSsjMZ%-SM?`VT{jPSlf!s ztyB7U)~90p>T8|U*zZkbPD{}ciNrnLtE(FOylYWP?IBWwI;E3h{hkk)P1oqNQCXfi z#yCT72+wj4xk{go^?4SdNY!U!v%GJUlBV6dIwI4Z6{@S^eBLLSMA;}C>J+^DJ50Jk z>FUOr?yKwR!X`dl4I@HH`y`gEAcOoen7kVxx)4Lodk>Xq9Oz$C^)inE9u|UOd#-?$mCkxhJ z;!$_#Q_cM96`hpe_w->A4w9MUT|$ao&x+FX9;}C+LU%-m>$c5(?!+iv*xcut8|8BK zHtk;{CEgNplT44hk;~Q5?7q}Sy0C@MTaH2+H?zGPFEN}}Cv{Sy-`y-)7bg1LqoQ?H zqE9`ePbK<2p)A?qX7xc*1I+Rgel={cJ{yzeQH@=$p?X7NmbaMH$V(OnV5Vc96fQ`H zNFAQ%NHMyxS?+VO(uk)mJ4RnA>?tMHU*hxZBxM?TE|Bt>1v@pNaecN~ma5e2+W0+t z;KR(ESoRW9lq(@Mz;yK`QX{QUqcXiQoUnR3Zn;Ot>$Yuu-gPLg&SJWIzk&^S3b_5v z^l3~Mv$Cyn(x30cI-3E9B{-hmKq^B|iOy2@>!fym@3)deHRinZ4rxvoS?xw;sx|sl zJ6W+w$$n1~_eot%PlKe`|Lj3|0mk`2%G07sl!-}X-fU9h2`fJ~({m@RhuLTEnilkm zC$go~NXL3|NDb9z+jQ0II`}>Rfc4hNZL_@BwQ{+xr4FS_qBFf3CWC3V8US%U5nNxCZ4 zr|#3IQvK?%PD=B;>$cT}X+CeDtu+`(K1KR_n~QbSiKW_lZ4Z87$(rkUPZ zuvD}85}L0PCd<_O5h>{xbCG*Hw6pdeWv7_+EdmUk;Ddwt^7>2|UodsC9F#Z^nE zuuOw6*%8d8?S0Z_HDo98{0w8@Y3|zgRu{U=rM(O$>xn^O`+N`95f)-@=@A`t+s;1k ztPWPo?0Vfg>vf&|-t%a>S{^sabSHPzZM*o~MICiv7oYbC<;72`z@=I#mL-h-7yy&T z*khVxdKbWCXP~cOui2KhWaswQqoZV;tsQ>^Ov;&?w08+iTCl=A0FxfK_9ida!m=4# zdD$?rqy_r_Hkeovc+*Vvs$SRC@4bLd0__x=EvUNe1h zr(L5@XZyUf(byf^HPibNOq{Vs|2vqq)Mat?q9>#Y<_}HG*GWD7-aY87ftNX@NKVjG zvN?H>k_~``NUaxOW2{4cMd-L-yi3Kh$gx9n%atj=W0sONP-FTcCC zUsv_=d1v@(0z+#xd<3JP*`%Yf?`2jXBQ+5wGs&Y<*rOiPr+PE8=wugS=Jm<)$SmM| z#XPIz>?)11+L~s1UWIXm!^nM0N;(FYm@AzbL~Bhl8$EMj=1JVUm6TX9+nCR)r_;lP z=E;JwkH%!VOM2?6em>7;6r5Zc-IJ1{m^l%=?WjRqaG4iGMKH;;rpyyCY1o{y>RX)@ z@VndflEv}_R~PeOG-q{%cMpu&TUpoafO_;DE6bi4-C@$fbXr`dr(Bqx(j?2fmK2jl z=14-OHh=CxrztDu&6a z?PHjA;%%IBsRphO3v+baJfG)Tl$d%mpWodOscUGY^XYE;)u#wjJhtw$a)((2j z^;T2X>0u#kIC<8@ji)k3occ;~S#_`;W;t0&>tJK_l-w+@H`kg-<_h!nf!To&u&VBLx$n*a2BcJUqb)}u1kx)N8~G(;wFQX5Rv?E+;oE@ZzX;^GQcC^~Q)@}F zdlg8JybdIPuPyf>#o+@sKZxY0a7jWuJq)CRcYzdq&zA4o@+eYz=tG+yL&_mi8ax4{ z+^04_iIn=z0%`c1Ex$pE-M1#EU%n?Hhe$E}5r{)S+gxES4c9=5zLu1>rSQ78KFrpO z6n(hOMM^TlmXWqzq|&#Js#PJiN5iD`SfrGWM~Yrvr-(k$*0-_w)kuC^?QGt`mZ`Q( zw`FH!ZS1-urM@0WIYdf3y%l31x;}Qs|3*qh{cL-Y;@|+Js0P}4k&+x_bCFVRu+1+^ zdktJxSe^~F3k*Ywr#IRKuar{ZNLw#bk~i7>N-1`u?ffxF;bU#Pu@!d4I9ralWzaSd zDFr6l`boAduw@}qjw_{X2Gec*l~U@NX_uR2=T|rhkw#U3J8s(aX$kJ#2%O38oJ){B(t zm)rbGDXJB={z@tKt8M)i(wZ7qC`?Blw+%%~!KZBgM^fz8*>)l&S;a5WZ?O4BQF4fs zA4+9;k#`6-;%ONy-vQE?6u`S+kT&nxHR~NopHd< z5GndMZT(xe{z@sTL$>}(DG~n}y*T)uoi9?7@7vsw6_OASKe98fl(MaSj$X39u=7QV zV_(_&lSnz$|J~;2YyoBO zIetruT|D_>lE5$F&28C2l=dO@JE&vo7=J7A+fusn|KOs?Y`TT~miZ?MX{Z$t&;Rc) zimaZJYa(;!`L|yb(Xbp`JXM%u_V2zZx_nJU|4Wzun=gtkUlYmr%Z&W@UKCxvCJO#r zE{gP&QH}m{5yUY6`szqF=>J><{pTV`Hi2JX9mzTCKNms&zgz^3{lC8m(p{#McYmni zJ1ZLP{-)}cBO8k{)Ahn*_dmOB#~ayO&fVO!xYfJ2Y`Xuw=y!K_J@mk#H@ALp&foOh zMq_T+wy-k;s%Blk`*@2Fc@H`zT5p>g7${dO&Ji+s#>_X~Yk4aE=fCBZ>>U)AHKOUG z1EQ8x{PW3csyF|$bJeGD3Dfp;dt^{qZuR-~?>*WpO-nptt>w9nMG4lN_ z_nuj@e%XR9lgG?Gbo#@7+vdAQ-cr%_{e+sUUT?ol4=j$>k<$XImd>A+r)y6O)i+HI zD6dw4non^%t-hoiaU7cb*ZdmrM_+SbYL^ z64q-*KsC{eXXNQeW`^qTVDUP8W}falD^x!@GoTXmdDul*?yP`np;yn!(`#pkYR~L| zYN>N(=jq`$hw5#x)>_?6d~=BC=737l)v)cb$T6I_0xDC_t02Bg;)8Y3DV4-G zkN7GBDodY$orLw87f{{w;(5e(8}Y&XI{P-_n@@bV1yr^^54#A&h5l^2l2uBXmtniEg-%-0;->`hHZyME(oZA&R;-$cM>0Lpl)y{@hv32I|J%E zy$iMv)^cG$<>-=y#CI3*!LHW{cM;zr;=3!La`h3|QCRw-fXdVJ77^dw#0R@Ur`%0^ z_YmLR0X0IOfSrW(x+kD+(u?mQzI%xeHd<%jOMLec-@O5qug}9S!gB8msBwDreZ+S^ z@!cO#L7j6y@jXC%u!&kdKzxgd?}30S(ABW*u*k&$HCg8`CcX!W4>mMN*hyF~BcN{8iw)v?g!o_;I{OjgTS|P71k^lz9(ECyyEMRCtkp}2?@{7= zG{7s{oJWc8G2(;Wsnuh|w~Y863#hwvHEcU9a#=v#t@D==-*V!E-K!fcC%zTLw>-d0 z^Ifogu$C(VYOyX^L3}HT52kg(O5$5Zd@BQLi9P~53QJ!VP==njiuhI&A8e^kSxtOv zh;Ma(S6C-tCtv_)+-}A%=`#`5W zPkh^n@A-f_rcc05!g_5F@a}E#cH(=1_+X#t>=%gdMdEuQpib!Xu#2$V7X#`uz4}Gs z+d+Ig0_qE$vxE3{5+Cdin08?`7hHoz)FqCcamQ z@8y8{yWR!c2W$CCK%LhmuMpp>#0UFEC%j60yNK`Afcj1!fgOdV?+U2z^}Jog_b1|m z{h(9+M0~r6?@s~VI-Y=?g!S4TP(SI#yNT~L;)DI7vtJ{=J;e7~z^#=2`n6&DA~bhT zK-JKz_YmOg1o(PDd34U}1h|&~VYRf{OMv?baIc&It6|$=k^AHXn7@wz_Y)v2R5#d9 zfNv1se)9wf+Xrj;hIs;fg8&Z@AS_%b93a4h1bDza0m6>L(hr&^z=H(%CIP}?bjq6q z_!a@aX`TRKCt3#PU&V?#C{yuri)q2ITMV>_7BZcbK$13#tk7~PXcG&XVM;RgX|Jmc$xo_&e#djhj z#M5;Uf5}KS*SWXs`)Ww0ZuZXDI>Qs>;Q?8d7vyuU0WwNhg+lqj;oMjBUea8wl2)JlV@ecmaf88-&V>KxoNgx1Ee%6 zPxIt;m!##Dg-CkLl?&uBUsoXIc^K~|%_GXLPn?Sc;zT36EZypQhqO2vW7=^aBM-~u zDYrP#*i@Q-i0Yb6T8>y-7e;!Nt&6jD_0h?TJ8`s$t!qHKqpfRd>l(tx0C7GZoj4T^ zI>>#Lc-q`Hj36zAMc2aCNfbPHlD`o(U%8-`KYBKG;(SZnt`TW@A0S67TNh1wG-+|Z zwXKUG&FqoCo^@5Uv6YP>`9M5PvURbf$Jx5JwocxYCILCFwsrjNnpvSlB=3Isk-vyF z9TW$WZCyO+=|I|QZ|N#r%^(t;G}gg3Odu`qOQo@nwyrs8d9x^$q}aL^q$LuumzPZZ zxDsuh*r(aLNr{rfOL&QlqfXc9X{o+uotl*yqnfK$b>so9JW)(@84H@Ln=52K$q2~g zmB}hoLxxx$*fs<5XgVC(2*iL`5CY`M@fq+5_zcKm*an^hi@@FB9myVU0ArF$rV3QBzDfgE^ zo{Gz}|DE6^@G^J>yb4|bFM|7kynvC%x*O#soV?QXdvrF zUMtN2K_F{I9u@Ti{Xqc88@`_4TF?uOprMiACLr&#Wx!=N$?S=jC(X@BBmj9ab`IPK zMgV!KunYVN><0G(dB^h@SO(<%&=N2Q+zMs_SsR1EbzmSE0D6O)secZ*1>6cIfQg_$ z7UzvX=DSQanOfs0FcRzo`@!o#UM4LC4}%I&3Fd*@zz)jn1TTS?!7Jcf@Ey1SD#1K( z8@L_Z0TzI}K+8q^x*OaB?gRIO2f%|sgC*c$<48-@+OwME8sn#yszb$inmq{K0DHmh z;0`b!6o3Pm%951D&=#}?vhmdcHNh$J-voz5!WV#r;4u6M_%nDGz7-519SPnhEj#PG zGHce5cmT+*dNYt6QZ}_QpgC~3EPKIRY(@iF%JPIiAIQE_2prus(t=6YNtyOwI*_z% z6~ZNdhO3#Ckb(lq5KNMAIlfIX2C?QFgp!U&C4(w!Tp5d7t48uaN#;f?OJ24B*$TXX z6=iM#AxJr63QqvdKz$(Pq{=WL8?|g_vK(czl|4}Q#)d%74zf|pGL_9&Frr* zr@y1HmVvb-q~l}~hTwB7hygNca&8L;(I5(hfqEbk$cTDDEpV2yXTWJ7lkl%VM*1^w z9DD+vktJ6}A_K?-lZmz%+zDh>wgd^F5@2es4Dq}bkX~pF+JLSg3tR)7d>N(ApdCmB zPS41QcLWl&c|MfUx*EvniE$^82GW5zmI<65>OxvJ42O#&{lOT}9drXe;0JSotncf9 z*vNX9bua6GAm{@sM)PYnm00qp z3X7CVoq{saelI=O&Kra54#Xj;Lnf(AN+~DlJHYK=K9~n?19!>zFC=jvxCbl(cY}Mu z1K@t}Bv=m8z(Zh(onD4~1Q_69@F-Xc9s?`E3Lxu#HCP4KfXBfT;3**WJx%+rwO}3i z7#ss1fTQ3YuopZFHi8XcJ&?Vk8f*fafi$ueYyocrDf1lI26liKz;^IFcoDn=c7oSI z#Vh=J*;edE{t3Jac7fNx9`I-I7I+iv0|&u=Z~(kvr$rtChruD>1Mh?Pz`LLw_z)z6 zzkrVbJy>D>t=I{RxK1L!1fPR1z*pcjhyZdPkh8TLDDZc5e*^zI3@-A1M*j`?8k`5` zz)#?N@GTIX4AFNo{ufC62rhyjfEfM*{1f~Fq=8St2_R0?0Mg~+ga?RDPCCcYeFAFR zI_Ja`YUjT~p4ildUuEkap?&l0B$dcnO)5Hzu0C=l@=nlzbVCpU{?+a{`J$7vqnsJ9 zJPS%a&Y4oqkWSrFM(P$`Ax4g8i%B~T$m!K7AbLqlo;1=FNOVrV+zB4XM(z@;kkSJW zBBfk2(#=7FEn6Vlr1Fd7xdLT0>ZsNXbDNZ3&UXl$f8}$lQ7P>^wokFnAEBIxoqr0H zoYeLy?bFPirvW+6zx}yY$M$Kd?bA9LVJRxs7?q+L$DO9ZtFSp2{pTOf^gg?W(rMJo zFkVo`#uSyR%8U~!D$SkkHZFElsqR5;qg{%MRuha~DQck6BU6PN>rz#g`?NB?OjQHi zl4bF_eG50TB zU%i%`&N$MMEy8Hhi5i|aQo101GIE3+G$vo6JYuYb#XA2aJgCE;*2KKAJAyhoS#=yV z?(S;Vp{~JBzJ(xNHnt?OTlczQoy7~QGuIml=_T2+d|Izt*SIf()|`KA9rAwlYby>9Yi`z+A$7H_YdkO>|FY^D zABm;&ue`m3r#l_I?Mxb$(yMHw`E`x@nOIJ(Yw#Z%6Vz;DG5-yr78=Q&Y49Oq;6#D1 z?==)o>T_}C@vIk3KbUdM@=)WgF7%J{g@@*O^BYZH-m#x)(Xo9eHerLYndqJxY7}0> zv@@qqR0tE?o3hPhBbiyyV;fI4r^#U z(4Q`kZfKnAt0FzLn$*y!*9}#thDMui1h?Nf-$TX3IiE7PcF#Y%_z#pUvO9yAi^Giv zD4-S@2^iW{M#{eitup@7jR>8uBOJ~+qXr#_?0>oF9;1Pe2tPCyUyC?ntnQ2W&KQj{ z!}-QSz3va6j-7M;a7xLYCrbX#sLvKjpFMN`;1@wDlOLe>_qe(`n&{?pR~)8GcwX>n$7%nzPqvZ>sfaWUz+f| zX_4B#3)LAp+4%X0F&Po-e9zK$daU=Oc1{MTp(x{TIN=VDG6wfiQ8m)pbgrB~Mr>b|=`M~kDtoKQ5WKdA zMuzs~9t@a0S*+P<4WGFJFOOj3Sm#SAbwBua$?)gumYA)ZTg|U$oK=Z4YY6*TPr`P- zpYm?ps~=sz^|dGL(k%a9we1u$MqaBLUz$%=15VzrXSi_@%Q)wAEIVeM&NOU>lg@>z@e@U@6%pm3QEM!s?JJ(^edXM zH#cXmsvo#y9x4AP=!9d{mi}3S^<|08bw8Xk>yGE%W`OPMhyQ7QyM4`#qCqTCdyf>K z4iAJoEm%wK4>l<$2-|rXg*5QI8d! zv%cK+hUvIwY#nHIiuh}f=w;jAKj+yI$NkR6!FtJVWvrr4;+&5rty)?qBmRYZ{%$*Y z#WJyn$=Sk;CfBJZ|9!OWQ^%!@%M;Z)r(DXoG(YSTe{e$nmvJ&@TsOAZG;4-MG&9eY z&UdEHZ0fc;Vq0z-GXnGa$3C|mHI5Co&gb2c=J_uDL>uc8{F5)+6-2%FgQieg`%ZFF zc0OH{{ExLCeBW-sY;roaPst$bf^kz0_0&i*X60}_vDrwyMKQB z0kbCaq>_8JaXg1#^T!{DrIAhli>`?w%>e;GvvNoevgu z`{vc^H|nQPqNz;tir>6htcSZH!&8U8S+i;nIqBBb^pI3z1!ZHGhgks)%Q-M8zy8*} zSaAI-SEQyB1Jz6;K9`eaD`pXB7{9 zZ;Pn&R10$G1#u#yi}5l2;O^bU7&Dyyc*|%oT&2Z1A9Xspc2U@S@70_}0sB78Hq7{q zYw0wrO6P-5_pN?5_orJA<(h7EY|qJ)o3V|+k!Us8`U>3(T^IxApXG4GRSZ~BlURUjOnv~9?`^dg4SHlhoNh zkvn<@EzfTZx{*6+U$(LAMy{v1B|3~$Um7(=Fx#9DNacK%Q*o_#yR2R5UA$jt@lIblp{rxYXQS1?IOp3`S5+N8d`-wQ z+0+;TS$_19JSuS8x9G;)G3yKJzr6K$OJp)0(=`pvu{3v5>#>Ye?hpFiJ=8cjffmAs zK0J{}U)E!lSm!%mbIweBb78~3^87|NRQ3?pG2^j`%tYq{V2>YoWcJF!KNqkhJ9DF% z(h-A07$iC$1oIYmef@OKqdPF*J~Tu2utMue;j4wlRg+Zr)KN981^0Z^lN*xnC`>7W z%cVg0iJLYI+v#gItE?=UFN99*c5LL@QLp$$71c0qo1}bPIqaIG;;#vGuolg}MbG|o z_u!nTM!@CV@mcw81@*`GI~Ih8!=Icy;_eB3vg6L#&8P1&8W*VBTd{!B+m}zgxp-P| zYW}o>WckEtS@*JnspE?B3raf(9a}uhSXIbHPs%J6XRIz%HyG0v?{uxQ1vl> OEL6*%Z&RcyuKzE}i)F_E delta 21613 zcmeI4d3;UR`v1>5oN_`+NK8RQDMdmOB!m;xm}81DtuY7*K}NGCsMbuns++2!Ep1f| zMF-SS)hkrnbe>xsE*yi{qi}_`aElTp0%E});{~l zKC`IOga5o{P3#TxAH1o}hZmntdVSlwzTk_uZF(d2nOqJr0_7q zyZW4pw;S04S(^r;kX4X9C?_4aM^-~-7L3ox$(WEiJ}0{)_&VZ6m$N2>8V8X%k|Kay{`hEoL>-(cWgouCe>5B2GPml-4XHc9a}O zSrdtE(`PbFgKId>>_soFY@}(a_cKy#4yTgnCr!_tfU%zEsEh$6RnSYjGvFd8PS20Y z%otysm24`krxnR)T*v9~TMCjanSi9vl0^_PC>$=+`vEag54p;8ybU65CPq5#%rYG| zgG(6ILW-fGCVvCvt9Xh%CG)5y9W0=N9Y=7P?*2x0GWm^>(qSaY*re5K=tqiIjkWz1O6G#6UhDyCx%!utmiesCN1E-sMJjK$%DiksT z1CU~QMtv4P+N&&XR`CqZiM1v~(K$04@#A&dABhE-3W0 zYvu5Q%&9Xnv8W-fN=G$}wA+ixoq9CJ8Oa7Z&e!awyw=XR)QEL@oldT7Fu7AE#Z1V~ z_oTIP^3Ncpw{iIy<1-69p6+puei>4R?Omkgy9-MeRPADwU|KCIoRB%K&@(l2y6jw@ z=53wQkw_Vw{7gIB^A%idPHyM4{*lSAj+8-i>*Z%=Oo+imm#=B>ln*32(G!hqz=W1$ z=I6`Wnw5~`G*I0XcnL1ya1<%CI;VqU(A9RKugDLl+|{Pr$5M&mj=9SugxEPjYyJROP@o}XDTJ!eLy=WM{^iGsg@6a&kUGQRGt7T_tKm3&S| zV)zu<$fi&9ob2MX$8yS^INRfSm;7sGL7gEX9Zb#0jhQ}fvge3oz!!FNJa+ZFO}VV1 zsTtE^vL{R|VAu-tGq&I8G(5Rr`m~AJIhhrh2WhWxYOc(r=h5y?JB3IwxI+)8JR_MO zJ0Y{slQ}J~C_6th|0i-~4##K5WKWwo-SY)p=Kgi0jL@Z?`kPAE7q9N+w6~(SQ=A=R zQqJ^oo+^Eu_UEA&$0ueK6z0)?er8dDc)W)En&{I~o&Gl>#m*d4o?x~c$__g1-%3AX zj_c=`A6NHOfd0LPF^NNIocKyi?ex!x4~1cfxX)>KGG)`y3i{3K)}_}duD zZdq~>Sr>T-*$}yYuv32pQU-c5Qeu>9_Dr1|;uuh6s3UF1M^IoP3W?3#cQ;Q~%W9>lY1&^q#k;GbPOTmAeo#>_s~zwMv4}NT za*g!nFEz~W+kXZ&%x)trD$#%4H69PEv81vVmH|tJd39LrMDHWl=#aVr{{@u4nQ#O5 zI`MV|X?g;zBdn5C_dZ-nhtv!Bze6d$dZl;OMK7(FqUPzcdMW;$m7U5J#T)P6E9=zA zfIpt8mG&!28>&E;MW*=16U80%%G$}kZKT@kel?T5?^e;v>IZz`%woJ=SwGo7ij;JY zwX}Aks!nYX@V6!^x}vP1OKT-nhR8T5ofets-5;t$8V0;yh3eFX0e=+hpo?7_D}7^x z>CmWT@1|>YNL0Xk;aZ&<74UZ^s%3m^ntuCYgWIZSM3X1XL*grtfqs+wLJouc~avgj0FISaDCgo8hdrNtrPIOrP- z>!#0yCwuR!p-(jqcy~tVkm~}z%C$V6o_6{Eq{LX4rv^u%W|Mz zSufeQoYWBAuW7RXXHp|Cml#^l80I@jYLK+y>%wgY6I(yoJ10^)@$Ey=TPpj0A(blr z`FgS=*oAy0qyl!u!=(D^mC?y6MEAKN#W#QhW0($Y)2X7JAV7Q18bg{>VmdJsHjnSd#7UdQy&| zACqdYL+d81y1J}YivJe2EVF)Ct?$AH+KrahPE@V*(imB-Wictfy^V=WXG}@4Wtlm- zk*_*z?K0s5lPW^G=reJhDw7cVE9lbtiM}vy{<_%%HilFe84}+nQk`T}eeaMOA}y=d zx=))F-{Ph+2WMI&``;qf-R>!@Zlb@z_0C)pqYV@N*|0%Q9y5DTm;{b5w3$6QHIw~A zNsYEk$XM)!rQ0Q{G`Ck{OtR{%%i>e~8{v)z#~Ft@7f!5 zNZWw_wHuu1^Xl>ziN0zr>}h5tjhB?|Yg*Gp{}#j8x!9e&EuD2m{9!{|VX|WV6G_SV z*sI+C6s+52UoOH3rI=)Ypq0Bu*?WA2FniDSA0#DzOOL(6ua9+nsbH`0v9LDeRkU~ddt>#fq=4Vs+UeX@swCZ~LyCVcx=s|d zm!tQI*7{V3fcJ;iI;3O3KY|w8JB_e~l)|JWwHcJxVX}+~5~i*?_o!li1$!dk1p?pADvVD zC1}K9>e6Q%#w5eYk4g0Bz+^HQnhr^oA&#l+AO4FZWu)y<^Yv&a5!^4nQ)LoT!`X0; z!^Eyi_EDnr(v%c`GG;s0vAz24hIQ2aB9i^Dk`mwX7X1Kjic@uHlVtyTQsdCuu zOVVee2?!dIwz0HDA_vsct|`8^(P2i-WK~Q1yQTOSb#TVaw#NH(2feIYz+Zt;G{f3C z(ccRu<~Xxo3>yd|Mp<=c(iI-R!mwxS7Rk!G}V2bZ?boNsBe%)QC z_6_)Z;I%Zs-V>gv7VEOUj3zqi0rMLq`oD$Aym6nxc*p6b{ZiBA zJN#~-J~cex>y&0k3+B!sB@5DttWz*4>}*#dgSf$P4j)pZ(K|cmvoPt(IT&j2wO7yG**WWOZID05=2f&mBYj*2BBfkYAzW9;VvjV` z97xBU?e_JbNU6`+Yq!%HDebf|l7q{>u9RXJ2ajE@t(-V=wFQj;qh?z=0x1oRG;$PD zE|J%O@t}(CH>!3q2~VzkaQ&H-Zu9L@j+E95fXG54i;!}O6t`v>ITtCHNGU%LR0JhF z)Ug!{Ad;~V$R#os>;^J1`+?*?4&)Lk{0SiWhk#sHO38oP);dzkJp;u3lR)yz=-*D9 zf{4XuOvVdHxkQSkF9T`d4ImZYH1aJY-$u#^y=(Y;NV!BxhaUr}_le=3Bc;8sWc;P$ zZ;bpNDFuHp{AZ+GBBk6fKn#*X!Yd%9V;@rVeyMDv@KB=}j`XcQu);vmESOp0N{O!;)A@R6q8l~URp zZS*1~ImU2X%KS@03TBuB;|!l*3W}6urjZkkoFqyvky1a~=qDRF#mF3_Tvy5}@R{hP ze>>qyX=t{oILB1HQp!-yH+qrc`2xd53SVsal~Po<8NEp9XNi)Z9h`(+afK-$vVwju zBkBfGl=30+4*B>ODc#+L-m4pob2hWPg`vI2lolyQZ!-K!DXPszf2EZ2_Z$76N$F>+ zDTln-?&u+-_%kUTJYvd;l;jRRM8C`M-J;|YsdVW0+Qrh{UZedZDLc(mZ|sJJ3n;rAV2Pr%ZW~(%zWF@5NtIGJxkk&v-otj9w)o6s2>yML;lb2XQ<5e$G0<4|Gl4ahR43Ik!X1a zNJReke&)aTGylDx`S1OVoEDqq<6org$$xeqBjqJu4y6Cy&&X)}|8PGukm>$^dOxFo zo?6(Y$LBjIp6g#SxbEJL2f}Xq!q;#3ugeF%(tcjVup8E|II;fuR@X%&+;U5sKDQqH z)mnJVrL?Zwf4!&PiM$g1`rwDV);ao^eso$LT{Jx?zw+h|*ICm;^@(Xg<mtNkCLC(dCI;k*Je-EpvyA}=C=V4omf+|8^fUPSE)k9|lRc*ax#&F$xMyU49 z4Dy>-+RWklC)j>iq*k+r>xX8B>daX|)j;oorOyi05wn9TN@vU-t}D;R2UxTYn=@QL z2`ictRM+Wau&g=wFgM79=e)W2H#bzj2ID{6%)>v};(0-JgD!{7numY$gQ}%oFdzTs z;~y+W$K8T|u(h`Y`R(g0Y{@P7R~+Q|ZD}$772_W)L3b^|KiJliAir^5fUPURzXd^+ zsJAS@zXkZWFsM4{w1xNw+Yd|DY7zb|#J@#Be(l-=OJ9V4w+2;;&bSr-ZpA-XR~@z( z|6oOngX%_o43@PR|85Ja9y;$f{JRbRV7+wo?f3^smf+tK z{Dbw=aZB+JwsvVy4bW#{OP1o_vLFwHOPAr_GW>%L)?Jt5A8hOLpc<+#z}79tzZF3> zTyI%{e=G2BWsv`2khT*4VEbVsv|5FKEAektP>s@iVCk#yZ*@?O(HX1pZ#DkGGIZD) z{DT#(399k>7%XcI{*?w*rp_zHzf$~zP14bK;2&)99YK|?%VD$bz&{;SQ}hCje;WT_ zQ+3>(_y=2iXHZSoXJJe3#J{ycm8VPB;@?{QgB9qmci|sw>s>)rq%XkM-GzTvP|eg^ zEc~PuE!E|)SsU^1-k@5p7u<_~_u?OH zrH$Lmv54In+L8}Mw?|%GyAgDI#J+Sl#@Na8SZPFQA@oy{s!M5nIZTJT( z+7?vz>tnF2ZTRDXM?Zvru*DAr)x)|RHtQk$dpM{b(F-2NzlZS; z_Nb2Aj(@PV+ku?%LG_eQ+lha${jh^t?ZUsE__r&lp4NL{>AUc6cTgSC8N2asH~ztn>99Tc z2P@hWRLAu(Sk@l=dn~9<=)A}9?=k#?{Y6Lb#Xs2My+Ku`%VD$j;@`fYI;|J%!@qs_ z2RozV_TwLH?f#&8QJ;k^;Xjo0cs!`eb?M{y_c;E+UeR5jz(3g5CxYrVeF3)a3H*C9 zsNT?9p2WW=@$adidP}E0g@3U9uy?dNfPYWn-+`cdPw#=HAHctZLG^*oIEa4-@eg)Z zhaJK{Ska-N`dA->WgWu5r-SNKo%b~UJ&k{`b2|Dk{=pU>4yw;|Ic(Np{5ujj^ZC|?a`q6N}q)-If{SBg6e`UJ%)eB@DKK_?)nV=!L~jVRNw0huyxPi z-|?XOQExepf5-9f*`WGar#*{*u>G)$TAjeZXYubuP+ih{VCg6D@42A&pB_E#xna8U zb2#{1kpBP`_7@z475zmHz+VCNeE>d>gU{n2tfuaI1_xnV&)5ebY~2|ge8D~d zU%e+dW6?E|o!2`tAySoGeomusuN17G<> zHJI#o{-fHs>}j!+`6WT_Jp(+JL|xKLr@hw8@Ba7jo_n{vrYre=Y$3l{gq@pvN6m`* z`Hw35cJi1Z)F?tG7sSlYn405R6T7$n`|T_GZs!L#DV^e$K9|COl-nD2Zh&`h?q_c* z)FJMF`g^Y5-fPc2&>>`OlymRgy|HyPTZLQk(W(UxoqqkNbIDy{2qmOYHz8cY#c}~Z zY}r>;Q%3H(uQfV(4lniO$=7~Ka9xX(GV)D+n$d+JB`No1d6JNaUNwwTp6%p|lB*_C zI+Z73@~e-eyX)!bsSu6$R zfs;H076a-dMJJxkl?0dkC?LsdV64$KG&*@GCBI*Zp)$Ub9R}i!u94BzfR6)Wy!^Nz zM%5JmrLtJs#1srCEtN&r)aW8e%db~r{q;sy%jm@TW=1F8%42f5nj2jm((>qCjF;!l zl3f?bk8#yy{972MJgXiL#L|{V7fCwP=vo~ zYg0Fh^h_XqwGo}ne{HjHv{Ggsz0D1cpqo zOrAuVJo0P|1${uOq{$lwWJTNrWEIG$$*{|#SoyhSJdo$a=Yc#| zmZ#^3!4Ys290SjQgWwQY16Iou@#ZwL8?pUBCkKfvok}Kvu{=kOt(jQh(4J%%lDJ;1(cpnI#|DUp>9O6mW(z zvM^<_w*of++5W16%HRv~Uj*eM;Y+|W@G|@r@G95`e;f=UExXTa5;d=*cmq5D?f|m4 z&I7X5#e;D`7OBf+@e4{QlL2HwCji-vWcSGdt}dUnU^3;TP8%RQilk-3Xd^Kz1!SiX zDHR1$P%x_*HMy#`v3fR9*B4hI-2jzDn5=QhmGv&`Ue>>y6cvCM$U!N*Dv*6Z_Jcnw z8%4f^vg{>NNBWYzMe52P%o;7WhfOx~Fi;&-1G3?VgBn2gRLPUH?3=Ppx?C2z?7gCE z1fqc#59GM34eEdh5C&?2a1aV)ife+(vH@2j;R7o^RxgmRT5A#GBSO! zK)j9tGS4@FmO#qJgE$~DBZjpD?g%B4zH-FGkltW42!P^bK01NUKz4w+pdXMzbHHpM zu@nS7z$hRAR|E>dji3PJgKi)dbO&95w3`d2gI-`7mlz+^B6WP^!d z0+5xG0mg!nU<8m2UN-vSU;sz~uE9M?ixFLb7$C-p(c;{dMu{{F z+m$GDWgXFp(NbU1GPg2Kq7#Q?O2&XI^W1c?1c4NG8yRoXU5pg3#e+=4rEzI^5^zT= zi?np&HY(-a4n^l0CT&SwiHkfyUa_r^44HpvTs)IXQc(16qhe?lkhz-yWN5npDI-!E zb?Zr>T`fJ&={vQ2%t}#mYWU68c$WTim zKRm?fB!EOOjmW7c4SfN>7AXcU2GvMc2Q|RImXr2GC#R#FjK$6|_*+BnQ7K2H+qhJc z#$E0jw%Ro2)^YPiFKOvWI+437an8+;BrR>$Q=DxNk&qEsgOrN$&9V`YZ<8XUL68{tVWKrRK_z%+daT_EDn3$-tj!|8k6WXe_($d{+RbOf8JW{o{(mJVH72C0&SOeOr zx@wpe)n3)Iirc9MYMiwZRiyie_+L)AdFXqcj&{eqBpe|)T3F>$r>nJvIt_8m{Tu$9 z)jjX@{Bct*McYx79J^h&XcP+fFX4y2@@11b)2rV^(L|@!dn;Mf+N(O4k`i!Ruj@Tm z$vT9#zWZnW<>}jB>hs;EPrd3E8LYN!K9#JmDdGLFvK5l3;=Mclca2kZ+Pir>&jC*z~3*4C+3XvJlzy7k?^!))}zv+XW4JMaVr7y}uwwISB3OiX+r#2T4|+!tc~ zeLV6+h_y!e>mk=tpq!#O)P=4uc~!VvfEBw>xE?1 z-}_Tlt6?Vs)ef(^%89K=_tgk>&OUVHuSXt&T5j|jO|!mHNvVw7IrQ-t}SD*EgyN9}DijFl*od6%j>a?kg9TpS-2Y8)x3X=!^wp=s6r_y&J&0!B({t zRkz()_U$?>-|i=V>5y_dZe>TQY(1SyBdcm!v0VuC%{A?>AtisC@X2RC zEScNaF<2I#`znIb{XZQ1Sz7b1c1~PudwE}kHMS=K=f1R{TE%-m8WUf?#B|Asd{)!i z-G$ZkZB45cB~=ycNOyJ&+lNQ`s=8L2uBx7jvVvW)+KHU0}x{>ZXCUU#Xd*oeb})0#?EetWhL{1RV{!Y4Ox=;_7=bK<^Y-Vwd)bN^kQH`zDQf85LAytK6V{gB|YuYVnRqx3Hb7RM8 z(#`vQ57rJ!)^}esa&Xq?X>aztlwy3xsw)=hhI-a75{i!9-fi`)Hoa7xcJ8Z2KF`?c z>s+|)wcm!^eHlrWkCHzR>e5g%9<>;n? z#ghKL2}dWV?-h%~iR5-y_s4ycOZtwH-$dsOdezv(U+n#EZ}|5%uiDw=-Nejd7gHM&WzFV9A zc$GX8IP9_VhM8+Qi0b!TFys82UO8Xoj*~sdj_O)+z?l&@>)31^R{22HKpcoa zx^Kvdx%kjqKep;Sn*rjqXiu*tR@h)#TG`xcHJFpVpEU;Bo;T*m)4!6#gR+l%e|=>Q z<>feI7~NOneEsbF+zBdAO*<=Mi0T*VzV)WTv!7@F zt)Q5DLb(X-8biKNIl#C;gU&m|3!C~Z*A&wt9N9yinVGERZ*&e z6*`9Vx9?EZR^?d>hO#EotkQ|7_6=42Bi;Au3?2LBKmV~N;|JPed?gyXTMdSBc)2gu zshA!=jwiz z|GIdb+^v|q-Q5Y+--mGoyDx;vsd)VL4|X1!Mv3-##W1?Bj@e&+?$UE(&$lN>ZsysT z-%YSO4d;M&-$K)Ao2PvDxt^tzkehAl{v*N4rG%<(EgjCiM5OifaHiOOU(c}`$={FZ zd&gD-IBufzFf z-lJWPqrBzf6=@5V>}e9Uf2 zHcL;y8g&!#=DtJdhiP{$3=i3Moo&3_ae9t&&rzL0K9bk$oiB@1!^ugEjc?01yp&|^ zxQSs%w@%PUefKp(=O0>D?e*8Mk#At;%ZVu{H^Q+uGqUarhgNNQZ0M!=FAk-o>=xWQ z@daw!&3rF)-=sD!uSw@FRjMV~B|3<`?X6umtCrl?zjL#SRzs~zH)Fl?Mj}tihOcWr za{S%(a>~kxV*Q6aCE@!+$($}NKT3!hQpM!Z(3hR8z9Ser_oYRZQgW6ZXmt3BZ!6=h z;t{M&Q;;tOHZ2QSFHo<(`x2u?yE9*G(P6+-#t8=cY{2r4WF4ODVhtS0{VQKdZbd5l z%LK>!NLJ87>x+>@v%RQlS&^eugGl#fM8CX|>g_S)g(Wm-qPd1OcocC~Xx)zRKHA+n zGztUUml6FO+IW9Nm(5>$RZI`ZS?8OY`~AwtFzobOw`6b}@vSc~8u>*}D}A);8+oax zvu`izxoXT)2dg>viuNk7E$e4iy1LbBHioWO_p(NgQ4=HG_YN(eaOa*wZ~rTQ82oX7jDu@{U(om&vW;e{0D3KJR9^ z-?}5+_a!~B`h{m6_O5o~o&87#gMGDu72i?S@?IKXzxjU#{9zx%)<^6HNzoW~K~t35?9PY-_oxZ+QqMz~KO zoO&isE)SnOEY|xI`9}Xc%gvLJe;q(;=Ok|L+*VBu$L2`))ldh9w>V#TsbdX$Ozdsc znbphHj1%4!gRPP**6sSi*1jyBR5?#KBHdR?&HZxHnI$#8Xe{Tby;S&KH6WW)&3)I@ z{ioN>-Zc5;Y@QD##kPx$YvXZWyH(oy=9Sm)@7%$wJ$3B^A)@%eCw1E%P02gTEC0@ z%*|Eph;?eRx^)=KIWD$v(wr^@+0(}7WMt>JmX~Udn?5UZ25-B|oSI$8d%zOpO=U5> z5p(*4%$TD5?3nS>r{+$dMgcn`uP8Hrc0pl&_B7rSnK>;ZH`|Iks-ms#IqF6$d#dVf eJv2q#Z0(t=3.5.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.6.2.post1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.27.2)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.8.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.10.19)\n", + "Requirement already satisfied: sniffio in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.3.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.12.2)\n", + "Requirement already satisfied: idna>=2.8 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anyio<5,>=3.5.0->anthropic) (3.10)\n", + "Requirement already satisfied: certifi in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (2024.8.30)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->anthropic) (0.14.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "17325.82s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: tenacity in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (9.0.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "17331.47s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: apollo in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (4.2.13)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (2.32.3)\n", + "Requirement already satisfied: biopython>=1.77 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.84)\n", + "Requirement already satisfied: cachetools<4 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (3.1.1)\n", + "Requirement already satisfied: click>=6.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (8.1.7)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.17.0)\n", + "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (6.0.2)\n", + "Requirement already satisfied: decorator in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (5.1.1)\n", + "Requirement already satisfied: bcbio-gff>=0.6.6 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (0.7.1)\n", + "Requirement already satisfied: six in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from bcbio-gff>=0.6.6->apollo) (1.16.0)\n", + "Requirement already satisfied: numpy in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from biopython>=1.77->apollo) (2.1.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2024.8.30)\n" + ] + } + ], + "source": [ + "!pip install python-dotenv\n", + "!pip install anthropic\n", + "!pip install tenacity\n", + "!pip install apollo\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "import anthropic\n", + "\n", + "load_dotenv()\n", + "ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\")\n", + "\n", + "client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample user prompt\n", + "user_prompt = \"\"\"\n", + "Generate a job that iterates over a patients' list using openFn functions \n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Here's an example of a job that iterates over a patients' list using openFn functions:\n", + "\n", + "```js\n", + "alterState(state => {\n", + " const patients = state.data.patients; // Assuming patients data is in state\n", + " \n", + " return each(\n", + " 'patients[*]',\n", + " alterState(state => {\n", + " const patient = state.data; // Current patient in the iteration\n", + " \n", + " return upsert('Patient__c', 'External_ID__c', {\n", + " External_ID__c: patient.id,\n", + " First_Name__c: patient.firstName,\n", + " Last_Name__c: patient.lastName,\n", + " Date_Of_Birth__c: patient.birthDate,\n", + " Gender__c: patient.gender,\n", + " Phone__c: patient.phone,\n", + " Email__c: patient.email,\n", + " Address__c: patient.address,\n", + " Status__c: patient.status\n", + " });\n", + " })\n", + " )(state);\n", + "});\n", + "```\n", + "\n", + "Here's another example that includes additional operations and error handling:\n", + "\n", + "```js\n", + "alterState(state => {\n", + " const patients = state.data.patients;\n", + " \n", + " return each(\n", + " 'patients[*]',\n", + " compose(\n", + " alterState(state => {\n", + " const patient = state.data;\n", + " console.log(`Processing patient: ${patient.id}`);\n", + " \n", + " // Data transformation\n", + " const transformedPatient = {\n", + " External_ID__c: patient.id,\n", + " First_Name__c: patient.firstName,\n", + " Last_Name__c: patient.lastName,\n", + " Date_Of_Birth__c: patient.birthDate,\n", + " Gender__c: patient.gender,\n", + " Phone__c: patient.phone,\n", + " Email__c: patient.email,\n", + " Address__c: patient.address,\n", + " Status__c: patient.status,\n", + " Last_Visit_Date__c: patient.lastVisit,\n", + " Medical_History__c: patient.medicalHistory\n", + " };\n", + " \n", + " return { ...state, transformedPatient };\n", + " }),\n", + " \n", + " // Validate required fields\n", + " alterState(state => {\n", + " const { transformedPatient } = state;\n", + " const requiredFields = ['External_ID__c', 'First_Name__c', 'Last_Name__c'];\n", + " \n", + " const missingFields = requiredFields.filter(field => !transformedPatient[field]);\n", + " \n", + " if (missingFields.length > 0) {\n", + " console.log(`Warning: Missing required fields for patient ${transformedPatient.External_ID__c}: ${missingFields.join(', ')}`);\n", + " }\n", + " \n", + " return state;\n", + " }),\n", + " \n", + " // Perform upsert operation\n", + " alterState(state => {\n", + " const { transformedPatient } = state;\n", + " \n", + " return upsert('Patient__c', 'External_ID__c', transformedPatient)\n", + " .then(result => {\n", + " console.log(`Successfully processed patient ${transformedPatient.External_ID__c}`);\n", + " return state;\n", + " })\n", + " .catch(error => {\n", + " console.error(`Error processing patient ${transformedPatient.External_ID__c}:`, error);\n", + " return state;\n", + " });\n", + " }),\n", + " \n", + " // Create related records\n", + " alterState(state => {\n", + " const patient = state.data;\n", + " \n", + " if (patient.appointments) {\n", + " return each(\n", + " 'appointments[*]',\n", + " upsert('Appointment__c', 'External_ID__c', {\n", + " External_ID__c: appointment => appointment.id,\n", + " Patient__c: patient.id,\n", + " Date__c: appointment => appointment.date,\n", + " Type__c: appointment => appointment.type,\n", + " Status__c: appointment => appointment.status\n", + " })\n", + " )(state);\n", + " }\n", + " return state;\n", + " })\n", + " )\n", + " )(state);\n", + "});\n", + "```\n", + "\n", + "This expanded example includes:\n", + "\n", + "1. Initial data transformation\n", + "2. Validation of required fields\n", + "3. Main upsert operation for patient records\n", + "4. Error handling and logging\n", + "5. Creation of related records (appointments)\n", + "6. Use of compose to chain multiple operations\n", + "\n", + "You can also add more complex operations:\n", + "\n", + "```js\n", + "alterState(\n" + ] + } + ], + "source": [ + "# Generate job code\n", + "\n", + "message = client.messages.create(\n", + " model=\"claude-3-5-sonnet-20241022\", # TODO change to cheaper model\n", + " max_tokens=1000,\n", + " temperature=0,\n", + " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": user_prompt\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + ")\n", + "print(message.content[0].text)" + ] + }, + { + "cell_type": "code", + "execution_count": 288, + "metadata": {}, + "outputs": [], + "source": [ + "# New edited system prompt\n", + "\n", + "system_prompt_role_v2 =\"\"\"\n", + "You are a software engineer helping a non-expert user write a job for OpenFn,\n", + "the world's leading digital public good for workflow automation.\n", + "\n", + "Where reasonable, assume questions are related to workflow automation, \n", + "professional platforms or programming. You may provide general information around these topics, \n", + "e.g. general programming assistance unrelated to job writing.\n", + "If a question is entirely irrelevant, do not answer it.\n", + "\n", + "Your responses short be short, accurate and friendly unless otherwise instructed.\n", + "\n", + "Do not thank the user or be obsequious.\n", + "\n", + "Only follow the instructios given in the context\n", + "\n", + "Address the user directly.\n", + "\n", + "The `each()` function iterates over an array of items and invokes an operation upon each one, where the state object is scoped so that state.data is the item under iteration. The rest of the state object is untouched and can be referenced as usual. You can pass an array directly: `each(['1234'])`, or use lazy state:`each($.patient)` or a JSONPath string: `each('$.patient[*]')` to reference a slice of state.\n", + "\n", + "Ensure that `fn()` and `each()` are not nested within each other. For example:\n", + "\n", + "```\n", + "each('$.data[*]', operation); // CORRECT\n", + "each('$.data[*]', fn(state => { /* ... */ })); // INCORRECT\n", + "\n", + "```\n", + " \n", + "\n", + "Only adaptor-provided operations are allowed as the second argument to `each()`. For example:\n", + "\n", + "```\n", + "each('$.data[*]', appendValues({ spreadsheetId: '123', range: 'Sheet1!A1' }));\n", + "\n", + "```\n", + " \n", + "\n", + "Adaptor functions like `appendValues`, `getValues`, etc., must be used directly as part of an operation. Avoid wrapping them with `fn()` or any custom state functions unless explicitly required by the job logic.\n", + "\n", + "Use `fn()` only when custom state manipulation is required.\n", + "\n", + "Avoid wrapping adaptor operations inside `fn()` unless explicitly necessary\n", + "\n", + "\n", + "\n", + "DO THIS:\n", + "```\n", + "each('$.form.responses[*]', appendValues({\n", + " spreadsheetId: '123',\n", + " range: 'Sheet1!A1',\n", + " values: [['Name', 'Response'], ['Alice', 'Yes']],\n", + "}));\n", + "\n", + "```\n", + "\n", + "\n", + "\n", + "DO NOT DO THIS:\n", + "```\n", + "each('$.form.responses[*]', fn(state => {\n", + " // Avoid fn() inside each\n", + " return appendValues({\n", + " spreadsheetId: '123',\n", + " range: 'Sheet1!A1',\n", + " values: [['Name', 'Response'], ['Alice', 'Yes']],\n", + " })(state);\n", + "}));\n", + "\n", + "```\n", + "\n", + "\n", + "\n", + "Additional context is attached.\n", + "\"\"\"\n", + "\n", + "# formatted like in apollo\n", + "system_prompt_v2 = [{'type': 'text', 'text': system_prompt_role_v2}, {'type': 'text', 'text': \"\\nAn OpenFn Job is written in a DSL which is very similar to Javascript.\\n\\nJob code does not use import statements or async/await.\\n\\nJob code must only contain function calls at the top level.\\n\\nEach job is associated with an adaptor, which provides functions for the job.\\nAll jobs have the fn() and each() function, which are very important.\\n\\nDO NOT use the `alterState()` function. Use `fn()` instead.\\n\\nThe `fn()` and `each()` operations SHOULD NOT be nested within each other.\\n\\nThe adaptor API may be attached.\\n\\nThe functions provided by an adaptor are called Operations.\\n\\nAn Operation is a factory function which returns a function that takes state and returns state, like this:\\n```\\nconst myOperation = (arg) => (state) => { /* do something with arg and state */ return state; }\\n```\\n\\n\\nHere's how we issue a GET request with the http adaptor:\\n```\\nget('/patients');\\n```\\nThe first argument to get is the path to request from (the configuration will tell\\nthe adaptor what base url to use). In this case we're passing a static string,\\nbut we can also pass a value from state:\\n```\\nget(state => state.endpoint);\\n```\\n\\n\\nExample job code with the HTTP adaptor:\\n```\\nget('/patients');\\nfn(state => {\\n const patients = state.data.map(p => {\\n return { ...p, enrolled: true }\\n });\\n\\n return { ...state, data: { patients } };\\n})\\npost('/patients', dataValue('patients'));\\n\\n\\n```\\nExample job code with the Salesforce adaptor:\\n```\\neach(\\n '$.form.participants[*]',\\n upsert('Person__c', 'Participant_PID__c', state => ({\\n Participant_PID__c: state.pid,\\n First_Name__c: state.participant_first_name,\\n Surname__c: state.participant_surname,\\n }))\\n);\\n```\\n\\n\\nExample job code with the ODK adaptor:\\n```\\ncreate(\\n 'ODK_Submission__c',\\n fields(\\n field('Site_School_ID_Number__c', dataValue('school')),\\n field('Date_Completed__c', dataValue('date')),\\n field('comments__c', dataValue('comments')),\\n field('ODK_Key__c', dataValue('*meta-instance-id*'))\\n )\\n);\\n```\\n\\n\\n\"}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': 'The user is using the OpenFn @openfn/language-googlesheets@3.0.5 adaptor. Use functions provided by its API.Typescript definitions for doc @openfn/language-common/**\\n * Execute a sequence of operations.\\n * Main outer API for executing expressions.\\n * @public\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Promise}\\n */\\nexport function execute(...operations: Operations): Promise;\\n/**\\n * alias for \"fn()\"\\n * @function\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function alterState(func: Function): Operation;\\n/**\\n * Creates a custom step (or operation) for more flexible job writing.\\n * @public\\n * @function\\n * @example\\n * fn(state => {\\n * // do some things to state\\n * return state;\\n * });\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function fn(func: Function): Operation;\\n/**\\n * A custom operation that will only execute the function if the condition returns true\\n * @public\\n * @function\\n * @example\\n * fnIf((state) => state?.data?.name, get(\"https://example.com\"));\\n * @param {Boolean} condition - The condition that returns true\\n * @param {Operation} operation - The operation needed to be executed.\\n * @returns {Operation}\\n */\\nexport function fnIf(condition: boolean, operation: Operation): Operation;\\n/**\\n * Picks out a single value from a JSON object.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * jsonValue({ a:1 }, \\'a\\')\\n * @param {object} obj - A valid JSON object.\\n * @param {String} path - JSONPath referencing a point in given JSON object.\\n * @returns {Operation}\\n */\\nexport function jsonValue(obj: object, path: string): Operation;\\n/**\\n * Picks out a single value from source data.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * sourceValue(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Operation}\\n */\\nexport function sourceValue(path: string): Operation;\\n/**\\n * Picks out a value from source data.\\n * Will return whatever JSONPath returns, which will always be an array.\\n * If you need a single value use `sourceValue` instead.\\n * @public\\n * @function\\n * @example\\n * source(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Array.}\\n */\\nexport function source(path: string): Array;\\n/**\\n * Ensures a path points at the data.\\n * @public\\n * @function\\n * @example\\n * dataPath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `data`.\\n * @returns {string}\\n */\\nexport function dataPath(path: string): string;\\n/**\\n * Picks out a single value from the source data object—usually `state.data`.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * dataValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `data`.\\n * @returns {Operation}\\n */\\nexport function dataValue(path: string): Operation;\\n/**\\n * Ensures a path points at references.\\n * @public\\n * @function\\n * @example\\n * referencePath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `references`.\\n * @returns {string}\\n */\\nexport function referencePath(path: string): string;\\n/**\\n * Picks out the last reference value from source data.\\n * @public\\n * @function\\n * @example\\n * lastReferenceValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `references`.\\n * @returns {Operation}\\n */\\nexport function lastReferenceValue(path: string): Operation;\\n/**\\n * Simple switcher allowing other expressions to use either a JSONPath or\\n * object literals as a data source.\\n * - JSONPath referencing a point in `state`\\n * - Object Literal of the data itself.\\n * - Function to be called with state.\\n * @public\\n * @function\\n * @example\\n * asData(\\'$.key\\'| key | callback)\\n * @param {String|object|function} data\\n * @param {object} state - The current state.\\n * @returns {array}\\n */\\nexport function asData(data: string | object | Function, state: object): any[];\\n/**\\n * Iterates over an array of items and invokes an operation upon each one, where the state\\n * object is _scoped_ so that state.data is the item under iteration.\\n * The rest of the state object is untouched and can be referenced as usual.\\n * You can pass an array directly, or use lazy state or a JSONPath string to\\n * reference a slice of state.\\n * @public\\n * @function\\n * @example Using lazy state ($) to iterate over items in state.data and pass each into an \"insert\" operation\\n * each(\\n * $.data,\\n * // Inside the callback operation, `$.data` is scoped to the item under iteration\\n * insert(\"patient\", {\\n * patient_name: $.data.properties.case_name,\\n * patient_id: $.data.case_id,\\n * })\\n * );\\n * @example Iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * $.data,\\n * insert(\"patient\", (state) => ({\\n * patient_id: state.data.case_id,\\n * ...state.data\\n * }))\\n * );\\n * @example Using JSON path to iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * \"$.data[*]\",\\n * insert(\"patient\", (state) => ({\\n * patient_name: state.data.properties.case_name,\\n * patient_id: state.data.case_id,\\n * }))\\n * );\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n/**\\n * Combines two operations into one\\n * @public\\n * @function\\n * @example\\n * combine(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function combine(...operations: Operations): Operation;\\n/**\\n * Adds data from a target object\\n * @public\\n * @function\\n * @example\\n * join(\\'$.key\\',\\'$.data\\',\\'newKey\\')\\n * @param {String} targetPath - Target path\\n * @param {String} sourcePath - Source path\\n * @param {String} targetKey - Target Key\\n * @returns {Operation}\\n */\\nexport function join(targetPath: string, sourcePath: string, targetKey: string): Operation;\\n/**\\n * Recursively resolves objects that have resolvable values (functions).\\n * @public\\n * @function\\n * @param {object} value - data\\n * @param {Function} [skipFilter] - a function which returns true if a value should be skipped\\n * @returns {Operation}\\n */\\nexport function expandReferences(value: object, skipFilter?: Function): Operation;\\n/**\\n * Returns a key, value pair in an array.\\n * @public\\n * @function\\n * @example\\n * field(\\'destination_field_name__c\\', \\'value\\')\\n * @param {string} key - Name of the field\\n * @param {Value} value - The value itself or a sourceable operation.\\n * @returns {Field}\\n */\\nexport function field(key: string, value: Value): Field;\\n/**\\n * Zips key value pairs into an object.\\n * @public\\n * @function\\n * @example\\n * fields(list_of_fields)\\n * @param {Fields} fields - a list of fields\\n * @returns {Object}\\n */\\nexport function fields(...fields: Fields): any;\\n/**\\n * Merges fields into each item in an array.\\n * @public\\n * @example\\n * merge(\\n * \"$.books[*]\",\\n * fields(\\n * field( \"publisher\", sourceValue(\"$.publisher\") )\\n * )\\n * )\\n * @function\\n * @public\\n * @param {DataSource} dataSource\\n * @param {Object} fields - Group of fields to merge in.\\n * @returns {DataSource}\\n */\\nexport function merge(dataSource: DataSource, fields: any): DataSource;\\n/**\\n * Groups an array of objects by a specified key path.\\n * @public\\n * @example\\n * const users = [\\n * { name: \\'Alice\\', age: 25, city: \\'New York\\' },\\n * { name: \\'Bob\\', age: 30, city: \\'San Francisco\\' },\\n * { name: \\'Charlie\\', age: 25, city: \\'New York\\' },\\n * { name: \\'David\\', age: 30, city: \\'San Francisco\\' }\\n * ];\\n * group(users, \\'city\\');\\n * // state is { data: { \\'New York\\': [/Alice, Charlie/], \\'San Francisco\\': [ /Bob, David / ] }\\n * @function\\n * @public\\n * @param {Object[]} arrayOfObjects - The array of objects to be grouped.\\n * @param {string} keyPath - The key path to group by.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function group(arrayOfObjects: any[], keyPath: string, callback?: Function): Operation;\\n/**\\n * Returns the index of the current array being iterated.\\n * To be used with `each` as a data source.\\n * @public\\n * @function\\n * @example\\n * index()\\n * @returns {DataSource}\\n */\\nexport function index(): DataSource;\\n/**\\n * Turns an array into a string, separated by X.\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", function(state) {\\n * return arrayToString(dataValue(\"path_of_array\")(state), \\', \\')\\n * })\\n * @param {array} arr - Array of toString\\'able primatives.\\n * @param {string} separator - Separator string.\\n * @returns {string}\\n */\\nexport function arrayToString(arr: any[], separator: string): string;\\n/**\\n * Ensures primitive data types are wrapped in an array.\\n * Does not affect array objects.\\n * @public\\n * @function\\n * @example\\n * each(function(state) {\\n * return toArray( dataValue(\"path_of_array\")(state) )\\n * }, ...)\\n * @param {any} arg - Data required to be in an array\\n * @returns {array}\\n */\\nexport function toArray(arg: any): any[];\\n/**\\n * Prepares next state\\n * @public\\n * @function\\n * @example\\n * composeNextState(state, response)\\n * @param {State} state - state\\n * @param {Object} response - Response to be added\\n * @returns {State}\\n */\\nexport function composeNextState(state: State, response: any): State;\\n/**\\n * Substitutes underscores for spaces and proper-cases a string\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", humanProper(state.data.path_to_string))\\n * @param {string} str - String that needs converting\\n * @returns {string}\\n */\\nexport function humanProper(str: string): string;\\n/**\\n * Splits an object into two objects based on a list of keys.\\n * The first object contains the keys that are not in the list,\\n * and the second contains the keys that are.\\n * @public\\n * @function\\n * @param {Object} obj - The object to split.\\n * @param {string[]} keys - List of keys to split on.\\n * @returns {Object[]} - Tuple of objects, first object contains keys not in list, second contains keys that are.\\n */\\nexport function splitKeys(obj: any, keys: string[]): any[];\\n/**\\n * Replaces emojis in a string.\\n * @public\\n * @function\\n * @example\\n * scrubEmojis(\\'Dove🕊️⭐ 29\\')\\n * @param {string} text - String that needs to be cleaned\\n * @param {string} replacementChars - Characters that replace the emojis\\n * @returns {string}\\n */\\nexport function scrubEmojis(text: string, replacementChars: string): string;\\n/**\\n * Chunks an array into an array of arrays, each with no more than a certain size.\\n * @public\\n * @function\\n * @example\\n * chunk([1,2,3,4,5], 2)\\n * @param {Object} array - Array to be chunked\\n * @param {Integer} chunkSize - The maxiumum size of each chunks\\n * @returns {Object}\\n */\\nexport function chunk(array: any, chunkSize: Integer): any;\\n/**\\n * Takes a CSV file string or stream and parsing options as input, and returns a promise that\\n * resolves to the parsed CSV data as an array of objects.\\n * Options for `parsingOptions` include:\\n * - `delimiter` {string/Buffer/[string/Buffer]} - Defines the character(s) used to delineate the fields inside a record. Default: `\\',\\'`\\n * - `quote` {string/Buffer/[string/Buffer]} - Defines the characters used to surround a field. Default: `\\'\"\\'`\\n * - `escape` {Buffer/string/null/boolean} - Set the escape character as one character/byte only. Default: `\"`\\n * - `columns` {boolean / array / function} - Generates record in the form of object literals. Default: `true`\\n * - `bom` {boolean} - Strips the {@link https://en.wikipedia.org/wiki/Byte_order_mark byte order mark (BOM)} from the input string or buffer. Default: `true`\\n * - `trim` {boolean} - Ignore whitespace characters immediately around the `delimiter`. Default: `true`\\n * - `ltrim` {boolean} - Ignore whitespace characters from the left side of a CSV field. Default: `true`\\n * - `rtrim` {boolean} - Ignore whitespace characters from the right side of a CSV field. Default: `true`\\n * - `chunkSize` {number} - The size of each chunk of CSV data. Default: `Infinity`\\n * - `skip_empty_lines` {boolean} - Ignore empty lines in the CSV file. Default: `true`\\n * @public\\n * @function\\n * @param {String | Stream} csvData - A CSV string or a readable stream\\n * @param {Object} [parsingOptions] - Optional. Parsing options for converting CSV to JSON.\\n * @param {function} [callback] - (Optional) callback function. If used it will be called state and an array of rows.\\n * @returns {Operation} The function returns a Promise that resolves to the result of parsing a CSV `stringOrStream`.\\n */\\nexport function parseCsv(csvData: string | Stream, parsingOptions?: any, callback?: Function): Operation;\\n/**\\n * Validate against a JSON schema. Any erors are written to an array at `state.validationErrors`.\\n * Schema can be passed directly, loaded as a JSON path from state, or loaded from a URL\\n * Data can be passed directly or loaded as a JSON path from state.\\n * By default, schema is loaded from `state.schema` and data from `state.data`.\\n * @pubic\\n * @function\\n * @param {string|object} schema - The schema, path or URL to validate against\\n * @param {string|object} data - The data or path to validate\\n * @example Validate `state.data` with `state.schema`\\n * validate()\\n * @example Validate form data at `state.form` with a schema from a URL\\n * validate(\"https://www.example.com/schema/record\", \"form\")\\n * @example Validate the each item in `state.records` with a schema from a URL\\n * each(\"records[*]\", validate(\"https://www.example.com/schema/record\"))\\n * @returns {Operation}\\n */\\nexport function validate(schema?: string | object, data?: string | object): Operation;\\n/**\\n * Sets a cursor property on state.\\n * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\\n * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones\\n * are not yet supported.\\n * You can provide a formatter to customise the final cursor value, which is useful for normalising\\n * different inputs. The custom formatter runs after natural language date conversion.\\n * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}\\n * @public\\n * @function\\n * @example Use a cursor from state if present, or else use the default value\\n * cursor($.cursor, { defaultValue: \\'today\\' })\\n * @example Use a pagination cursor\\n * cursor(22)\\n * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number\\n * @param {object} options - options to control the cursor.\\n * @param {string} options.key - set the cursor key. Will persist through the whole run.\\n * @param {any} options.defaultValue - the value to use if value is falsy\\n * @param {Function} options.format - custom formatter for the final cursor value\\n * @returns {Operation}\\n */\\nexport function cursor(value: any, options?: {\\n key: string;\\n defaultValue: any;\\n format: Function;\\n}): Operation;\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n * @public\\n * @function\\n * @example\\n * map(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\"))\\n * )\\n * )\\n * @param {string} path - JSONPath referencing a point in `state.data`.\\n * @param {function} operation - The operation needed to be repeated.\\n * @param {State} state - Runtime state.\\n * @returns {State}\\n */\\nexport const map: any;\\n\\n\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n *\\n * It also ensures the results of an operation make their way back into\\n * the state\\'s references.\\n * @public\\n * @example\\n * each(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\")))\\n * )\\n * @function\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n\\n\\nexport { parse, format } from \"date-fns\";\\n\\n\\n/**\\n * Builder function to create request options. Returns an object with helpers to\\n * easily add commonly used options. The return object is chainable so you can set\\n * as many options as you want.\\n * Pass an object to set your own options.\\n * @param {CommonRequestOptions} options - options to pass to the request\\n * @returns {OptionsHelpers}\\n * @function\\n * @public\\n * @example Get with a query an oath token\\n * get($.data.url, http.options({ query: $.query }).oath($.configuration.access_token)\\n */\\nexport function options(opts?: {}): any;\\n/**\\n * Make a GET request.\\n * @public\\n * @function\\n * @example Request a resource\\n * http.get(\\'https://jsonplaceholder.typicode.com/todos\\')\\n * @example Request a resource with basic auth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().basic(\\'user\\', \\'pass\\')\\n * )\\n * @example Request a resource with oauth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().oauth($.configuration.access_token)\\n * )\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function get(url: string, options: CommonRequestOptions): Operation;\\n/**\\n * Make a POST request.\\n * @public\\n * @function\\n * @example Post a JSON object (setting the content-type header)\\n * http.post(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * $.data,\\n * options().json(),\\n * })\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function post(path: any, data: any, options: CommonRequestOptions): Operation;\\nexport { req as request };\\n/**\\n * Helper functions provided by `http.options`.\\n */\\nexport type OptionsHelpers = any;\\n/**\\n * Options provided to the HTTP request\\n */\\nexport type CommonRequestOptions = {\\n /**\\n * - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n */\\n errors: object;\\n /**\\n * - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n */\\n form: object;\\n /**\\n * - An object of query parameters to be encoded into the URL.\\n */\\n query: object;\\n /**\\n * - An object of headers to append to the request.\\n */\\n headers: object;\\n /**\\n * - Parse the response body as json, text or stream. By default will use the response headers.\\n */\\n parseAs: string;\\n /**\\n * - Request timeout in ms. Default: 300 seconds.\\n */\\n timeout: number;\\n /**\\n * - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n tls: object;\\n};\\n/**\\n * State object\\n */\\nexport type CommonHttpState = any;\\n/**\\n * Options provided to the HTTP request\\n * @typedef {Object} CommonRequestOptions\\n * @property {object} errors - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n * @property {object} form - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n * @property {object} query - An object of query parameters to be encoded into the URL.\\n * @property {object} headers - An object of headers to append to the request.\\n * @property {string} parseAs - Parse the response body as json, text or stream. By default will use the response headers.\\n * @property {number} timeout - Request timeout in ms. Default: 300 seconds.\\n * @property {object} tls - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n/**\\n * State object\\n * @typedef {Object} CommonHttpState\\n * @private\\n * @property data - the parsed response body\\n * @property response - the response from the HTTP server, including headers, statusCode, body, etc\\n * @property references - an array of all previous data objects used in the Job\\n **/\\n/**\\n * Make a HTTP request.\\n * @public\\n * @function\\n * @example\\n * http.request(\\n * \\'GET\\',\\n * \\'https://jsonplaceholder.typicode.com/todos\\'\\n * )\\n * @name request\\n * @param {string} method - The HTTP method to use.\\n * @param {string} url - URL to resource.\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\ndeclare function req(method: string, url: string, options: CommonRequestOptions): Operation;\\n\\n\\nimport * as Adaptor from \\'./Adaptor\\';\\nexport default Adaptor;\\nexport * from \\'./Adaptor\\';\\nexport * as beta from \\'./beta\\';\\nexport * as http from \\'./http\\';\\nexport * as dateFns from \\'./dateFns\\';\\nimport * as metadata from \\'./metadata\\';\\nexport { metadata };\\n\\n\\ndeclare type Entity = {\\n name: string;\\n type: string;\\n label?: string;\\n datatype?: string;\\n desc?: string;\\n children?: Entity[] | Record;\\n meta?: Record;\\n addChild: (e: Entity, name?: string) => void;\\n};\\ndeclare type DataType = \\'string\\' | \\'boolean\\' | \\'date\\';\\n\\n\\nexport function encode(data: string): string;\\nexport function decode(base64Data: string): string;\\nexport function uuid(): string;\\n\\n\\n/**\\n * `request` is a helper function that sends HTTP requests and returns the response\\n * body, headers, and status code.\\n * Use the error map to provide custom error messages or get hold of the response in case of errors.\\n * @param method - The HTTP method to use for the request (e.g., \"GET\", \"POST\", \"PUT\", \"DELETE\", etc.).\\n * @param fullUrlOrPath - The full or partial URL for the request.\\n * @param [options] - The `options` parameter is an object that contains additional configuration\\n * options for the request.\\n * @returns an object with the following properties:\\n * - method: the request method\\n * - url: the request url\\n * - code: the status code of the response\\n * - headers: the headers of the response\\n * - body: the body of the response\\n * - message: the status text of the response\\n * - duration: the response time\\n */\\nexport function request(method: any, fullUrlOrPath: any, options?: {}): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function makeBasicAuthHeader(username: any, password: any): {\\n Authorization: string;\\n};\\nexport function logResponse(response: any): any;\\nexport function enableMockClient(baseUrl: any): import(\"undici/types/mock-interceptor\").Interceptable;\\nexport const ERROR_ABSOLUTE_URL: \"Absolute URLs not suppored\";\\nexport function assertRelativeUrl(path: any): void;\\nexport const ERROR_URL_MISMATCH: \"Target origin does not match baseUrl origin\";\\nexport function parseUrl(pathOrUrl: string, baseUrl: any): {\\n url: string;\\n baseUrl: string;\\n path: string;\\n query: any;\\n};\\nexport function get(url: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function post(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function put(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function del(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\n\\n\\nexport * from \"./http\";\\nexport * from \"./helpers\";\\nexport * from \"./references\";\\nimport parseDate from \"./parse-date\";\\nimport throwError from \"./throw-error\";\\nexport { parseDate, throwError };\\n\\n\\ndeclare function _default(d: any, startDate: any): any;\\nexport default _default;\\n\\n\\nexport function expandReferences(state: any, ...args: any[]): any[];\\nexport function normalizeOauthConfig(configuration: any): any;\\n\\n\\ndeclare function _default(code: any, { description, fix, ...extras }?: {\\n description: any;\\n fix: any;\\n}): never;\\nexport default _default;\\nTypescript definitions for doc @openfn/language-googlesheets/**\\n * Execute a sequence of oper.\\n * Wraps `language-common/execute`, and prepends initial state for http.\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )(state)\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function execute(...operations: Operations): Operation;\\n/**\\n * Add an array of rows to the spreadsheet.\\n * https://developers.google.com/sheets/api/samples/writing#append_values\\n * @public\\n * @example\\n * appendValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function appendValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Batch update values in a Spreadsheet.\\n * @example\\n * batchUpdateValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @public\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {string} [params.valueInputOption] (Optional) Value update options. Defaults to \\'USER_ENTERED\\'\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function batchUpdateValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n valueInputOption?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Gets cell values from a Spreadsheet.\\n * @public\\n * @example\\n * getValues(\\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\'Sheet1!A1:E1\\')\\n * @function\\n * @param {string} spreadsheetId The spreadsheet ID.\\n * @param {string} range The sheet range.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function getValues(spreadsheetId: string, range: string, callback?: Function): Operation;\\nexport { alterState, combine, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, lastReferenceValue, merge, sourceValue } from \"@openfn/language-common\";\\n\\n\\nexport default Adaptor;\\nexport * from \"./Adaptor\";\\nimport * as Adaptor from \"./Adaptor\";\\n'}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': '// write your job code here'}]" + ] + }, + { + "cell_type": "code", + "execution_count": 289, + "metadata": {}, + "outputs": [], + "source": [ + "# Import tenacity to implement waiting after failed api calls\n", + "\n", + "from tenacity import (\n", + " retry,\n", + " stop_after_attempt,\n", + " wait_random_exponential,\n", + ") # for exponential backoff" + ] + }, + { + "cell_type": "code", + "execution_count": 290, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate answers with the new system prompt\n", + "# @retry(wait=wait_random_exponential(min=1, max=90), stop=stop_after_attempt(20))\n", + "\n", + "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", + "def generate_new_format(question, new_system_prompt, model=\"claude-3-5-sonnet-20240620\"):\n", + " \"\"\"Use the apollo format to call claude with a custom system prompt.\"\"\"\n", + " prompt = [{'role': 'user', 'content': question}]\n", + " print(question)\n", + " message = client.beta.prompt_caching.messages.create(\n", + " max_tokens=1024, messages=prompt, model=model, system=new_system_prompt\n", + " )\n", + " return message.content[0].text" + ] + }, + { + "cell_type": "code", + "execution_count": 291, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using an array of people, map it to create a single sheet for each person\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "Certainly! I'll show you how to map an array of people to create a single sheet for each person using the Google Sheets adaptor. Here's an example of how you can do this:\n", + "\n", + "```javascript\n", + "fn(state => {\n", + " const people = [\n", + " { name: 'Alice', age: 30, city: 'New York' },\n", + " { name: 'Bob', age: 25, city: 'San Francisco' },\n", + " { name: 'Charlie', age: 35, city: 'Chicago' }\n", + " ];\n", + " \n", + " return { ...state, people };\n", + "});\n", + "\n", + "each('$.people[*]', fn(state => {\n", + " const person = state.data;\n", + " const sheetName = person.name;\n", + " \n", + " return appendValues({\n", + " spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\n", + " range: `${sheetName}!A1:C2`,\n", + " values: [\n", + " ['Name', 'Age', 'City'],\n", + " [person.name, person.age, person.city]\n", + " ]\n", + " })(state);\n", + "}));\n", + "```\n", + "\n", + "This job does the following:\n", + "\n", + "1. We use `fn()` to create an array of people and add it to the state.\n", + "\n", + "2. We use `each()` to iterate over each person in the array.\n", + "\n", + "3. For each person, we create a new sheet (if it doesn't exist) and append their information to it.\n", + "\n", + "4. The `appendValues()` function is used to add the data to the sheet. It will create a new sheet with the person's name if it doesn't exist.\n", + "\n", + "5. We're using a header row ('Name', 'Age', 'City') and then adding the person's data in the next row.\n", + "\n", + "Remember to replace the `spreadsheetId` with your actual Google Sheets ID.\n", + "\n", + "Note: This approach assumes that you have the necessary permissions to create new sheets in the specified spreadsheet. If the sheets already exist, it will append the data to the existing sheets.\n", + "\n", + "Also, keep in mind that creating a new sheet for each person might not be the most efficient approach if you have a large number of people. In that case, you might want to consider a different structure, such as having all people in one sheet or grouping them differently.\n" + ] + } + ], + "source": [ + "new_answer = generate_new_format(\n", + " 'Using an array of people, map it to create a single sheet for each person', system_prompt_v2)\n", + "print(new_answer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "apollo", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 6ef2ca2cb3987729e4470cd511e85e5d2b0dcca2 Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Fri, 6 Dec 2024 14:17:20 +0300 Subject: [PATCH 2/6] generate table with comparison Signed-off-by: Hunter Achieng --- .../evaluation/test_prompts_issue_97_a.ipynb | 926 +++++++++++++----- 1 file changed, 692 insertions(+), 234 deletions(-) diff --git a/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb b/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb index c87cf08..90f48ff 100644 --- a/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb +++ b/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb @@ -11,14 +11,14 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 321, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "17314.25s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + "688457.10s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" ] }, { @@ -32,7 +32,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "17320.13s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + "688463.81s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" ] }, { @@ -57,7 +57,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "17325.82s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + "688469.64s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" ] }, { @@ -71,7 +71,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "17331.47s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + "688475.29s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" ] }, { @@ -94,6 +94,34 @@ "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2024.8.30)\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "688481.03s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting pandas\n", + " Downloading pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl.metadata (89 kB)\n", + "Requirement already satisfied: numpy>=1.23.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.1.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", + "Collecting pytz>=2020.1 (from pandas)\n", + " Downloading pytz-2024.2-py2.py3-none-any.whl.metadata (22 kB)\n", + "Collecting tzdata>=2022.7 (from pandas)\n", + " Using cached tzdata-2024.2-py2.py3-none-any.whl.metadata (1.4 kB)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n", + "Downloading pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl (11.3 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading pytz-2024.2-py2.py3-none-any.whl (508 kB)\n", + "Using cached tzdata-2024.2-py2.py3-none-any.whl (346 kB)\n", + "Installing collected packages: pytz, tzdata, pandas\n", + "Successfully installed pandas-2.2.3 pytz-2024.2 tzdata-2024.2\n" + ] } ], "source": [ @@ -101,6 +129,7 @@ "!pip install anthropic\n", "!pip install tenacity\n", "!pip install apollo\n", + "!pip install pandas\n", "\n" ] }, @@ -122,148 +151,60 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 301, "metadata": {}, "outputs": [], "source": [ "# Sample user prompt\n", "user_prompt = \"\"\"\n", - "Generate a job that iterates over a patients' list using openFn functions \n", + "Generate 10 short code questions that a user might ask about openFn job writing that makes use of different adaptor functions and mappings. \n", + "\n", + "The questions can span around different platforms such as:\n", + "\n", + "commcare\n", + "satusehat\n", + "googlesheets\n", + "fhir\n", + "openmrs\n", + "odk\n", + "\n", + "Output your answers with new lines and without numbers or bullet points.\n", + "\n", + "\n", + "Using an array of people, map it to create a single sheet for each person\n", + "\n", "\"\"\"" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 302, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Here's an example of a job that iterates over a patients' list using openFn functions:\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", "\n", - "```js\n", - "alterState(state => {\n", - " const patients = state.data.patients; // Assuming patients data is in state\n", - " \n", - " return each(\n", - " 'patients[*]',\n", - " alterState(state => {\n", - " const patient = state.data; // Current patient in the iteration\n", - " \n", - " return upsert('Patient__c', 'External_ID__c', {\n", - " External_ID__c: patient.id,\n", - " First_Name__c: patient.firstName,\n", - " Last_Name__c: patient.lastName,\n", - " Date_Of_Birth__c: patient.birthDate,\n", - " Gender__c: patient.gender,\n", - " Phone__c: patient.phone,\n", - " Email__c: patient.email,\n", - " Address__c: patient.address,\n", - " Status__c: patient.status\n", - " });\n", - " })\n", - " )(state);\n", - "});\n", - "```\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", "\n", - "Here's another example that includes additional operations and error handling:\n", + "How do I transform a Google Sheets row into a CommCare case update?\n", "\n", - "```js\n", - "alterState(state => {\n", - " const patients = state.data.patients;\n", - " \n", - " return each(\n", - " 'patients[*]',\n", - " compose(\n", - " alterState(state => {\n", - " const patient = state.data;\n", - " console.log(`Processing patient: ${patient.id}`);\n", - " \n", - " // Data transformation\n", - " const transformedPatient = {\n", - " External_ID__c: patient.id,\n", - " First_Name__c: patient.firstName,\n", - " Last_Name__c: patient.lastName,\n", - " Date_Of_Birth__c: patient.birthDate,\n", - " Gender__c: patient.gender,\n", - " Phone__c: patient.phone,\n", - " Email__c: patient.email,\n", - " Address__c: patient.address,\n", - " Status__c: patient.status,\n", - " Last_Visit_Date__c: patient.lastVisit,\n", - " Medical_History__c: patient.medicalHistory\n", - " };\n", - " \n", - " return { ...state, transformedPatient };\n", - " }),\n", - " \n", - " // Validate required fields\n", - " alterState(state => {\n", - " const { transformedPatient } = state;\n", - " const requiredFields = ['External_ID__c', 'First_Name__c', 'Last_Name__c'];\n", - " \n", - " const missingFields = requiredFields.filter(field => !transformedPatient[field]);\n", - " \n", - " if (missingFields.length > 0) {\n", - " console.log(`Warning: Missing required fields for patient ${transformedPatient.External_ID__c}: ${missingFields.join(', ')}`);\n", - " }\n", - " \n", - " return state;\n", - " }),\n", - " \n", - " // Perform upsert operation\n", - " alterState(state => {\n", - " const { transformedPatient } = state;\n", - " \n", - " return upsert('Patient__c', 'External_ID__c', transformedPatient)\n", - " .then(result => {\n", - " console.log(`Successfully processed patient ${transformedPatient.External_ID__c}`);\n", - " return state;\n", - " })\n", - " .catch(error => {\n", - " console.error(`Error processing patient ${transformedPatient.External_ID__c}:`, error);\n", - " return state;\n", - " });\n", - " }),\n", - " \n", - " // Create related records\n", - " alterState(state => {\n", - " const patient = state.data;\n", - " \n", - " if (patient.appointments) {\n", - " return each(\n", - " 'appointments[*]',\n", - " upsert('Appointment__c', 'External_ID__c', {\n", - " External_ID__c: appointment => appointment.id,\n", - " Patient__c: patient.id,\n", - " Date__c: appointment => appointment.date,\n", - " Type__c: appointment => appointment.type,\n", - " Status__c: appointment => appointment.status\n", - " })\n", - " )(state);\n", - " }\n", - " return state;\n", - " })\n", - " )\n", - " )(state);\n", - "});\n", - "```\n", + "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", "\n", - "This expanded example includes:\n", + "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", "\n", - "1. Initial data transformation\n", - "2. Validation of required fields\n", - "3. Main upsert operation for patient records\n", - "4. Error handling and logging\n", - "5. Creation of related records (appointments)\n", - "6. Use of compose to chain multiple operations\n", + "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", "\n", - "You can also add more complex operations:\n", + "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", "\n", - "```js\n", - "alterState(\n" + "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", + "\n", + "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", + "\n", + "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n" ] } ], @@ -292,93 +233,62 @@ }, { "cell_type": "code", - "execution_count": 288, + "execution_count": 303, "metadata": {}, "outputs": [], "source": [ - "# New edited system prompt\n", - "\n", - "system_prompt_role_v2 =\"\"\"\n", - "You are a software engineer helping a non-expert user write a job for OpenFn,\n", - "the world's leading digital public good for workflow automation.\n", - "\n", - "Where reasonable, assume questions are related to workflow automation, \n", - "professional platforms or programming. You may provide general information around these topics, \n", - "e.g. general programming assistance unrelated to job writing.\n", - "If a question is entirely irrelevant, do not answer it.\n", + "# Create list from generated questions\n", "\n", - "Your responses short be short, accurate and friendly unless otherwise instructed.\n", - "\n", - "Do not thank the user or be obsequious.\n", - "\n", - "Only follow the instructios given in the context\n", - "\n", - "Address the user directly.\n", - "\n", - "The `each()` function iterates over an array of items and invokes an operation upon each one, where the state object is scoped so that state.data is the item under iteration. The rest of the state object is untouched and can be referenced as usual. You can pass an array directly: `each(['1234'])`, or use lazy state:`each($.patient)` or a JSONPath string: `each('$.patient[*]')` to reference a slice of state.\n", - "\n", - "Ensure that `fn()` and `each()` are not nested within each other. For example:\n", - "\n", - "```\n", - "each('$.data[*]', operation); // CORRECT\n", - "each('$.data[*]', fn(state => { /* ... */ })); // INCORRECT\n", - "\n", - "```\n", - " \n", - "\n", - "Only adaptor-provided operations are allowed as the second argument to `each()`. For example:\n", - "\n", - "```\n", - "each('$.data[*]', appendValues({ spreadsheetId: '123', range: 'Sheet1!A1' }));\n", + "generated_questions = [q for q in message.content[0].text.split(\"\\n\") if q]" + ] + }, + { + "cell_type": "code", + "execution_count": 309, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/hunter/apollo/services\n" + ] + } + ], + "source": [ + "import sys\n", + "import os\n", "\n", - "```\n", - " \n", + "# Add the parent directory to sys.path\n", + "parent_dir = os.path.abspath(\"../../\")\n", + "print(parent_dir)\n", + "sys.path.append(parent_dir)\n", "\n", - "Adaptor functions like `appendValues`, `getValues`, etc., must be used directly as part of an operation. Avoid wrapping them with `fn()` or any custom state functions unless explicitly required by the job logic.\n", + "from prompt import job_writing_summary, system_role, build_prompt\n", + "from util import DictObj\n", "\n", - "Use `fn()` only when custom state manipulation is required.\n", "\n", - "Avoid wrapping adaptor operations inside `fn()` unless explicitly necessary\n", "\n", - "\n", - "\n", - "DO THIS:\n", - "```\n", - "each('$.form.responses[*]', appendValues({\n", - " spreadsheetId: '123',\n", - " range: 'Sheet1!A1',\n", - " values: [['Name', 'Response'], ['Alice', 'Yes']],\n", - "}));\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 331, + "metadata": {}, + "outputs": [], + "source": [ + "# New edited system prompt\n", "\n", - "```\n", - "\n", "\n", - "\n", - "DO NOT DO THIS:\n", - "```\n", - "each('$.form.responses[*]', fn(state => {\n", - " // Avoid fn() inside each\n", - " return appendValues({\n", - " spreadsheetId: '123',\n", - " range: 'Sheet1!A1',\n", - " values: [['Name', 'Response'], ['Alice', 'Yes']],\n", - " })(state);\n", - "}));\n", - "\n", - "```\n", - "\n", - "\n", - "\n", - "Additional context is attached.\n", - "\"\"\"\n", "\n", "# formatted like in apollo\n", - "system_prompt_v2 = [{'type': 'text', 'text': system_prompt_role_v2}, {'type': 'text', 'text': \"\\nAn OpenFn Job is written in a DSL which is very similar to Javascript.\\n\\nJob code does not use import statements or async/await.\\n\\nJob code must only contain function calls at the top level.\\n\\nEach job is associated with an adaptor, which provides functions for the job.\\nAll jobs have the fn() and each() function, which are very important.\\n\\nDO NOT use the `alterState()` function. Use `fn()` instead.\\n\\nThe `fn()` and `each()` operations SHOULD NOT be nested within each other.\\n\\nThe adaptor API may be attached.\\n\\nThe functions provided by an adaptor are called Operations.\\n\\nAn Operation is a factory function which returns a function that takes state and returns state, like this:\\n```\\nconst myOperation = (arg) => (state) => { /* do something with arg and state */ return state; }\\n```\\n\\n\\nHere's how we issue a GET request with the http adaptor:\\n```\\nget('/patients');\\n```\\nThe first argument to get is the path to request from (the configuration will tell\\nthe adaptor what base url to use). In this case we're passing a static string,\\nbut we can also pass a value from state:\\n```\\nget(state => state.endpoint);\\n```\\n\\n\\nExample job code with the HTTP adaptor:\\n```\\nget('/patients');\\nfn(state => {\\n const patients = state.data.map(p => {\\n return { ...p, enrolled: true }\\n });\\n\\n return { ...state, data: { patients } };\\n})\\npost('/patients', dataValue('patients'));\\n\\n\\n```\\nExample job code with the Salesforce adaptor:\\n```\\neach(\\n '$.form.participants[*]',\\n upsert('Person__c', 'Participant_PID__c', state => ({\\n Participant_PID__c: state.pid,\\n First_Name__c: state.participant_first_name,\\n Surname__c: state.participant_surname,\\n }))\\n);\\n```\\n\\n\\nExample job code with the ODK adaptor:\\n```\\ncreate(\\n 'ODK_Submission__c',\\n fields(\\n field('Site_School_ID_Number__c', dataValue('school')),\\n field('Date_Completed__c', dataValue('date')),\\n field('comments__c', dataValue('comments')),\\n field('ODK_Key__c', dataValue('*meta-instance-id*'))\\n )\\n);\\n```\\n\\n\\n\"}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': 'The user is using the OpenFn @openfn/language-googlesheets@3.0.5 adaptor. Use functions provided by its API.Typescript definitions for doc @openfn/language-common/**\\n * Execute a sequence of operations.\\n * Main outer API for executing expressions.\\n * @public\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Promise}\\n */\\nexport function execute(...operations: Operations): Promise;\\n/**\\n * alias for \"fn()\"\\n * @function\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function alterState(func: Function): Operation;\\n/**\\n * Creates a custom step (or operation) for more flexible job writing.\\n * @public\\n * @function\\n * @example\\n * fn(state => {\\n * // do some things to state\\n * return state;\\n * });\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function fn(func: Function): Operation;\\n/**\\n * A custom operation that will only execute the function if the condition returns true\\n * @public\\n * @function\\n * @example\\n * fnIf((state) => state?.data?.name, get(\"https://example.com\"));\\n * @param {Boolean} condition - The condition that returns true\\n * @param {Operation} operation - The operation needed to be executed.\\n * @returns {Operation}\\n */\\nexport function fnIf(condition: boolean, operation: Operation): Operation;\\n/**\\n * Picks out a single value from a JSON object.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * jsonValue({ a:1 }, \\'a\\')\\n * @param {object} obj - A valid JSON object.\\n * @param {String} path - JSONPath referencing a point in given JSON object.\\n * @returns {Operation}\\n */\\nexport function jsonValue(obj: object, path: string): Operation;\\n/**\\n * Picks out a single value from source data.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * sourceValue(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Operation}\\n */\\nexport function sourceValue(path: string): Operation;\\n/**\\n * Picks out a value from source data.\\n * Will return whatever JSONPath returns, which will always be an array.\\n * If you need a single value use `sourceValue` instead.\\n * @public\\n * @function\\n * @example\\n * source(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Array.}\\n */\\nexport function source(path: string): Array;\\n/**\\n * Ensures a path points at the data.\\n * @public\\n * @function\\n * @example\\n * dataPath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `data`.\\n * @returns {string}\\n */\\nexport function dataPath(path: string): string;\\n/**\\n * Picks out a single value from the source data object—usually `state.data`.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * dataValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `data`.\\n * @returns {Operation}\\n */\\nexport function dataValue(path: string): Operation;\\n/**\\n * Ensures a path points at references.\\n * @public\\n * @function\\n * @example\\n * referencePath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `references`.\\n * @returns {string}\\n */\\nexport function referencePath(path: string): string;\\n/**\\n * Picks out the last reference value from source data.\\n * @public\\n * @function\\n * @example\\n * lastReferenceValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `references`.\\n * @returns {Operation}\\n */\\nexport function lastReferenceValue(path: string): Operation;\\n/**\\n * Simple switcher allowing other expressions to use either a JSONPath or\\n * object literals as a data source.\\n * - JSONPath referencing a point in `state`\\n * - Object Literal of the data itself.\\n * - Function to be called with state.\\n * @public\\n * @function\\n * @example\\n * asData(\\'$.key\\'| key | callback)\\n * @param {String|object|function} data\\n * @param {object} state - The current state.\\n * @returns {array}\\n */\\nexport function asData(data: string | object | Function, state: object): any[];\\n/**\\n * Iterates over an array of items and invokes an operation upon each one, where the state\\n * object is _scoped_ so that state.data is the item under iteration.\\n * The rest of the state object is untouched and can be referenced as usual.\\n * You can pass an array directly, or use lazy state or a JSONPath string to\\n * reference a slice of state.\\n * @public\\n * @function\\n * @example Using lazy state ($) to iterate over items in state.data and pass each into an \"insert\" operation\\n * each(\\n * $.data,\\n * // Inside the callback operation, `$.data` is scoped to the item under iteration\\n * insert(\"patient\", {\\n * patient_name: $.data.properties.case_name,\\n * patient_id: $.data.case_id,\\n * })\\n * );\\n * @example Iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * $.data,\\n * insert(\"patient\", (state) => ({\\n * patient_id: state.data.case_id,\\n * ...state.data\\n * }))\\n * );\\n * @example Using JSON path to iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * \"$.data[*]\",\\n * insert(\"patient\", (state) => ({\\n * patient_name: state.data.properties.case_name,\\n * patient_id: state.data.case_id,\\n * }))\\n * );\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n/**\\n * Combines two operations into one\\n * @public\\n * @function\\n * @example\\n * combine(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function combine(...operations: Operations): Operation;\\n/**\\n * Adds data from a target object\\n * @public\\n * @function\\n * @example\\n * join(\\'$.key\\',\\'$.data\\',\\'newKey\\')\\n * @param {String} targetPath - Target path\\n * @param {String} sourcePath - Source path\\n * @param {String} targetKey - Target Key\\n * @returns {Operation}\\n */\\nexport function join(targetPath: string, sourcePath: string, targetKey: string): Operation;\\n/**\\n * Recursively resolves objects that have resolvable values (functions).\\n * @public\\n * @function\\n * @param {object} value - data\\n * @param {Function} [skipFilter] - a function which returns true if a value should be skipped\\n * @returns {Operation}\\n */\\nexport function expandReferences(value: object, skipFilter?: Function): Operation;\\n/**\\n * Returns a key, value pair in an array.\\n * @public\\n * @function\\n * @example\\n * field(\\'destination_field_name__c\\', \\'value\\')\\n * @param {string} key - Name of the field\\n * @param {Value} value - The value itself or a sourceable operation.\\n * @returns {Field}\\n */\\nexport function field(key: string, value: Value): Field;\\n/**\\n * Zips key value pairs into an object.\\n * @public\\n * @function\\n * @example\\n * fields(list_of_fields)\\n * @param {Fields} fields - a list of fields\\n * @returns {Object}\\n */\\nexport function fields(...fields: Fields): any;\\n/**\\n * Merges fields into each item in an array.\\n * @public\\n * @example\\n * merge(\\n * \"$.books[*]\",\\n * fields(\\n * field( \"publisher\", sourceValue(\"$.publisher\") )\\n * )\\n * )\\n * @function\\n * @public\\n * @param {DataSource} dataSource\\n * @param {Object} fields - Group of fields to merge in.\\n * @returns {DataSource}\\n */\\nexport function merge(dataSource: DataSource, fields: any): DataSource;\\n/**\\n * Groups an array of objects by a specified key path.\\n * @public\\n * @example\\n * const users = [\\n * { name: \\'Alice\\', age: 25, city: \\'New York\\' },\\n * { name: \\'Bob\\', age: 30, city: \\'San Francisco\\' },\\n * { name: \\'Charlie\\', age: 25, city: \\'New York\\' },\\n * { name: \\'David\\', age: 30, city: \\'San Francisco\\' }\\n * ];\\n * group(users, \\'city\\');\\n * // state is { data: { \\'New York\\': [/Alice, Charlie/], \\'San Francisco\\': [ /Bob, David / ] }\\n * @function\\n * @public\\n * @param {Object[]} arrayOfObjects - The array of objects to be grouped.\\n * @param {string} keyPath - The key path to group by.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function group(arrayOfObjects: any[], keyPath: string, callback?: Function): Operation;\\n/**\\n * Returns the index of the current array being iterated.\\n * To be used with `each` as a data source.\\n * @public\\n * @function\\n * @example\\n * index()\\n * @returns {DataSource}\\n */\\nexport function index(): DataSource;\\n/**\\n * Turns an array into a string, separated by X.\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", function(state) {\\n * return arrayToString(dataValue(\"path_of_array\")(state), \\', \\')\\n * })\\n * @param {array} arr - Array of toString\\'able primatives.\\n * @param {string} separator - Separator string.\\n * @returns {string}\\n */\\nexport function arrayToString(arr: any[], separator: string): string;\\n/**\\n * Ensures primitive data types are wrapped in an array.\\n * Does not affect array objects.\\n * @public\\n * @function\\n * @example\\n * each(function(state) {\\n * return toArray( dataValue(\"path_of_array\")(state) )\\n * }, ...)\\n * @param {any} arg - Data required to be in an array\\n * @returns {array}\\n */\\nexport function toArray(arg: any): any[];\\n/**\\n * Prepares next state\\n * @public\\n * @function\\n * @example\\n * composeNextState(state, response)\\n * @param {State} state - state\\n * @param {Object} response - Response to be added\\n * @returns {State}\\n */\\nexport function composeNextState(state: State, response: any): State;\\n/**\\n * Substitutes underscores for spaces and proper-cases a string\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", humanProper(state.data.path_to_string))\\n * @param {string} str - String that needs converting\\n * @returns {string}\\n */\\nexport function humanProper(str: string): string;\\n/**\\n * Splits an object into two objects based on a list of keys.\\n * The first object contains the keys that are not in the list,\\n * and the second contains the keys that are.\\n * @public\\n * @function\\n * @param {Object} obj - The object to split.\\n * @param {string[]} keys - List of keys to split on.\\n * @returns {Object[]} - Tuple of objects, first object contains keys not in list, second contains keys that are.\\n */\\nexport function splitKeys(obj: any, keys: string[]): any[];\\n/**\\n * Replaces emojis in a string.\\n * @public\\n * @function\\n * @example\\n * scrubEmojis(\\'Dove🕊️⭐ 29\\')\\n * @param {string} text - String that needs to be cleaned\\n * @param {string} replacementChars - Characters that replace the emojis\\n * @returns {string}\\n */\\nexport function scrubEmojis(text: string, replacementChars: string): string;\\n/**\\n * Chunks an array into an array of arrays, each with no more than a certain size.\\n * @public\\n * @function\\n * @example\\n * chunk([1,2,3,4,5], 2)\\n * @param {Object} array - Array to be chunked\\n * @param {Integer} chunkSize - The maxiumum size of each chunks\\n * @returns {Object}\\n */\\nexport function chunk(array: any, chunkSize: Integer): any;\\n/**\\n * Takes a CSV file string or stream and parsing options as input, and returns a promise that\\n * resolves to the parsed CSV data as an array of objects.\\n * Options for `parsingOptions` include:\\n * - `delimiter` {string/Buffer/[string/Buffer]} - Defines the character(s) used to delineate the fields inside a record. Default: `\\',\\'`\\n * - `quote` {string/Buffer/[string/Buffer]} - Defines the characters used to surround a field. Default: `\\'\"\\'`\\n * - `escape` {Buffer/string/null/boolean} - Set the escape character as one character/byte only. Default: `\"`\\n * - `columns` {boolean / array / function} - Generates record in the form of object literals. Default: `true`\\n * - `bom` {boolean} - Strips the {@link https://en.wikipedia.org/wiki/Byte_order_mark byte order mark (BOM)} from the input string or buffer. Default: `true`\\n * - `trim` {boolean} - Ignore whitespace characters immediately around the `delimiter`. Default: `true`\\n * - `ltrim` {boolean} - Ignore whitespace characters from the left side of a CSV field. Default: `true`\\n * - `rtrim` {boolean} - Ignore whitespace characters from the right side of a CSV field. Default: `true`\\n * - `chunkSize` {number} - The size of each chunk of CSV data. Default: `Infinity`\\n * - `skip_empty_lines` {boolean} - Ignore empty lines in the CSV file. Default: `true`\\n * @public\\n * @function\\n * @param {String | Stream} csvData - A CSV string or a readable stream\\n * @param {Object} [parsingOptions] - Optional. Parsing options for converting CSV to JSON.\\n * @param {function} [callback] - (Optional) callback function. If used it will be called state and an array of rows.\\n * @returns {Operation} The function returns a Promise that resolves to the result of parsing a CSV `stringOrStream`.\\n */\\nexport function parseCsv(csvData: string | Stream, parsingOptions?: any, callback?: Function): Operation;\\n/**\\n * Validate against a JSON schema. Any erors are written to an array at `state.validationErrors`.\\n * Schema can be passed directly, loaded as a JSON path from state, or loaded from a URL\\n * Data can be passed directly or loaded as a JSON path from state.\\n * By default, schema is loaded from `state.schema` and data from `state.data`.\\n * @pubic\\n * @function\\n * @param {string|object} schema - The schema, path or URL to validate against\\n * @param {string|object} data - The data or path to validate\\n * @example Validate `state.data` with `state.schema`\\n * validate()\\n * @example Validate form data at `state.form` with a schema from a URL\\n * validate(\"https://www.example.com/schema/record\", \"form\")\\n * @example Validate the each item in `state.records` with a schema from a URL\\n * each(\"records[*]\", validate(\"https://www.example.com/schema/record\"))\\n * @returns {Operation}\\n */\\nexport function validate(schema?: string | object, data?: string | object): Operation;\\n/**\\n * Sets a cursor property on state.\\n * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\\n * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones\\n * are not yet supported.\\n * You can provide a formatter to customise the final cursor value, which is useful for normalising\\n * different inputs. The custom formatter runs after natural language date conversion.\\n * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}\\n * @public\\n * @function\\n * @example Use a cursor from state if present, or else use the default value\\n * cursor($.cursor, { defaultValue: \\'today\\' })\\n * @example Use a pagination cursor\\n * cursor(22)\\n * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number\\n * @param {object} options - options to control the cursor.\\n * @param {string} options.key - set the cursor key. Will persist through the whole run.\\n * @param {any} options.defaultValue - the value to use if value is falsy\\n * @param {Function} options.format - custom formatter for the final cursor value\\n * @returns {Operation}\\n */\\nexport function cursor(value: any, options?: {\\n key: string;\\n defaultValue: any;\\n format: Function;\\n}): Operation;\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n * @public\\n * @function\\n * @example\\n * map(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\"))\\n * )\\n * )\\n * @param {string} path - JSONPath referencing a point in `state.data`.\\n * @param {function} operation - The operation needed to be repeated.\\n * @param {State} state - Runtime state.\\n * @returns {State}\\n */\\nexport const map: any;\\n\\n\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n *\\n * It also ensures the results of an operation make their way back into\\n * the state\\'s references.\\n * @public\\n * @example\\n * each(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\")))\\n * )\\n * @function\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n\\n\\nexport { parse, format } from \"date-fns\";\\n\\n\\n/**\\n * Builder function to create request options. Returns an object with helpers to\\n * easily add commonly used options. The return object is chainable so you can set\\n * as many options as you want.\\n * Pass an object to set your own options.\\n * @param {CommonRequestOptions} options - options to pass to the request\\n * @returns {OptionsHelpers}\\n * @function\\n * @public\\n * @example Get with a query an oath token\\n * get($.data.url, http.options({ query: $.query }).oath($.configuration.access_token)\\n */\\nexport function options(opts?: {}): any;\\n/**\\n * Make a GET request.\\n * @public\\n * @function\\n * @example Request a resource\\n * http.get(\\'https://jsonplaceholder.typicode.com/todos\\')\\n * @example Request a resource with basic auth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().basic(\\'user\\', \\'pass\\')\\n * )\\n * @example Request a resource with oauth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().oauth($.configuration.access_token)\\n * )\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function get(url: string, options: CommonRequestOptions): Operation;\\n/**\\n * Make a POST request.\\n * @public\\n * @function\\n * @example Post a JSON object (setting the content-type header)\\n * http.post(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * $.data,\\n * options().json(),\\n * })\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function post(path: any, data: any, options: CommonRequestOptions): Operation;\\nexport { req as request };\\n/**\\n * Helper functions provided by `http.options`.\\n */\\nexport type OptionsHelpers = any;\\n/**\\n * Options provided to the HTTP request\\n */\\nexport type CommonRequestOptions = {\\n /**\\n * - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n */\\n errors: object;\\n /**\\n * - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n */\\n form: object;\\n /**\\n * - An object of query parameters to be encoded into the URL.\\n */\\n query: object;\\n /**\\n * - An object of headers to append to the request.\\n */\\n headers: object;\\n /**\\n * - Parse the response body as json, text or stream. By default will use the response headers.\\n */\\n parseAs: string;\\n /**\\n * - Request timeout in ms. Default: 300 seconds.\\n */\\n timeout: number;\\n /**\\n * - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n tls: object;\\n};\\n/**\\n * State object\\n */\\nexport type CommonHttpState = any;\\n/**\\n * Options provided to the HTTP request\\n * @typedef {Object} CommonRequestOptions\\n * @property {object} errors - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n * @property {object} form - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n * @property {object} query - An object of query parameters to be encoded into the URL.\\n * @property {object} headers - An object of headers to append to the request.\\n * @property {string} parseAs - Parse the response body as json, text or stream. By default will use the response headers.\\n * @property {number} timeout - Request timeout in ms. Default: 300 seconds.\\n * @property {object} tls - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n/**\\n * State object\\n * @typedef {Object} CommonHttpState\\n * @private\\n * @property data - the parsed response body\\n * @property response - the response from the HTTP server, including headers, statusCode, body, etc\\n * @property references - an array of all previous data objects used in the Job\\n **/\\n/**\\n * Make a HTTP request.\\n * @public\\n * @function\\n * @example\\n * http.request(\\n * \\'GET\\',\\n * \\'https://jsonplaceholder.typicode.com/todos\\'\\n * )\\n * @name request\\n * @param {string} method - The HTTP method to use.\\n * @param {string} url - URL to resource.\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\ndeclare function req(method: string, url: string, options: CommonRequestOptions): Operation;\\n\\n\\nimport * as Adaptor from \\'./Adaptor\\';\\nexport default Adaptor;\\nexport * from \\'./Adaptor\\';\\nexport * as beta from \\'./beta\\';\\nexport * as http from \\'./http\\';\\nexport * as dateFns from \\'./dateFns\\';\\nimport * as metadata from \\'./metadata\\';\\nexport { metadata };\\n\\n\\ndeclare type Entity = {\\n name: string;\\n type: string;\\n label?: string;\\n datatype?: string;\\n desc?: string;\\n children?: Entity[] | Record;\\n meta?: Record;\\n addChild: (e: Entity, name?: string) => void;\\n};\\ndeclare type DataType = \\'string\\' | \\'boolean\\' | \\'date\\';\\n\\n\\nexport function encode(data: string): string;\\nexport function decode(base64Data: string): string;\\nexport function uuid(): string;\\n\\n\\n/**\\n * `request` is a helper function that sends HTTP requests and returns the response\\n * body, headers, and status code.\\n * Use the error map to provide custom error messages or get hold of the response in case of errors.\\n * @param method - The HTTP method to use for the request (e.g., \"GET\", \"POST\", \"PUT\", \"DELETE\", etc.).\\n * @param fullUrlOrPath - The full or partial URL for the request.\\n * @param [options] - The `options` parameter is an object that contains additional configuration\\n * options for the request.\\n * @returns an object with the following properties:\\n * - method: the request method\\n * - url: the request url\\n * - code: the status code of the response\\n * - headers: the headers of the response\\n * - body: the body of the response\\n * - message: the status text of the response\\n * - duration: the response time\\n */\\nexport function request(method: any, fullUrlOrPath: any, options?: {}): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function makeBasicAuthHeader(username: any, password: any): {\\n Authorization: string;\\n};\\nexport function logResponse(response: any): any;\\nexport function enableMockClient(baseUrl: any): import(\"undici/types/mock-interceptor\").Interceptable;\\nexport const ERROR_ABSOLUTE_URL: \"Absolute URLs not suppored\";\\nexport function assertRelativeUrl(path: any): void;\\nexport const ERROR_URL_MISMATCH: \"Target origin does not match baseUrl origin\";\\nexport function parseUrl(pathOrUrl: string, baseUrl: any): {\\n url: string;\\n baseUrl: string;\\n path: string;\\n query: any;\\n};\\nexport function get(url: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function post(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function put(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function del(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\n\\n\\nexport * from \"./http\";\\nexport * from \"./helpers\";\\nexport * from \"./references\";\\nimport parseDate from \"./parse-date\";\\nimport throwError from \"./throw-error\";\\nexport { parseDate, throwError };\\n\\n\\ndeclare function _default(d: any, startDate: any): any;\\nexport default _default;\\n\\n\\nexport function expandReferences(state: any, ...args: any[]): any[];\\nexport function normalizeOauthConfig(configuration: any): any;\\n\\n\\ndeclare function _default(code: any, { description, fix, ...extras }?: {\\n description: any;\\n fix: any;\\n}): never;\\nexport default _default;\\nTypescript definitions for doc @openfn/language-googlesheets/**\\n * Execute a sequence of oper.\\n * Wraps `language-common/execute`, and prepends initial state for http.\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )(state)\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function execute(...operations: Operations): Operation;\\n/**\\n * Add an array of rows to the spreadsheet.\\n * https://developers.google.com/sheets/api/samples/writing#append_values\\n * @public\\n * @example\\n * appendValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function appendValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Batch update values in a Spreadsheet.\\n * @example\\n * batchUpdateValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @public\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {string} [params.valueInputOption] (Optional) Value update options. Defaults to \\'USER_ENTERED\\'\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function batchUpdateValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n valueInputOption?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Gets cell values from a Spreadsheet.\\n * @public\\n * @example\\n * getValues(\\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\'Sheet1!A1:E1\\')\\n * @function\\n * @param {string} spreadsheetId The spreadsheet ID.\\n * @param {string} range The sheet range.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function getValues(spreadsheetId: string, range: string, callback?: Function): Operation;\\nexport { alterState, combine, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, lastReferenceValue, merge, sourceValue } from \"@openfn/language-common\";\\n\\n\\nexport default Adaptor;\\nexport * from \"./Adaptor\";\\nimport * as Adaptor from \"./Adaptor\";\\n'}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': '// write your job code here'}]" + "system_prompt_v2 = [{'type': 'text', 'text': system_role}, {'type': 'text', 'text': job_writing_summary}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': 'The user is using the OpenFn @openfn/language-googlesheets@3.0.5 adaptor. Use functions provided by its API.Typescript definitions for doc @openfn/language-common/**\\n * Execute a sequence of operations.\\n * Main outer API for executing expressions.\\n * @public\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Promise}\\n */\\nexport function execute(...operations: Operations): Promise;\\n/**\\n * alias for \"fn()\"\\n * @function\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function alterState(func: Function): Operation;\\n/**\\n * Creates a custom step (or operation) for more flexible job writing.\\n * @public\\n * @function\\n * @example\\n * fn(state => {\\n * // do some things to state\\n * return state;\\n * });\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function fn(func: Function): Operation;\\n/**\\n * A custom operation that will only execute the function if the condition returns true\\n * @public\\n * @function\\n * @example\\n * fnIf((state) => state?.data?.name, get(\"https://example.com\"));\\n * @param {Boolean} condition - The condition that returns true\\n * @param {Operation} operation - The operation needed to be executed.\\n * @returns {Operation}\\n */\\nexport function fnIf(condition: boolean, operation: Operation): Operation;\\n/**\\n * Picks out a single value from a JSON object.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * jsonValue({ a:1 }, \\'a\\')\\n * @param {object} obj - A valid JSON object.\\n * @param {String} path - JSONPath referencing a point in given JSON object.\\n * @returns {Operation}\\n */\\nexport function jsonValue(obj: object, path: string): Operation;\\n/**\\n * Picks out a single value from source data.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * sourceValue(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Operation}\\n */\\nexport function sourceValue(path: string): Operation;\\n/**\\n * Picks out a value from source data.\\n * Will return whatever JSONPath returns, which will always be an array.\\n * If you need a single value use `sourceValue` instead.\\n * @public\\n * @function\\n * @example\\n * source(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Array.}\\n */\\nexport function source(path: string): Array;\\n/**\\n * Ensures a path points at the data.\\n * @public\\n * @function\\n * @example\\n * dataPath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `data`.\\n * @returns {string}\\n */\\nexport function dataPath(path: string): string;\\n/**\\n * Picks out a single value from the source data object—usually `state.data`.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * dataValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `data`.\\n * @returns {Operation}\\n */\\nexport function dataValue(path: string): Operation;\\n/**\\n * Ensures a path points at references.\\n * @public\\n * @function\\n * @example\\n * referencePath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `references`.\\n * @returns {string}\\n */\\nexport function referencePath(path: string): string;\\n/**\\n * Picks out the last reference value from source data.\\n * @public\\n * @function\\n * @example\\n * lastReferenceValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `references`.\\n * @returns {Operation}\\n */\\nexport function lastReferenceValue(path: string): Operation;\\n/**\\n * Simple switcher allowing other expressions to use either a JSONPath or\\n * object literals as a data source.\\n * - JSONPath referencing a point in `state`\\n * - Object Literal of the data itself.\\n * - Function to be called with state.\\n * @public\\n * @function\\n * @example\\n * asData(\\'$.key\\'| key | callback)\\n * @param {String|object|function} data\\n * @param {object} state - The current state.\\n * @returns {array}\\n */\\nexport function asData(data: string | object | Function, state: object): any[];\\n/**\\n * Iterates over an array of items and invokes an operation upon each one, where the state\\n * object is _scoped_ so that state.data is the item under iteration.\\n * The rest of the state object is untouched and can be referenced as usual.\\n * You can pass an array directly, or use lazy state or a JSONPath string to\\n * reference a slice of state.\\n * @public\\n * @function\\n * @example Using lazy state ($) to iterate over items in state.data and pass each into an \"insert\" operation\\n * each(\\n * $.data,\\n * // Inside the callback operation, `$.data` is scoped to the item under iteration\\n * insert(\"patient\", {\\n * patient_name: $.data.properties.case_name,\\n * patient_id: $.data.case_id,\\n * })\\n * );\\n * @example Iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * $.data,\\n * insert(\"patient\", (state) => ({\\n * patient_id: state.data.case_id,\\n * ...state.data\\n * }))\\n * );\\n * @example Using JSON path to iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * \"$.data[*]\",\\n * insert(\"patient\", (state) => ({\\n * patient_name: state.data.properties.case_name,\\n * patient_id: state.data.case_id,\\n * }))\\n * );\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n/**\\n * Combines two operations into one\\n * @public\\n * @function\\n * @example\\n * combine(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function combine(...operations: Operations): Operation;\\n/**\\n * Adds data from a target object\\n * @public\\n * @function\\n * @example\\n * join(\\'$.key\\',\\'$.data\\',\\'newKey\\')\\n * @param {String} targetPath - Target path\\n * @param {String} sourcePath - Source path\\n * @param {String} targetKey - Target Key\\n * @returns {Operation}\\n */\\nexport function join(targetPath: string, sourcePath: string, targetKey: string): Operation;\\n/**\\n * Recursively resolves objects that have resolvable values (functions).\\n * @public\\n * @function\\n * @param {object} value - data\\n * @param {Function} [skipFilter] - a function which returns true if a value should be skipped\\n * @returns {Operation}\\n */\\nexport function expandReferences(value: object, skipFilter?: Function): Operation;\\n/**\\n * Returns a key, value pair in an array.\\n * @public\\n * @function\\n * @example\\n * field(\\'destination_field_name__c\\', \\'value\\')\\n * @param {string} key - Name of the field\\n * @param {Value} value - The value itself or a sourceable operation.\\n * @returns {Field}\\n */\\nexport function field(key: string, value: Value): Field;\\n/**\\n * Zips key value pairs into an object.\\n * @public\\n * @function\\n * @example\\n * fields(list_of_fields)\\n * @param {Fields} fields - a list of fields\\n * @returns {Object}\\n */\\nexport function fields(...fields: Fields): any;\\n/**\\n * Merges fields into each item in an array.\\n * @public\\n * @example\\n * merge(\\n * \"$.books[*]\",\\n * fields(\\n * field( \"publisher\", sourceValue(\"$.publisher\") )\\n * )\\n * )\\n * @function\\n * @public\\n * @param {DataSource} dataSource\\n * @param {Object} fields - Group of fields to merge in.\\n * @returns {DataSource}\\n */\\nexport function merge(dataSource: DataSource, fields: any): DataSource;\\n/**\\n * Groups an array of objects by a specified key path.\\n * @public\\n * @example\\n * const users = [\\n * { name: \\'Alice\\', age: 25, city: \\'New York\\' },\\n * { name: \\'Bob\\', age: 30, city: \\'San Francisco\\' },\\n * { name: \\'Charlie\\', age: 25, city: \\'New York\\' },\\n * { name: \\'David\\', age: 30, city: \\'San Francisco\\' }\\n * ];\\n * group(users, \\'city\\');\\n * // state is { data: { \\'New York\\': [/Alice, Charlie/], \\'San Francisco\\': [ /Bob, David / ] }\\n * @function\\n * @public\\n * @param {Object[]} arrayOfObjects - The array of objects to be grouped.\\n * @param {string} keyPath - The key path to group by.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function group(arrayOfObjects: any[], keyPath: string, callback?: Function): Operation;\\n/**\\n * Returns the index of the current array being iterated.\\n * To be used with `each` as a data source.\\n * @public\\n * @function\\n * @example\\n * index()\\n * @returns {DataSource}\\n */\\nexport function index(): DataSource;\\n/**\\n * Turns an array into a string, separated by X.\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", function(state) {\\n * return arrayToString(dataValue(\"path_of_array\")(state), \\', \\')\\n * })\\n * @param {array} arr - Array of toString\\'able primatives.\\n * @param {string} separator - Separator string.\\n * @returns {string}\\n */\\nexport function arrayToString(arr: any[], separator: string): string;\\n/**\\n * Ensures primitive data types are wrapped in an array.\\n * Does not affect array objects.\\n * @public\\n * @function\\n * @example\\n * each(function(state) {\\n * return toArray( dataValue(\"path_of_array\")(state) )\\n * }, ...)\\n * @param {any} arg - Data required to be in an array\\n * @returns {array}\\n */\\nexport function toArray(arg: any): any[];\\n/**\\n * Prepares next state\\n * @public\\n * @function\\n * @example\\n * composeNextState(state, response)\\n * @param {State} state - state\\n * @param {Object} response - Response to be added\\n * @returns {State}\\n */\\nexport function composeNextState(state: State, response: any): State;\\n/**\\n * Substitutes underscores for spaces and proper-cases a string\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", humanProper(state.data.path_to_string))\\n * @param {string} str - String that needs converting\\n * @returns {string}\\n */\\nexport function humanProper(str: string): string;\\n/**\\n * Splits an object into two objects based on a list of keys.\\n * The first object contains the keys that are not in the list,\\n * and the second contains the keys that are.\\n * @public\\n * @function\\n * @param {Object} obj - The object to split.\\n * @param {string[]} keys - List of keys to split on.\\n * @returns {Object[]} - Tuple of objects, first object contains keys not in list, second contains keys that are.\\n */\\nexport function splitKeys(obj: any, keys: string[]): any[];\\n/**\\n * Replaces emojis in a string.\\n * @public\\n * @function\\n * @example\\n * scrubEmojis(\\'Dove🕊️⭐ 29\\')\\n * @param {string} text - String that needs to be cleaned\\n * @param {string} replacementChars - Characters that replace the emojis\\n * @returns {string}\\n */\\nexport function scrubEmojis(text: string, replacementChars: string): string;\\n/**\\n * Chunks an array into an array of arrays, each with no more than a certain size.\\n * @public\\n * @function\\n * @example\\n * chunk([1,2,3,4,5], 2)\\n * @param {Object} array - Array to be chunked\\n * @param {Integer} chunkSize - The maxiumum size of each chunks\\n * @returns {Object}\\n */\\nexport function chunk(array: any, chunkSize: Integer): any;\\n/**\\n * Takes a CSV file string or stream and parsing options as input, and returns a promise that\\n * resolves to the parsed CSV data as an array of objects.\\n * Options for `parsingOptions` include:\\n * - `delimiter` {string/Buffer/[string/Buffer]} - Defines the character(s) used to delineate the fields inside a record. Default: `\\',\\'`\\n * - `quote` {string/Buffer/[string/Buffer]} - Defines the characters used to surround a field. Default: `\\'\"\\'`\\n * - `escape` {Buffer/string/null/boolean} - Set the escape character as one character/byte only. Default: `\"`\\n * - `columns` {boolean / array / function} - Generates record in the form of object literals. Default: `true`\\n * - `bom` {boolean} - Strips the {@link https://en.wikipedia.org/wiki/Byte_order_mark byte order mark (BOM)} from the input string or buffer. Default: `true`\\n * - `trim` {boolean} - Ignore whitespace characters immediately around the `delimiter`. Default: `true`\\n * - `ltrim` {boolean} - Ignore whitespace characters from the left side of a CSV field. Default: `true`\\n * - `rtrim` {boolean} - Ignore whitespace characters from the right side of a CSV field. Default: `true`\\n * - `chunkSize` {number} - The size of each chunk of CSV data. Default: `Infinity`\\n * - `skip_empty_lines` {boolean} - Ignore empty lines in the CSV file. Default: `true`\\n * @public\\n * @function\\n * @param {String | Stream} csvData - A CSV string or a readable stream\\n * @param {Object} [parsingOptions] - Optional. Parsing options for converting CSV to JSON.\\n * @param {function} [callback] - (Optional) callback function. If used it will be called state and an array of rows.\\n * @returns {Operation} The function returns a Promise that resolves to the result of parsing a CSV `stringOrStream`.\\n */\\nexport function parseCsv(csvData: string | Stream, parsingOptions?: any, callback?: Function): Operation;\\n/**\\n * Validate against a JSON schema. Any erors are written to an array at `state.validationErrors`.\\n * Schema can be passed directly, loaded as a JSON path from state, or loaded from a URL\\n * Data can be passed directly or loaded as a JSON path from state.\\n * By default, schema is loaded from `state.schema` and data from `state.data`.\\n * @pubic\\n * @function\\n * @param {string|object} schema - The schema, path or URL to validate against\\n * @param {string|object} data - The data or path to validate\\n * @example Validate `state.data` with `state.schema`\\n * validate()\\n * @example Validate form data at `state.form` with a schema from a URL\\n * validate(\"https://www.example.com/schema/record\", \"form\")\\n * @example Validate the each item in `state.records` with a schema from a URL\\n * each(\"records[*]\", validate(\"https://www.example.com/schema/record\"))\\n * @returns {Operation}\\n */\\nexport function validate(schema?: string | object, data?: string | object): Operation;\\n/**\\n * Sets a cursor property on state.\\n * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\\n * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones\\n * are not yet supported.\\n * You can provide a formatter to customise the final cursor value, which is useful for normalising\\n * different inputs. The custom formatter runs after natural language date conversion.\\n * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}\\n * @public\\n * @function\\n * @example Use a cursor from state if present, or else use the default value\\n * cursor($.cursor, { defaultValue: \\'today\\' })\\n * @example Use a pagination cursor\\n * cursor(22)\\n * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number\\n * @param {object} options - options to control the cursor.\\n * @param {string} options.key - set the cursor key. Will persist through the whole run.\\n * @param {any} options.defaultValue - the value to use if value is falsy\\n * @param {Function} options.format - custom formatter for the final cursor value\\n * @returns {Operation}\\n */\\nexport function cursor(value: any, options?: {\\n key: string;\\n defaultValue: any;\\n format: Function;\\n}): Operation;\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n * @public\\n * @function\\n * @example\\n * map(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\"))\\n * )\\n * )\\n * @param {string} path - JSONPath referencing a point in `state.data`.\\n * @param {function} operation - The operation needed to be repeated.\\n * @param {State} state - Runtime state.\\n * @returns {State}\\n */\\nexport const map: any;\\n\\n\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n *\\n * It also ensures the results of an operation make their way back into\\n * the state\\'s references.\\n * @public\\n * @example\\n * each(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\")))\\n * )\\n * @function\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n\\n\\nexport { parse, format } from \"date-fns\";\\n\\n\\n/**\\n * Builder function to create request options. Returns an object with helpers to\\n * easily add commonly used options. The return object is chainable so you can set\\n * as many options as you want.\\n * Pass an object to set your own options.\\n * @param {CommonRequestOptions} options - options to pass to the request\\n * @returns {OptionsHelpers}\\n * @function\\n * @public\\n * @example Get with a query an oath token\\n * get($.data.url, http.options({ query: $.query }).oath($.configuration.access_token)\\n */\\nexport function options(opts?: {}): any;\\n/**\\n * Make a GET request.\\n * @public\\n * @function\\n * @example Request a resource\\n * http.get(\\'https://jsonplaceholder.typicode.com/todos\\')\\n * @example Request a resource with basic auth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().basic(\\'user\\', \\'pass\\')\\n * )\\n * @example Request a resource with oauth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().oauth($.configuration.access_token)\\n * )\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function get(url: string, options: CommonRequestOptions): Operation;\\n/**\\n * Make a POST request.\\n * @public\\n * @function\\n * @example Post a JSON object (setting the content-type header)\\n * http.post(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * $.data,\\n * options().json(),\\n * })\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function post(path: any, data: any, options: CommonRequestOptions): Operation;\\nexport { req as request };\\n/**\\n * Helper functions provided by `http.options`.\\n */\\nexport type OptionsHelpers = any;\\n/**\\n * Options provided to the HTTP request\\n */\\nexport type CommonRequestOptions = {\\n /**\\n * - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n */\\n errors: object;\\n /**\\n * - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n */\\n form: object;\\n /**\\n * - An object of query parameters to be encoded into the URL.\\n */\\n query: object;\\n /**\\n * - An object of headers to append to the request.\\n */\\n headers: object;\\n /**\\n * - Parse the response body as json, text or stream. By default will use the response headers.\\n */\\n parseAs: string;\\n /**\\n * - Request timeout in ms. Default: 300 seconds.\\n */\\n timeout: number;\\n /**\\n * - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n tls: object;\\n};\\n/**\\n * State object\\n */\\nexport type CommonHttpState = any;\\n/**\\n * Options provided to the HTTP request\\n * @typedef {Object} CommonRequestOptions\\n * @property {object} errors - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n * @property {object} form - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n * @property {object} query - An object of query parameters to be encoded into the URL.\\n * @property {object} headers - An object of headers to append to the request.\\n * @property {string} parseAs - Parse the response body as json, text or stream. By default will use the response headers.\\n * @property {number} timeout - Request timeout in ms. Default: 300 seconds.\\n * @property {object} tls - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n/**\\n * State object\\n * @typedef {Object} CommonHttpState\\n * @private\\n * @property data - the parsed response body\\n * @property response - the response from the HTTP server, including headers, statusCode, body, etc\\n * @property references - an array of all previous data objects used in the Job\\n **/\\n/**\\n * Make a HTTP request.\\n * @public\\n * @function\\n * @example\\n * http.request(\\n * \\'GET\\',\\n * \\'https://jsonplaceholder.typicode.com/todos\\'\\n * )\\n * @name request\\n * @param {string} method - The HTTP method to use.\\n * @param {string} url - URL to resource.\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\ndeclare function req(method: string, url: string, options: CommonRequestOptions): Operation;\\n\\n\\nimport * as Adaptor from \\'./Adaptor\\';\\nexport default Adaptor;\\nexport * from \\'./Adaptor\\';\\nexport * as beta from \\'./beta\\';\\nexport * as http from \\'./http\\';\\nexport * as dateFns from \\'./dateFns\\';\\nimport * as metadata from \\'./metadata\\';\\nexport { metadata };\\n\\n\\ndeclare type Entity = {\\n name: string;\\n type: string;\\n label?: string;\\n datatype?: string;\\n desc?: string;\\n children?: Entity[] | Record;\\n meta?: Record;\\n addChild: (e: Entity, name?: string) => void;\\n};\\ndeclare type DataType = \\'string\\' | \\'boolean\\' | \\'date\\';\\n\\n\\nexport function encode(data: string): string;\\nexport function decode(base64Data: string): string;\\nexport function uuid(): string;\\n\\n\\n/**\\n * `request` is a helper function that sends HTTP requests and returns the response\\n * body, headers, and status code.\\n * Use the error map to provide custom error messages or get hold of the response in case of errors.\\n * @param method - The HTTP method to use for the request (e.g., \"GET\", \"POST\", \"PUT\", \"DELETE\", etc.).\\n * @param fullUrlOrPath - The full or partial URL for the request.\\n * @param [options] - The `options` parameter is an object that contains additional configuration\\n * options for the request.\\n * @returns an object with the following properties:\\n * - method: the request method\\n * - url: the request url\\n * - code: the status code of the response\\n * - headers: the headers of the response\\n * - body: the body of the response\\n * - message: the status text of the response\\n * - duration: the response time\\n */\\nexport function request(method: any, fullUrlOrPath: any, options?: {}): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function makeBasicAuthHeader(username: any, password: any): {\\n Authorization: string;\\n};\\nexport function logResponse(response: any): any;\\nexport function enableMockClient(baseUrl: any): import(\"undici/types/mock-interceptor\").Interceptable;\\nexport const ERROR_ABSOLUTE_URL: \"Absolute URLs not suppored\";\\nexport function assertRelativeUrl(path: any): void;\\nexport const ERROR_URL_MISMATCH: \"Target origin does not match baseUrl origin\";\\nexport function parseUrl(pathOrUrl: string, baseUrl: any): {\\n url: string;\\n baseUrl: string;\\n path: string;\\n query: any;\\n};\\nexport function get(url: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function post(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function put(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function del(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\n\\n\\nexport * from \"./http\";\\nexport * from \"./helpers\";\\nexport * from \"./references\";\\nimport parseDate from \"./parse-date\";\\nimport throwError from \"./throw-error\";\\nexport { parseDate, throwError };\\n\\n\\ndeclare function _default(d: any, startDate: any): any;\\nexport default _default;\\n\\n\\nexport function expandReferences(state: any, ...args: any[]): any[];\\nexport function normalizeOauthConfig(configuration: any): any;\\n\\n\\ndeclare function _default(code: any, { description, fix, ...extras }?: {\\n description: any;\\n fix: any;\\n}): never;\\nexport default _default;\\nTypescript definitions for doc @openfn/language-googlesheets/**\\n * Execute a sequence of oper.\\n * Wraps `language-common/execute`, and prepends initial state for http.\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )(state)\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function execute(...operations: Operations): Operation;\\n/**\\n * Add an array of rows to the spreadsheet.\\n * https://developers.google.com/sheets/api/samples/writing#append_values\\n * @public\\n * @example\\n * appendValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function appendValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Batch update values in a Spreadsheet.\\n * @example\\n * batchUpdateValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @public\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {string} [params.valueInputOption] (Optional) Value update options. Defaults to \\'USER_ENTERED\\'\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function batchUpdateValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n valueInputOption?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Gets cell values from a Spreadsheet.\\n * @public\\n * @example\\n * getValues(\\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\'Sheet1!A1:E1\\')\\n * @function\\n * @param {string} spreadsheetId The spreadsheet ID.\\n * @param {string} range The sheet range.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function getValues(spreadsheetId: string, range: string, callback?: Function): Operation;\\nexport { alterState, combine, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, lastReferenceValue, merge, sourceValue } from \"@openfn/language-common\";\\n\\n\\nexport default Adaptor;\\nexport * from \"./Adaptor\";\\nimport * as Adaptor from \"./Adaptor\";\\n'}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': '// write your job code here'}]" ] }, { "cell_type": "code", - "execution_count": 289, + "execution_count": 332, "metadata": {}, "outputs": [], "source": [ @@ -393,7 +303,62 @@ }, { "cell_type": "code", - "execution_count": 290, + "execution_count": 333, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", + "def generate_apollo_format(content, history=\"\", adaptor=\"@openfn/language-googlesheets@3.0.5\", model=\"claude-3-5-sonnet-20240620\"):\n", + " \"\"\"Use Apollo prompt formatting templates to generate a Claude answer.\"\"\"\n", + "\n", + " context = DictObj({\n", + " \"expression\": \"// write your job code here\",\n", + " \"adaptor\": adaptor\n", + " })\n", + " print(content)\n", + " system_message, prompt = build_prompt(content, history, context)\n", + " # print(system_message)\n", + " # print(prompt)\n", + "\n", + " message = client.beta.prompt_caching.messages.create(\n", + " max_tokens=1024, messages=prompt, model=model, system=system_message\n", + " )\n", + " return message.content[0].text" + ] + }, + { + "cell_type": "code", + "execution_count": 334, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How do I transform a Google Sheets row into a CommCare case update?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "data": { + "text/plain": [ + "\"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format CommCare expects\\n3. Send the update to CommCare\\n\\nHere's a basic outline of how you might structure this job:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E10')\\n\\n// Transform the data\\nfn(state => {\\n const rows = state.data.values;\\n const caseUpdates = rows.map(row => ({\\n case_id: row[0], // Assuming case ID is in the first column\\n properties: {\\n name: row[1],\\n age: row[2],\\n // Add more properties as needed\\n }\\n }));\\n return { ...state, caseUpdates };\\n})\\n\\n// Send updates to CommCare\\neach(\\n '$.caseUpdates[*]',\\n post(\\n 'https://www.commcarehq.org/a/your-domain/api/v0.5/case/',\\n state => ({\\n case_id: state.data.case_id,\\n properties: state.data.properties\\n }),\\n {\\n headers: {\\n 'Authorization': 'ApiKey your-api-key',\\n 'Content-Type': 'application/json'\\n }\\n }\\n )\\n)\\n```\\n\\nThis is a basic example and you'll need to adjust it based on your specific Google Sheets structure and CommCare case properties. Also, you'll need to replace 'your-spreadsheet-id', 'your-domain', and 'your-api-key' with your actual values.\\n\\nRemember to handle errors and edge cases in a production environment. You might also want to add logging or error reporting.\"" + ] + }, + "execution_count": 334, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "generate_apollo_format(\"How do I transform a Google Sheets row into a CommCare case update?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 335, "metadata": {}, "outputs": [], "source": [ @@ -413,67 +378,560 @@ }, { "cell_type": "code", - "execution_count": 291, + "execution_count": 336, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Using an array of people, map it to create a single sheet for each person\n", + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "Certainly! I'll show you how to map an array of people to create a single sheet for each person using the Google Sheets adaptor. Here's an example of how you can do this:\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I transform a Google Sheets row into a CommCare case update?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I transform a Google Sheets row into a CommCare case update?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" + ] + } + ], + "source": [ + "# Get answers to generated questions\n", + "qa_pairs = []\n", + "for question in generated_questions[:30]:\n", + " answer = generate_apollo_format(question)\n", + " qa_pairs.append((question, answer, \"v1\"))\n", + "for question in generated_questions[:30]: # two loops to make use of prompt caching\n", + " new_answer = generate_new_format(question, system_prompt_v2)\n", + " qa_pairs.append((question, new_answer, \"v2\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 337, + "metadata": {}, + "outputs": [], + "source": [ + "evaluate_answer_template = \"\"\"\n", + "For the given Question and Answer pair, evaluate whether the answer at least briefly answers the question (True), or if it entirely fails to answer the question (False).\n", + "\n", + "Additionally, check the content of the `answer` variable for the presence of the pattern `)(state)` using regex. If this pattern is found in the `answer`, consider the response invalid and mark it as `Success: False`.\n", + "\n", + "\n", + "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", + "A: \"get('Patient)\"\n", + "Regex Match: False\n", + "Success: True\n", + "\n", + "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", + "A: \"fn(state => {{ return {{ ...state, people: [] }}; }})(state);\"\n", + "Regex Match: True\n", + "Success: False\n", + "\n", + "Now, following Q/A pair, by ONLY outputting \"True\" or \"False\":\n", + "Q: {question}\n", + "A: {answer}\n", + "Regex Match: \n", + "Success:\n", + "\"\"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 338, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: False\n", "\n", - "```javascript\n", - "fn(state => {\n", - " const people = [\n", - " { name: 'Alice', age: 30, city: 'New York' },\n", - " { name: 'Bob', age: 25, city: 'San Francisco' },\n", - " { name: 'Charlie', age: 35, city: 'Chicago' }\n", - " ];\n", - " \n", - " return { ...state, people };\n", - "});\n", + "The answer provides information about using GoogleSheets adaptor and appendValues function, but does not address how to create an ODK form submission using CommCare case data. The answer is completely off-topic from what was asked in the question.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "each('$.people[*]', fn(state => {\n", - " const person = state.data;\n", - " const sheetName = person.name;\n", - " \n", - " return appendValues({\n", - " spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\n", - " range: `${sheetName}!A1:C2`,\n", - " values: [\n", - " ['Name', 'Age', 'City'],\n", - " [person.name, person.age, person.city]\n", - " ]\n", - " })(state);\n", - "}));\n", - "```\n", + "The answer provides a detailed explanation of how to merge OpenMRS observations data into a Google Sheet row, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer successfully addresses the question, so Success is True.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "This job does the following:\n", + "The answer provides a detailed explanation of how to use each() to process FHIR resources and create CommCare cases, including code examples and step-by-step instructions. While the answer contains state in function parameters, it doesn't match the specific pattern `)(state)`, so there's no regex match. The answer successfully addresses the question asked.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "1. We use `fn()` to create an array of people and add it to the state.\n", + "The answer provides a detailed explanation of how to map ODK repeat groups to create multiple OpenMRS encounters using Google Sheets adaptor, including code examples and step-by-step instructions. While it uses Google Sheets as an intermediary rather than direct OpenMRS integration, it still addresses the core concept of mapping repeat groups to multiple entries, making it a valid response to the question.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Let me evaluate this:\n", "\n", - "2. We use `each()` to iterate over each person in the array.\n", + "1. First, checking for the regex pattern `)(state)` in the answer:\n", + " - The answer contains code examples but doesn't have the exact pattern `)(state)`\n", + " - Regex Match: False\n", "\n", - "3. For each person, we create a new sheet (if it doesn't exist) and append their information to it.\n", + "2. Evaluating if the answer addresses the question about using combine() to merge data:\n", + " - The answer directly explains how to use combine()\n", + " - Provides a detailed example with steps\n", + " - Shows how to merge data from multiple sources\n", + " - Demonstrates creating a FHIR resource with the combined data\n", + " - Success: True\n", "\n", - "4. The `appendValues()` function is used to add the data to the sheet. It will create a new sheet with the person's name if it doesn't exist.\n", + "Therefore, the final answer is: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "5. We're using a header row ('Name', 'Age', 'City') and then adding the person's data in the next row.\n", + "The answer provides a detailed explanation of how to map CommCare case properties to OpenMRS patient attributes, including code examples and key steps. It directly addresses the question asked, and it doesn't contain the invalid pattern `)(state)`.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "Remember to replace the `spreadsheetId` with your actual Google Sheets ID.\n", + "The answer provides a detailed explanation of how to create ODK form submissions using CommCare case data, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer addresses the question directly, so it's successful.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "Note: This approach assumes that you have the necessary permissions to create new sheets in the specified spreadsheet. If the sheets already exist, it will append the data to the existing sheets.\n", + "The answer provides a detailed explanation of a general approach to mapping data from CommCare to FHIR resources, including specific steps and code examples. While it acknowledges not having specific information about SatuSehat's implementation, it still provides a relevant and helpful response to the question about mapping nested CommCare form data to FHIR resources.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: False\n", + "Success: True\n", "\n", - "Also, keep in mind that creating a new sheet for each person might not be the most efficient approach if you have a large number of people. In that case, you might want to consider a different structure, such as having all people in one sheet or grouping them differently.\n" + "The answer provides a detailed explanation of how to filter Google Sheets data, which addresses the core question about filtering Google Sheets data, even though it clarifies that direct integration with SatuSehat isn't built-in. The pattern `)(state)` is not present in the answer, and the response provides useful, relevant information about data filtering in Google Sheets.\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Let me evaluate this:\n", + "\n", + "1. First, checking if the answer addresses the question about using combine() to merge data:\n", + "The answer provides a detailed explanation of using combine() with multiple data sources and includes a code example, so it does answer the question.\n", + "\n", + "2. Checking for the regex pattern `)(state)`:\n", + "Looking through the answer, while it contains `state` in various places, it doesn't contain the specific pattern `)(state)`.\n", + "\n", + "Therefore:\n", + "Regex Match: False\n", + "Success: True\n", + "\n", + "The final answer is: True\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "Regex Match: True (the pattern `)(state)` appears in the code)\n", + "Success: False\n" + ] + } + ], + "source": [ + "# Evaluate QA pairs\n", + "\n", + "results = []\n", + "\n", + "for q, a, version in qa_pairs:\n", + " evaluate_answer_template_filled = evaluate_answer_template.format(question=q, answer=a)\n", + "\n", + " message = client.messages.create(\n", + " model=\"claude-3-5-sonnet-20241022\",\n", + " max_tokens=1000, #TODO limit to 1\n", + " temperature=0,\n", + " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": evaluate_answer_template_filled\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + " )\n", + " result = message.content[0].text\n", + " print(result)\n", + " results.append([q, a, version, result])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 339, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[['How can I map CommCare case data to create multiple FHIR patients in a single job?', 'To map CommCare case data to create multiple FHIR patients in a single job, you can use the `each` function to iterate over the CommCare cases and create a FHIR patient for each one. Here\\'s an example of how you could structure your job:\\n\\n```javascript\\neach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [{ value: caseData.case_id }],\\n name: [{ \\n given: [caseData.first_name],\\n family: caseData.last_name\\n }],\\n gender: caseData.gender,\\n birthDate: caseData.dob\\n };\\n \\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Patients!A:E\",\\n values: [[\\n patientData.identifier[0].value,\\n patientData.name[0].given[0],\\n patientData.name[0].family,\\n patientData.gender,\\n patientData.birthDate\\n ]]\\n })(state);\\n })\\n);\\n```\\n\\nThis job does the following:\\n\\n1. Uses `each` to iterate over all cases in the CommCare data.\\n2. For each case, it creates a FHIR Patient resource structure.\\n3. It then appends a row to a Google Sheet with the patient data.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with the actual ID of your Google Sheet.\\n\\nNote that this is a simplified example. You might need to adjust the field mappings based on your specific CommCare and FHIR data structures. Also, you may want to add error handling and additional data transformations as needed.', 'v1', 'True'], [\"What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\", 'To create a new ODK form submission using data from multiple CommCare cases, you\\'ll need to use a combination of functions from the GoogleSheets adaptor. Here\\'s a general approach:\\n\\n1. First, you\\'d fetch the CommCare case data (this would be done in a previous step or job).\\n2. Then, you\\'d use the `appendValues` function to add the data to a Google Sheet.\\n\\nHere\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.cases[*]\",\\n appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:E\",\\n values: state => [\\n [\\n state.data.case_id,\\n state.data.properties.name,\\n state.data.properties.age,\\n state.data.properties.village,\\n state.data.properties.status\\n ]\\n ]\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. It uses `each` to iterate over an array of cases stored in `state.cases`.\\n2. For each case, it calls `appendValues` to add a new row to the Google Sheet.\\n3. The `values` property is a function that returns an array of arrays (in this case, just one inner array per case). Each inner array represents a row in the spreadsheet.\\n4. We\\'re assuming the CommCare case data has properties like `case_id`, `name`, `age`, `village`, and `status`. You\\'ll need to adjust these to match your actual case structure.\\n\\nRemember to replace \"YOUR_SPREADSHEET_ID_HERE\" with your actual Google Spreadsheet ID.\\n\\nAlso, make sure you\\'ve correctly set up your configuration to authenticate with Google Sheets API.\\n\\nIf you need to do any data transformation or combine data from multiple cases into a single row, you can do that within the `values` function.', 'v1', 'Regex Match: False\\nSuccess: False\\n\\nThe answer provides information about using GoogleSheets adaptor and appendValues function, but does not address how to create an ODK form submission using CommCare case data. The answer is completely off-topic from what was asked in the question.'], ['How do I transform a Google Sheets row into a CommCare case update?', \"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format CommCare expects\\n3. Send the transformed data to CommCare\\n\\nHere's a basic example of how you might structure this:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A2:E', state => {\\n const [name, caseId, age, status] = state.data.values[0];\\n \\n // Transform data for CommCare\\n const caseData = {\\n case_id: caseId,\\n properties: {\\n name: name,\\n age: age,\\n status: status\\n }\\n };\\n \\n // Here you would use a CommCare adaptor to send the data\\n // This is just a placeholder since we're using Google Sheets adaptor\\n console.log('Case data to send to CommCare:', caseData);\\n \\n return state;\\n});\\n```\\n\\nNote that this example assumes:\\n1. Your Google Sheet has columns for name, case ID, age, and status\\n2. You're only processing one row (the first row of data)\\n\\nTo process multiple rows, you'd need to use `each()` or map over `state.data.values`.\\n\\nRemember, this job doesn't actually send data to CommCare - you'd need to use the CommCare adaptor for that part. But this shows how you can fetch and transform the data from Google Sheets.\", 'v1', 'Regex Match: False\\nSuccess: True'], [\"What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\", \"I apologize, but I can't provide specific advice about mapping CommCare form data to SatuSehat's FHIR resources, as that's outside the scope of the OpenFn job writing assistance I'm configured to provide. \\n\\nHowever, I can offer some general guidance on working with nested data in OpenFn jobs:\\n\\n1. Use the `each()` function to iterate over nested arrays of data.\\n\\n2. Use dot notation or bracket notation to access nested properties, e.g. `state.data.form.patient.name`.\\n\\n3. The `dataValue()` function can be useful for extracting values from nested paths.\\n\\n4. You may need to use `fn()` blocks to perform more complex data transformations.\\n\\n5. Consider breaking down complex mappings into multiple steps for readability.\\n\\nIf you have a specific OpenFn job writing question related to Google Sheets, I'd be happy to assist with that. Otherwise, you may want to consult documentation or experts familiar with CommCare and SatuSehat's FHIR implementation for the specific mapping details.\", 'v1', 'Regex Match: False\\nSuccess: True'], ['How can I merge data from multiple OpenMRS observations into a single Google Sheet row?', \"To merge data from multiple OpenMRS observations into a single Google Sheet row, you can follow these steps:\\n\\n1. Fetch the OpenMRS observations data (this step would be done in a separate OpenMRS adaptor job).\\n\\n2. Transform the data to create a single row per patient with all observations.\\n\\n3. Use the Google Sheets adaptor to append this data as a new row.\\n\\nHere's an example of how you might structure your Google Sheets job to accomplish this:\\n\\n```javascript\\nfn(state => {\\n const patients = state.data.map(patient => {\\n // Assuming patient data is structured with observations\\n return {\\n patientId: patient.id,\\n name: patient.name,\\n weight: patient.observations.find(obs => obs.concept === 'weight')?.value,\\n height: patient.observations.find(obs => obs.concept === 'height')?.value,\\n bloodPressure: patient.observations.find(obs => obs.concept === 'bloodPressure')?.value\\n };\\n });\\n\\n return { ...state, transformedData: patients };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.transformedData.map(patient => [\\n patient.patientId,\\n patient.name,\\n patient.weight,\\n patient.height,\\n patient.bloodPressure\\n ])\\n});\\n```\\n\\nThis code does the following:\\n\\n1. Uses `fn()` to transform the data from the OpenMRS format into a flat structure for each patient.\\n\\n2. Uses `appendValues()` to add the transformed data to your Google Sheet.\\n\\nMake sure to replace the `spreadsheetId` with your actual Google Sheet ID, and adjust the `range` as needed. Also, modify the data transformation logic to match your specific OpenMRS data structure and the desired Google Sheets format.\", 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to merge OpenMRS observations data into a Google Sheet row, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer successfully addresses the question, so Success is True.\"], [\"What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\", 'To use `each()` to process multiple FHIR resources and create CommCare cases, you\\'ll want to structure your code like this:\\n\\n```javascript\\neach(\\n \"$.bundle.entry[*].resource\",\\n fn(state => {\\n const resource = state.data;\\n // Process the FHIR resource and prepare data for CommCare\\n const caseData = {\\n case_type: \"patient\",\\n // Map FHIR fields to CommCare fields\\n name: resource.name[0].given[0],\\n gender: resource.gender,\\n // Add more fields as needed\\n };\\n \\n // Use appendValues to add the case data to a Google Sheet\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: [Object.values(caseData)]\\n })(state);\\n })\\n);\\n```\\n\\nThis code will:\\n\\n1. Use `each()` to iterate over each FHIR resource in the bundle.\\n2. For each resource, create a `caseData` object mapping FHIR fields to CommCare fields.\\n3. Use `appendValues()` to add the case data as a new row in your Google Sheet.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with your actual Google Sheets spreadsheet ID.\\n\\nYou may need to adjust the field mappings and add error handling based on your specific FHIR resources and CommCare case structure.', 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to use each() to process FHIR resources and create CommCare cases, including code examples and step-by-step instructions. While the answer contains state in function parameters, it doesn't match the specific pattern `)(state)`, so there's no regex match. The answer successfully addresses the question asked.\"], ['How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?', 'To map an array of ODK repeat groups to create multiple OpenMRS encounters using the Google Sheets adaptor, you\\'ll need to use a combination of `each()` and `appendValues()`. Here\\'s a general approach:\\n\\n1. First, access the repeat group data from your state.\\n2. Use `each()` to iterate over the repeat group array.\\n3. Inside the `each()` block, format each repeat group item into a row for Google Sheets.\\n4. Use `appendValues()` to add the formatted row to your Google Sheet.\\n\\nHere\\'s an example of how this might look:\\n\\n```javascript\\neach(\\n \"$.form.repeat_group[*]\",\\n fn(state => {\\n const repeatItem = state.data;\\n const formattedRow = [\\n repeatItem.encounter_date,\\n repeatItem.patient_id,\\n repeatItem.symptom,\\n repeatItem.severity\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:D\",\\n values: [formattedRow]\\n })(state);\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. `each(\"$.form.repeat_group[*]\", ...)` iterates over each item in the repeat group array.\\n2. Inside the `fn()`, we access the current repeat group item via `state.data`.\\n3. We format the repeat group item into a row array (`formattedRow`).\\n4. We use `appendValues()` to add this row to the Google Sheet.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with your actual Google Sheets spreadsheet ID, and adjust the range and column structure to match your specific needs.\\n\\nRemember that this approach will make one API call to Google Sheets for each repeat group item. If you have a large number of items, you might want to consider batching the updates for better performance.', 'v1', 'Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to map ODK repeat groups to create multiple OpenMRS encounters using Google Sheets adaptor, including code examples and step-by-step instructions. While it uses Google Sheets as an intermediary rather than direct OpenMRS integration, it still addresses the core concept of mapping repeat groups to multiple entries, making it a valid response to the question.'], [\"What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\", \"I apologize, but there's a misunderstanding in your question. The @openfn/language-googlesheets adaptor doesn't have built-in functionality to directly filter data and send it to SatuSehat. \\n\\nHowever, I can explain how you might approach this task using OpenFn:\\n\\n1. First, you'd use the `getValues` function to retrieve data from Google Sheets.\\n2. Then you'd use `fn()` to filter that data.\\n3. Finally, you'd use a different adaptor (not Google Sheets) to send the filtered data to SatuSehat.\\n\\nHere's a conceptual example:\\n\\n```javascript\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E10')\\nfn(state => {\\n const filteredData = state.data.values.filter(row => {\\n // Your filtering logic here\\n return true; // or false\\n });\\n return { ...state, filteredData };\\n})\\n// Here you'd use another adaptor to send filteredData to SatuSehat\\n```\\n\\nRemember, you'll need to use the appropriate adaptor and functions to send data to SatuSehat after filtering. The Google Sheets adaptor is only for interacting with Google Sheets.\", 'v1', 'Regex Match: False\\nSuccess: True'], ['How can I use combine() to merge data from multiple sources before creating a FHIR resource?', \"To use `combine()` to merge data from multiple sources before creating a FHIR resource, you can follow this pattern:\\n\\n1. Use `getValues()` to fetch data from different Google Sheets.\\n2. Use `combine()` to merge the results of those operations.\\n3. Use `fn()` to process the combined data and create your FHIR resource.\\n\\nHere's an example:\\n\\n```javascript\\ncombine(\\n getValues('spreadsheetId1', 'Sheet1!A1:C10'),\\n getValues('spreadsheetId2', 'Sheet2!A1:D5')\\n)\\n\\nfn(state => {\\n const [data1, data2] = state.references;\\n \\n // Process and merge the data\\n const mergedData = {\\n // ... combine data1 and data2 as needed\\n };\\n\\n // Create FHIR resource\\n const fhirResource = {\\n resourceType: 'Patient',\\n // ... populate with mergedData\\n };\\n\\n return { ...state, fhirResource };\\n})\\n\\n// You could then use this data in a subsequent operation\\n// For example, to append it to another sheet:\\nappendValues({\\n spreadsheetId: 'outputSpreadsheetId',\\n range: 'OutputSheet!A1',\\n values: [JSON.stringify(state.fhirResource)]\\n})\\n```\\n\\nThis approach allows you to fetch data from multiple sources, combine it, process it into a FHIR resource, and then use that resource in subsequent operations.\", 'v1', \"Let me evaluate this:\\n\\n1. First, checking for the regex pattern `)(state)` in the answer:\\n - The answer contains code examples but doesn't have the exact pattern `)(state)`\\n - Regex Match: False\\n\\n2. Evaluating if the answer addresses the question about using combine() to merge data:\\n - The answer directly explains how to use combine()\\n - Provides a detailed example with steps\\n - Shows how to merge data from multiple sources\\n - Demonstrates creating a FHIR resource with the combined data\\n - Success: True\\n\\nTherefore, the final answer is: True\"], [\"What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\", 'To map CommCare case properties to custom OpenMRS patient attributes in an OpenFn job, you\\'ll typically use a combination of the `each()` function to iterate over cases, and then use `field()` functions to map specific properties. Here\\'s a general approach:\\n\\n1. Use `each()` to iterate over CommCare cases\\n2. Inside `each()`, use `create()` or `upsert()` to create/update OpenMRS patients\\n3. Use `field()` functions to map CommCare properties to OpenMRS attributes\\n\\nHere\\'s an example structure:\\n\\n```javascript\\neach(\\n \"$.forms[*]\", // or wherever your CommCare cases are in state\\n create(\"patient\", {\\n attributes: [\\n field(\"attributeType\", \"CommCare_ID\"),\\n field(\"value\", dataValue(\"case.case_id\")),\\n ],\\n person: {\\n names: [\\n {\\n givenName: dataValue(\"case.first_name\"),\\n familyName: dataValue(\"case.last_name\"),\\n },\\n ],\\n gender: dataValue(\"case.gender\"),\\n birthdate: dataValue(\"case.dob\"),\\n },\\n })\\n);\\n```\\n\\nThis example assumes you\\'re using the OpenMRS adaptor. You\\'d need to adjust the paths (e.g., `case.first_name`) to match your actual CommCare data structure.\\n\\nRemember to configure your OpenMRS instance to have custom attributes that match what you\\'re sending from CommCare. The `attributeType` should match the name or UUID of the custom attribute in OpenMRS.', 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to map CommCare case properties to OpenMRS patient attributes, including code examples and key steps. It directly addresses the question asked, and it doesn't contain the invalid pattern `)(state)`.\"], ['How can I map CommCare case data to create multiple FHIR patients in a single job?', 'To map CommCare case data to create multiple FHIR patients in a single job, you can use the `each()` function to iterate over the CommCare cases and create a FHIR patient for each one. Here\\'s an example of how you could structure your job:\\n\\n```javascript\\neach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patient = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcare.org/case-id\",\\n value: caseData.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }\\n ],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n { body: patient },\\n {\\n headers: {\\n \"Content-Type\": \"application/fhir+json\"\\n }\\n }\\n )(state);\\n })\\n);\\n```\\n\\nThis job does the following:\\n\\n1. Uses `each()` to iterate over all cases in the CommCare data.\\n2. For each case, it creates a FHIR Patient resource object.\\n3. It then uses a POST request to create the patient in your FHIR server.\\n\\nMake sure to replace \"https://your-fhir-server.com/Patient\" with the actual URL of your FHIR server\\'s Patient endpoint.\\n\\nYou may need to adjust the mapping based on your specific CommCare case structure and the FHIR Patient resource requirements. Also, consider adding error handling and logging as needed for your use case.', 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\", \"To create a new ODK form submission using data from multiple CommCare cases, you'll need to use a combination of functions from the Google Sheets adaptor and some data manipulation. Here's an example of how you might structure this:\\n\\n```javascript\\n// Assuming you have already fetched CommCare case data into state.data.cases\\n\\n// First, transform the CommCare case data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_Field_1': caseData.commcare_field_1,\\n 'ODK_Field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n \\n return { ...state, odkData };\\n});\\n\\n// Now append the transformed data to a Google Sheet\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData,\\n});\\n```\\n\\nThis code does the following:\\n\\n1. Uses `fn()` to transform the CommCare case data into the format expected by ODK.\\n2. Maps each CommCare case to an object with ODK field names.\\n3. Stores the transformed data in `state.odkData`.\\n4. Uses `appendValues()` to add the transformed data to a Google Sheet, which can then be used as a data source for ODK.\\n\\nRemember to adjust the field mappings, spreadsheet ID, and range according to your specific CommCare and ODK form structures.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to create ODK form submissions using CommCare case data, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer addresses the question directly, so it's successful.\"], ['How do I transform a Google Sheets row into a CommCare case update?', \"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format required by CommCare\\n3. Send the update to CommCare\\n\\nHere's a basic outline of how you might structure this job:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A2:E', state => {\\n const rows = state.data.values;\\n \\n // Transform and send each row to CommCare\\n return each(rows, state => {\\n const [caseId, name, age, status] = state.data;\\n \\n // Prepare the case update\\n const caseUpdate = {\\n case_id: caseId,\\n properties: {\\n name: name,\\n age: age,\\n status: status\\n }\\n };\\n \\n // Here you would use a CommCare adaptor function to send the update\\n // This is a placeholder as the actual function depends on the CommCare adaptor\\n // return updateCase(caseUpdate);\\n })(state);\\n});\\n```\\n\\nNote that this is a skeleton and you'll need to adjust it based on:\\n\\n1. The actual structure of your Google Sheet\\n2. The specific fields you're updating in CommCare\\n3. The actual function names and parameters for the CommCare adaptor\\n\\nAlso, you'll need to ensure you have the CommCare adaptor installed and configured in your project alongside the Google Sheets adaptor.\\n\\nIs there anything specific about your sheets structure or CommCare case properties that you'd like me to address?\", 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\", \"I apologize, but I don't have specific information about mapping CommCare form data to SatuSehat's FHIR resources. However, I can provide some general guidance on mapping data to FHIR resources using OpenFn and Google Sheets:\\n\\n1. First, you'll want to extract the relevant CommCare form data, likely using the CommCare adaptor for OpenFn.\\n\\n2. Then, you can use Google Sheets as an intermediary to map the CommCare data fields to the corresponding FHIR resource fields. \\n\\n3. You could create a Google Sheet with columns for the CommCare fields and corresponding FHIR fields.\\n\\n4. Use the Google Sheets adaptor to read this mapping information:\\n\\n```javascript\\ngetValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B100')\\n```\\n\\n5. Then use this mapping to transform your CommCare data into the proper FHIR format:\\n\\n```javascript\\nfn(state => {\\n const mappings = state.data.values;\\n const commcareData = state.data.commcareForm;\\n \\n const fhirResource = {};\\n \\n mappings.forEach(([commcareField, fhirField]) => {\\n if (commcareData[commcareField]) {\\n fhirResource[fhirField] = commcareData[commcareField];\\n }\\n });\\n \\n return { ...state, fhirResource };\\n})\\n```\\n\\n6. Finally, you'd use an HTTP adaptor or a specific FHIR adaptor (if available) to send this data to SatuSehat.\\n\\nThis is a general approach and would need to be adjusted based on the specific structure of your CommCare forms and the requirements of SatuSehat's FHIR implementation. You may need to handle nested data structures and data type conversions as well.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of a general approach to mapping data from CommCare to FHIR resources, including specific steps and code examples. While it acknowledges not having specific information about SatuSehat's implementation, it still provides a relevant and helpful response to the question about mapping nested CommCare form data to FHIR resources.\"], ['How can I merge data from multiple OpenMRS observations into a single Google Sheet row?', \"To merge data from multiple OpenMRS observations into a single Google Sheet row, you can follow these steps:\\n\\n1. Fetch the OpenMRS observations data (assuming you have this data in your state).\\n2. Process and transform the data to create a single row with the desired format.\\n3. Use the `appendValues` function to add the row to your Google Sheet.\\n\\nHere's an example of how you might structure your job:\\n\\n```javascript\\nfn(state => {\\n // Assume observations are in state.data.observations\\n const observations = state.data.observations;\\n\\n // Process observations to create a single row\\n const row = processObservations(observations);\\n\\n return { ...state, row };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:Z1', // Adjust the range based on your data\\n values: [state.row],\\n});\\n\\n// Helper function to process observations\\nfunction processObservations(observations) {\\n // Initialize an object to store the processed data\\n const processedData = {\\n patientId: observations[0].person.uuid,\\n encounterDate: observations[0].obsDatetime,\\n // Add more fields as needed\\n };\\n\\n // Loop through observations and add data to processedData\\n observations.forEach(obs => {\\n // Use the concept name as the key (you might want to map these to more readable names)\\n const conceptName = obs.concept.display;\\n processedData[conceptName] = obs.value;\\n });\\n\\n // Convert the processed data object to an array (for the spreadsheet row)\\n return Object.values(processedData);\\n}\\n```\\n\\nThis code does the following:\\n\\n1. It uses `fn()` to process the observations data, creating a single row.\\n2. The `processObservations` function takes the array of observations and merges them into a single object, then converts it to an array for the spreadsheet row.\\n3. It uses `appendValues()` to add the processed row to the Google Sheet.\\n\\nMake sure to adjust the spreadsheet ID, range, and the processing logic to match your specific data structure and requirements. Also, ensure that you have the necessary permissions and configuration set up for accessing both OpenMRS and Google Sheets.\", 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\", 'To use `each()` to process multiple FHIR resources and create CommCare cases, you can follow this pattern:\\n\\n```javascript\\neach(\\n \"$.bundle.entry[*].resource\",\\n fn(state => {\\n const resource = state.data;\\n \\n // Extract relevant data from the FHIR resource\\n const caseData = {\\n case_name: resource.name[0].text,\\n case_type: \"patient\",\\n owner_id: \"some_owner_id\",\\n // Add more fields as needed\\n };\\n\\n // Create a CommCare case using the extracted data\\n return getValues(\\n state.configuration.spreadsheetId,\\n \"Cases!A:Z\",\\n {\\n valueInputOption: \"USER_ENTERED\",\\n resource: [Object.values(caseData)]\\n }\\n )(state);\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. Uses `each()` to iterate over FHIR resources in the bundle.\\n2. Inside `each()`, we use `fn()` to process each resource.\\n3. We extract relevant data from the FHIR resource and create a `caseData` object.\\n4. We then use `getValues()` to add a new row to a Google Sheet, which can represent our CommCare cases.\\n\\nMake sure to adjust the field names and data extraction logic based on your specific FHIR resources and CommCare case structure.', 'v2', 'Regex Match: False\\nSuccess: True'], ['How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?', 'To map an array of ODK repeat groups to create multiple OpenMRS encounters using the Google Sheets adaptor, you can use the `each` function to iterate over the repeat group data and create encounters for each item. Here\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.form.repeat_group[*]\",\\n fn(state => {\\n const repeatData = state.data;\\n \\n // Prepare the encounter data\\n const encounterData = {\\n spreadsheetId: \"your_spreadsheet_id_here\",\\n range: \"Encounters!A:Z\", // Adjust the range as needed\\n values: [\\n [\\n repeatData.encounter_date,\\n repeatData.patient_id,\\n repeatData.encounter_type,\\n // Add more fields as needed\\n ]\\n ]\\n };\\n\\n // Append the encounter data to the Google Sheet\\n return appendValues(encounterData)(state)\\n .then(state => {\\n console.log(\"Encounter created successfully\");\\n return state;\\n });\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. Uses `each` to iterate over items in the repeat group, which are assumed to be at `$.form.repeat_group[*]` in your source data.\\n\\n2. For each item in the repeat group, it prepares encounter data in a format suitable for Google Sheets.\\n\\n3. It then uses `appendValues` to add this data as a new row in your Google Sheet.\\n\\n4. The `fn` wrapper allows you to perform multiple operations and transformations for each item.\\n\\n5. After appending values, it logs a success message and returns the updated state.\\n\\nRemember to replace \"your_spreadsheet_id_here\" with your actual Google Sheets spreadsheet ID, and adjust the range and data fields to match your specific requirements.\\n\\nAlso, ensure that your Google Sheets is set up with appropriate columns to receive this encounter data.\\n\\nIf you need to perform any data transformations or add error handling, you can do so within the `fn` function before calling `appendValues`.', 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\", \"I apologize, but there seems to be a misunderstanding in your question. The Google Sheets adaptor for OpenFn does not have built-in functionality to directly send data to SatuSehat. However, I can explain how you might filter Google Sheets data before using it in a subsequent operation.\\n\\nTo filter data from Google Sheets, you would typically:\\n\\n1. Fetch the data using `getValues()`\\n2. Use `fn()` to filter the data\\n3. Use the filtered data in your next operation\\n\\nHere's an example of how you might structure this:\\n\\n```javascript\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E100')\\n.fn(state => {\\n const allData = state.data.values;\\n \\n // Filter the data\\n const filteredData = allData.filter(row => {\\n // Add your filtering logic here\\n // For example, to filter rows where the first column is not empty:\\n return row[0] !== '';\\n });\\n\\n return { ...state, filteredData };\\n})\\n// Use the filtered data in your next operation\\n// For example, if you were sending to another system:\\n// .then(postToAnotherSystem(state => state.filteredData))\\n```\\n\\nRemember to replace 'your-spreadsheet-id' with your actual Google Sheets ID, and adjust the range ('Sheet1!A1:E100') as needed.\\n\\nThe filtering logic in this example is very simple - you would replace it with whatever criteria you need to filter your data.\\n\\nIf you need to send this data to SatuSehat, you would need to use an appropriate adaptor or HTTP requests to interact with SatuSehat's API after filtering the Google Sheets data.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to filter Google Sheets data, which addresses the core question about filtering Google Sheets data, even though it clarifies that direct integration with SatuSehat isn't built-in. The pattern `)(state)` is not present in the answer, and the response provides useful, relevant information about data filtering in Google Sheets.\"], ['How can I use combine() to merge data from multiple sources before creating a FHIR resource?', \"To use `combine()` to merge data from multiple sources before creating a FHIR resource, you can follow this approach:\\n\\n1. Use `getValues()` to fetch data from different Google Sheets.\\n2. Use `combine()` to merge the results of those operations.\\n3. Use `fn()` to process the combined data and create your FHIR resource.\\n\\nHere's an example:\\n\\n```javascript\\ncombine(\\n getValues('spreadsheetId1', 'Sheet1!A1:C10'),\\n getValues('spreadsheetId2', 'Sheet2!A1:D5')\\n)\\n\\nfn(state => {\\n const [data1, data2] = state.references;\\n \\n // Process and merge the data\\n const mergedData = {\\n // Combine data from both sources\\n ...data1[0],\\n ...data2[0]\\n };\\n\\n // Create your FHIR resource\\n const fhirResource = {\\n resourceType: 'Patient',\\n name: [{\\n given: [mergedData.firstName],\\n family: mergedData.lastName\\n }],\\n // Add more FHIR fields as needed\\n };\\n\\n console.log('Created FHIR resource:', fhirResource);\\n\\n return { ...state, fhirResource };\\n})\\n```\\n\\nThis approach allows you to fetch data from multiple Google Sheets, combine the operations, and then process the results to create your FHIR resource.\", 'v2', \"Let me evaluate this:\\n\\n1. First, checking if the answer addresses the question about using combine() to merge data:\\nThe answer provides a detailed explanation of using combine() with multiple data sources and includes a code example, so it does answer the question.\\n\\n2. Checking for the regex pattern `)(state)`:\\nLooking through the answer, while it contains `state` in various places, it doesn't contain the specific pattern `)(state)`.\\n\\nTherefore:\\nRegex Match: False\\nSuccess: True\\n\\nThe final answer is: True\"], [\"What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\", 'To map CommCare case properties to custom OpenMRS patient attributes, you\\'ll typically use the `each()` function to iterate over CommCare cases, then use `createPatient()` or `updatePatient()` operations from the OpenMRS adaptor to create or update patients with the mapped attributes. \\n\\nHere\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.forms[*]\", // Assuming CommCare data is in state.data.forms\\n fn(state => {\\n const commCareCase = state.data;\\n \\n return createPatient({\\n person: {\\n names: [\\n {\\n givenName: dataValue(\"first_name\")(state),\\n familyName: dataValue(\"last_name\")(state)\\n }\\n ],\\n gender: dataValue(\"gender\")(state),\\n birthdate: dataValue(\"dob\")(state),\\n attributes: [\\n {\\n attributeType: \"00000000-0000-0000-0000-000000000000\", // Replace with actual UUID\\n value: dataValue(\"custom_field_1\")(state)\\n },\\n {\\n attributeType: \"11111111-1111-1111-1111-111111111111\", // Replace with actual UUID\\n value: dataValue(\"custom_field_2\")(state)\\n }\\n // Add more attribute mappings as needed\\n ]\\n }\\n })(state);\\n })\\n);\\n```\\n\\nKey points:\\n1. Use `each()` to iterate over CommCare cases.\\n2. Inside `each()`, use `fn()` to access the current case data.\\n3. Use `createPatient()` (or `updatePatient()` if updating existing patients) from the OpenMRS adaptor.\\n4. Map CommCare properties to OpenMRS fields using `dataValue()`.\\n5. For custom attributes, you\\'ll need to know the UUID of the attribute type in OpenMRS.\\n\\nRemember to replace the attribute type UUIDs with the actual UUIDs from your OpenMRS instance. Also, adjust the data paths (like \"$.forms[*]\") to match your actual CommCare data structure.', 'v2', 'Regex Match: True (the pattern `)(state)` appears in the code)\\nSuccess: False']]\n" + ] + } + ], + "source": [ + "print(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 340, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "V1 success rate: 0.8888888888888888, V2 success rate: 0.9\n" ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questionanswersystem_prompt_versionresultcode
0How can I map CommCare case data to create mul...To map CommCare case data to create multiple F...v1NaNeach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...
1What's the syntax to create a new ODK form sub...To create a new ODK form submission using data...v1Falseeach(\\n \"$.cases[*]\",\\n appendValues({\\n ...
2How do I transform a Google Sheets row into a ...To transform a Google Sheets row into a CommCa...v1True// Fetch data from Google Sheets\\ngetValues('y...
3What's the best way to map nested CommCare for...I apologize, but I can't provide specific advi...v1TrueNone
4How can I merge data from multiple OpenMRS obs...To merge data from multiple OpenMRS observatio...v1Truefn(state => {\\n const patients = state.data.m...
5What's the correct way to use each() to proces...To use `each()` to process multiple FHIR resou...v1Trueeach(\\n \"$.bundle.entry[*].resource\",\\n fn(s...
6How do I map an array of ODK repeat groups to ...To map an array of ODK repeat groups to create...v1Trueeach(\\n \"$.form.repeat_group[*]\",\\n fn(state...
7What's the syntax for filtering Google Sheets ...I apologize, but there's a misunderstanding in...v1TruegetValues('your-spreadsheet-id', 'Sheet1!A1:E1...
8How can I use combine() to merge data from mul...To use `combine()` to merge data from multiple...v1Truecombine(\\n getValues('spreadsheetId1', 'Sheet...
9What's the proper way to map CommCare case pro...To map CommCare case properties to custom Open...v1Trueeach(\\n \"$.forms[*]\", // or wherever your Com...
10How can I map CommCare case data to create mul...To map CommCare case data to create multiple F...v2Trueeach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...
11What's the syntax to create a new ODK form sub...To create a new ODK form submission using data...v2True// Assuming you have already fetched CommCare ...
12How do I transform a Google Sheets row into a ...To transform a Google Sheets row into a CommCa...v2True// Fetch data from Google Sheets\\ngetValues('y...
13What's the best way to map nested CommCare for...I apologize, but I don't have specific informa...v2TruegetValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B1...
14How can I merge data from multiple OpenMRS obs...To merge data from multiple OpenMRS observatio...v2Truefn(state => {\\n // Assume observations are in...
15What's the correct way to use each() to proces...To use `each()` to process multiple FHIR resou...v2Trueeach(\\n \"$.bundle.entry[*].resource\",\\n fn(s...
16How do I map an array of ODK repeat groups to ...To map an array of ODK repeat groups to create...v2Trueeach(\\n \"$.form.repeat_group[*]\",\\n fn(state...
17What's the syntax for filtering Google Sheets ...I apologize, but there seems to be a misunders...v2TruegetValues('your-spreadsheet-id', 'Sheet1!A1:E1...
18How can I use combine() to merge data from mul...To use `combine()` to merge data from multiple...v2Truecombine(\\n getValues('spreadsheetId1', 'Sheet...
19What's the proper way to map CommCare case pro...To map CommCare case properties to custom Open...v2Falseeach(\\n \"$.forms[*]\", // Assuming CommCare da...
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 How can I map CommCare case data to create mul... \n", + "1 What's the syntax to create a new ODK form sub... \n", + "2 How do I transform a Google Sheets row into a ... \n", + "3 What's the best way to map nested CommCare for... \n", + "4 How can I merge data from multiple OpenMRS obs... \n", + "5 What's the correct way to use each() to proces... \n", + "6 How do I map an array of ODK repeat groups to ... \n", + "7 What's the syntax for filtering Google Sheets ... \n", + "8 How can I use combine() to merge data from mul... \n", + "9 What's the proper way to map CommCare case pro... \n", + "10 How can I map CommCare case data to create mul... \n", + "11 What's the syntax to create a new ODK form sub... \n", + "12 How do I transform a Google Sheets row into a ... \n", + "13 What's the best way to map nested CommCare for... \n", + "14 How can I merge data from multiple OpenMRS obs... \n", + "15 What's the correct way to use each() to proces... \n", + "16 How do I map an array of ODK repeat groups to ... \n", + "17 What's the syntax for filtering Google Sheets ... \n", + "18 How can I use combine() to merge data from mul... \n", + "19 What's the proper way to map CommCare case pro... \n", + "\n", + " answer system_prompt_version \\\n", + "0 To map CommCare case data to create multiple F... v1 \n", + "1 To create a new ODK form submission using data... v1 \n", + "2 To transform a Google Sheets row into a CommCa... v1 \n", + "3 I apologize, but I can't provide specific advi... v1 \n", + "4 To merge data from multiple OpenMRS observatio... v1 \n", + "5 To use `each()` to process multiple FHIR resou... v1 \n", + "6 To map an array of ODK repeat groups to create... v1 \n", + "7 I apologize, but there's a misunderstanding in... v1 \n", + "8 To use `combine()` to merge data from multiple... v1 \n", + "9 To map CommCare case properties to custom Open... v1 \n", + "10 To map CommCare case data to create multiple F... v2 \n", + "11 To create a new ODK form submission using data... v2 \n", + "12 To transform a Google Sheets row into a CommCa... v2 \n", + "13 I apologize, but I don't have specific informa... v2 \n", + "14 To merge data from multiple OpenMRS observatio... v2 \n", + "15 To use `each()` to process multiple FHIR resou... v2 \n", + "16 To map an array of ODK repeat groups to create... v2 \n", + "17 I apologize, but there seems to be a misunders... v2 \n", + "18 To use `combine()` to merge data from multiple... v2 \n", + "19 To map CommCare case properties to custom Open... v2 \n", + "\n", + " result code \n", + "0 NaN each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... \n", + "1 False each(\\n \"$.cases[*]\",\\n appendValues({\\n ... \n", + "2 True // Fetch data from Google Sheets\\ngetValues('y... \n", + "3 True None \n", + "4 True fn(state => {\\n const patients = state.data.m... \n", + "5 True each(\\n \"$.bundle.entry[*].resource\",\\n fn(s... \n", + "6 True each(\\n \"$.form.repeat_group[*]\",\\n fn(state... \n", + "7 True getValues('your-spreadsheet-id', 'Sheet1!A1:E1... \n", + "8 True combine(\\n getValues('spreadsheetId1', 'Sheet... \n", + "9 True each(\\n \"$.forms[*]\", // or wherever your Com... \n", + "10 True each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... \n", + "11 True // Assuming you have already fetched CommCare ... \n", + "12 True // Fetch data from Google Sheets\\ngetValues('y... \n", + "13 True getValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B1... \n", + "14 True fn(state => {\\n // Assume observations are in... \n", + "15 True each(\\n \"$.bundle.entry[*].resource\",\\n fn(s... \n", + "16 True each(\\n \"$.form.repeat_group[*]\",\\n fn(state... \n", + "17 True getValues('your-spreadsheet-id', 'Sheet1!A1:E1... \n", + "18 True combine(\\n getValues('spreadsheetId1', 'Sheet... \n", + "19 False each(\\n \"$.forms[*]\", // Assuming CommCare da... " + ] + }, + "execution_count": 340, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "new_answer = generate_new_format(\n", - " 'Using an array of people, map it to create a single sheet for each person', system_prompt_v2)\n", - "print(new_answer)" + "import pandas as pd\n", + "import os\n", + "import re\n", + "\n", + "def extract_success(result_text):\n", + " # Use regex to extract \"Success: True\" or \"Success: False\"\n", + " import re\n", + " match = re.search(r'Success:\\s*(True|False)', result_text)\n", + " return match.group(1) if match else None\n", + "\n", + "def extract_code(answer_text):\n", + " # Regex to match code inside ```javascript ... ```\n", + " match = re.search(r\"```javascript\\s*([\\s\\S]*?)\\s*```\", answer_text)\n", + " return match.group(1).strip() if match else None\n", + "\n", + "\n", + "results_df = pd.DataFrame(results, columns=[\"question\", \"answer\", \"system_prompt_version\", \"result\"])\n", + "results_df['code'] = results_df['answer'].apply(extract_code)\n", + "results_df['result'] = results_df['result'].apply(extract_success).map({'True': True, 'False': False})\n", + "v1_mean = results_df[results_df['system_prompt_version'] == 'v1']['result'].mean()\n", + "v2_mean = results_df[results_df['system_prompt_version'] == 'v2']['result'].mean()\n", + "\n", + "print(f\"V1 success rate: {v1_mean}, V2 success rate: {v2_mean}\")\n", + "downloads_path = os.path.join(os.path.expanduser(\"~\"), \"Downloads\", \"202411_claude_system_prompt_issue_97.csv\")\n", + "results_df.to_csv(downloads_path, index=False)\n", + "\n", + "results_df" ] }, { From a0bc5d51b6932aa25320b31fe361e51215934827 Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Sat, 7 Dec 2024 20:50:23 +0300 Subject: [PATCH 3/6] fix: manually check for nested state Signed-off-by: Hunter Achieng --- .../evaluation/test_nested_state.ipynb | 809 +++++++++++++++ .../evaluation/test_prompts_issue_97_a.ipynb | 966 ------------------ ..._97.ipynb => test_related_questions.ipynb} | 0 3 files changed, 809 insertions(+), 966 deletions(-) create mode 100644 services/job_chat/evaluation/test_nested_state.ipynb delete mode 100644 services/job_chat/evaluation/test_prompts_issue_97_a.ipynb rename services/job_chat/evaluation/{test_prompts_issue_97.ipynb => test_related_questions.ipynb} (100%) diff --git a/services/job_chat/evaluation/test_nested_state.ipynb b/services/job_chat/evaluation/test_nested_state.ipynb new file mode 100644 index 0000000..22a6ba4 --- /dev/null +++ b/services/job_chat/evaluation/test_nested_state.ipynb @@ -0,0 +1,809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test Prompts for issue 97:a\n", + "\n", + "This notebook will be used to test prompt versions on the AI Assistant to address problem a on issue 97" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: python-dotenv in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (1.0.1)\n", + "Requirement already satisfied: anthropic in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (0.39.0)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.6.2.post1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.27.2)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.8.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.10.19)\n", + "Requirement already satisfied: sniffio in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.3.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.12.2)\n", + "Requirement already satisfied: idna>=2.8 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anyio<5,>=3.5.0->anthropic) (3.10)\n", + "Requirement already satisfied: certifi in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (2024.8.30)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->anthropic) (0.14.0)\n", + "Requirement already satisfied: tenacity in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (9.0.0)\n", + "Requirement already satisfied: apollo in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (4.2.13)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (2.32.3)\n", + "Requirement already satisfied: biopython>=1.77 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.84)\n", + "Requirement already satisfied: cachetools<4 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (3.1.1)\n", + "Requirement already satisfied: click>=6.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (8.1.7)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.17.0)\n", + "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (6.0.2)\n", + "Requirement already satisfied: decorator in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (5.1.1)\n", + "Requirement already satisfied: bcbio-gff>=0.6.6 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (0.7.1)\n", + "Requirement already satisfied: six in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from bcbio-gff>=0.6.6->apollo) (1.16.0)\n", + "Requirement already satisfied: numpy in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from biopython>=1.77->apollo) (2.1.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2024.8.30)\n", + "Requirement already satisfied: pandas in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (2.2.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.1.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2024.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install python-dotenv\n", + "!pip install anthropic\n", + "!pip install tenacity\n", + "!pip install apollo\n", + "!pip install pandas\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "import anthropic\n", + "\n", + "load_dotenv()\n", + "ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\")\n", + "\n", + "client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Sample user prompt\n", + "user_prompt = \"\"\"\n", + "Generate 10 short code questions that a user might ask about openFn job writing that makes use of different adaptor functions and mappings. \n", + "\n", + "The questions can span around different platforms such as:\n", + "\n", + "commcare\n", + "satusehat\n", + "googlesheets\n", + "fhir\n", + "openmrs\n", + "odk\n", + "\n", + "Output your answers with new lines and without numbers or bullet points.\n", + "\n", + "\n", + "Using an array of people, map it to create a single sheet for each person\n", + "\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + "\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + "\n", + "How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record?\n", + "\n", + "What's the best way to transform an array of CommCare referrals into Satu Sehat encounters?\n", + "\n", + "How can I split a single FHIR Bundle resource into multiple Google Sheets rows?\n", + "\n", + "What's the correct way to map nested CommCare form data to FHIR Observation resources?\n", + "\n", + "How do I create multiple Google Sheets worksheets from a single OpenMRS patient query?\n", + "\n", + "What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources?\n", + "\n", + "How can I combine data from multiple Satu Sehat resources to create a single CommCare case?\n", + "\n", + "What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy?\n" + ] + } + ], + "source": [ + "# Generate job code\n", + "\n", + "message = client.messages.create(\n", + " model=\"claude-3-5-sonnet-20241022\", # TODO change to cheaper model\n", + " max_tokens=1000,\n", + " temperature=0,\n", + " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": user_prompt\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + ")\n", + "print(message.content[0].text)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Create list from generated questions\n", + "\n", + "generated_questions = [q for q in message.content[0].text.split(\"\\n\") if q]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/Users/hunter/apollo/services\n" + ] + } + ], + "source": [ + "import sys\n", + "import os\n", + "\n", + "# Add the parent directory to sys.path\n", + "parent_dir = os.path.abspath(\"../../\")\n", + "print(parent_dir)\n", + "sys.path.append(parent_dir)\n", + "\n", + "\n", + "\n", + "from job_chat.prompt import job_writing_summary, system_role, build_prompt\n", + "from util import DictObj\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# New edited system prompt\n", + "\n", + "\n", + "\n", + "# formatted like in apollo\n", + "system_prompt_v2 = [{'type': 'text', 'text': system_role}, {'type': 'text', 'text': job_writing_summary}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': 'The user is using the OpenFn @openfn/language-googlesheets@3.0.5 adaptor. Use functions provided by its API.Typescript definitions for doc @openfn/language-common/**\\n * Execute a sequence of operations.\\n * Main outer API for executing expressions.\\n * @public\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Promise}\\n */\\nexport function execute(...operations: Operations): Promise;\\n/**\\n * alias for \"fn()\"\\n * @function\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function alterState(func: Function): Operation;\\n/**\\n * Creates a custom step (or operation) for more flexible job writing.\\n * @public\\n * @function\\n * @example\\n * fn(state => {\\n * // do some things to state\\n * return state;\\n * });\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function fn(func: Function): Operation;\\n/**\\n * A custom operation that will only execute the function if the condition returns true\\n * @public\\n * @function\\n * @example\\n * fnIf((state) => state?.data?.name, get(\"https://example.com\"));\\n * @param {Boolean} condition - The condition that returns true\\n * @param {Operation} operation - The operation needed to be executed.\\n * @returns {Operation}\\n */\\nexport function fnIf(condition: boolean, operation: Operation): Operation;\\n/**\\n * Picks out a single value from a JSON object.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * jsonValue({ a:1 }, \\'a\\')\\n * @param {object} obj - A valid JSON object.\\n * @param {String} path - JSONPath referencing a point in given JSON object.\\n * @returns {Operation}\\n */\\nexport function jsonValue(obj: object, path: string): Operation;\\n/**\\n * Picks out a single value from source data.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * sourceValue(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Operation}\\n */\\nexport function sourceValue(path: string): Operation;\\n/**\\n * Picks out a value from source data.\\n * Will return whatever JSONPath returns, which will always be an array.\\n * If you need a single value use `sourceValue` instead.\\n * @public\\n * @function\\n * @example\\n * source(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Array.}\\n */\\nexport function source(path: string): Array;\\n/**\\n * Ensures a path points at the data.\\n * @public\\n * @function\\n * @example\\n * dataPath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `data`.\\n * @returns {string}\\n */\\nexport function dataPath(path: string): string;\\n/**\\n * Picks out a single value from the source data object—usually `state.data`.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * dataValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `data`.\\n * @returns {Operation}\\n */\\nexport function dataValue(path: string): Operation;\\n/**\\n * Ensures a path points at references.\\n * @public\\n * @function\\n * @example\\n * referencePath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `references`.\\n * @returns {string}\\n */\\nexport function referencePath(path: string): string;\\n/**\\n * Picks out the last reference value from source data.\\n * @public\\n * @function\\n * @example\\n * lastReferenceValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `references`.\\n * @returns {Operation}\\n */\\nexport function lastReferenceValue(path: string): Operation;\\n/**\\n * Simple switcher allowing other expressions to use either a JSONPath or\\n * object literals as a data source.\\n * - JSONPath referencing a point in `state`\\n * - Object Literal of the data itself.\\n * - Function to be called with state.\\n * @public\\n * @function\\n * @example\\n * asData(\\'$.key\\'| key | callback)\\n * @param {String|object|function} data\\n * @param {object} state - The current state.\\n * @returns {array}\\n */\\nexport function asData(data: string | object | Function, state: object): any[];\\n/**\\n * Iterates over an array of items and invokes an operation upon each one, where the state\\n * object is _scoped_ so that state.data is the item under iteration.\\n * The rest of the state object is untouched and can be referenced as usual.\\n * You can pass an array directly, or use lazy state or a JSONPath string to\\n * reference a slice of state.\\n * @public\\n * @function\\n * @example Using lazy state ($) to iterate over items in state.data and pass each into an \"insert\" operation\\n * each(\\n * $.data,\\n * // Inside the callback operation, `$.data` is scoped to the item under iteration\\n * insert(\"patient\", {\\n * patient_name: $.data.properties.case_name,\\n * patient_id: $.data.case_id,\\n * })\\n * );\\n * @example Iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * $.data,\\n * insert(\"patient\", (state) => ({\\n * patient_id: state.data.case_id,\\n * ...state.data\\n * }))\\n * );\\n * @example Using JSON path to iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * \"$.data[*]\",\\n * insert(\"patient\", (state) => ({\\n * patient_name: state.data.properties.case_name,\\n * patient_id: state.data.case_id,\\n * }))\\n * );\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n/**\\n * Combines two operations into one\\n * @public\\n * @function\\n * @example\\n * combine(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function combine(...operations: Operations): Operation;\\n/**\\n * Adds data from a target object\\n * @public\\n * @function\\n * @example\\n * join(\\'$.key\\',\\'$.data\\',\\'newKey\\')\\n * @param {String} targetPath - Target path\\n * @param {String} sourcePath - Source path\\n * @param {String} targetKey - Target Key\\n * @returns {Operation}\\n */\\nexport function join(targetPath: string, sourcePath: string, targetKey: string): Operation;\\n/**\\n * Recursively resolves objects that have resolvable values (functions).\\n * @public\\n * @function\\n * @param {object} value - data\\n * @param {Function} [skipFilter] - a function which returns true if a value should be skipped\\n * @returns {Operation}\\n */\\nexport function expandReferences(value: object, skipFilter?: Function): Operation;\\n/**\\n * Returns a key, value pair in an array.\\n * @public\\n * @function\\n * @example\\n * field(\\'destination_field_name__c\\', \\'value\\')\\n * @param {string} key - Name of the field\\n * @param {Value} value - The value itself or a sourceable operation.\\n * @returns {Field}\\n */\\nexport function field(key: string, value: Value): Field;\\n/**\\n * Zips key value pairs into an object.\\n * @public\\n * @function\\n * @example\\n * fields(list_of_fields)\\n * @param {Fields} fields - a list of fields\\n * @returns {Object}\\n */\\nexport function fields(...fields: Fields): any;\\n/**\\n * Merges fields into each item in an array.\\n * @public\\n * @example\\n * merge(\\n * \"$.books[*]\",\\n * fields(\\n * field( \"publisher\", sourceValue(\"$.publisher\") )\\n * )\\n * )\\n * @function\\n * @public\\n * @param {DataSource} dataSource\\n * @param {Object} fields - Group of fields to merge in.\\n * @returns {DataSource}\\n */\\nexport function merge(dataSource: DataSource, fields: any): DataSource;\\n/**\\n * Groups an array of objects by a specified key path.\\n * @public\\n * @example\\n * const users = [\\n * { name: \\'Alice\\', age: 25, city: \\'New York\\' },\\n * { name: \\'Bob\\', age: 30, city: \\'San Francisco\\' },\\n * { name: \\'Charlie\\', age: 25, city: \\'New York\\' },\\n * { name: \\'David\\', age: 30, city: \\'San Francisco\\' }\\n * ];\\n * group(users, \\'city\\');\\n * // state is { data: { \\'New York\\': [/Alice, Charlie/], \\'San Francisco\\': [ /Bob, David / ] }\\n * @function\\n * @public\\n * @param {Object[]} arrayOfObjects - The array of objects to be grouped.\\n * @param {string} keyPath - The key path to group by.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function group(arrayOfObjects: any[], keyPath: string, callback?: Function): Operation;\\n/**\\n * Returns the index of the current array being iterated.\\n * To be used with `each` as a data source.\\n * @public\\n * @function\\n * @example\\n * index()\\n * @returns {DataSource}\\n */\\nexport function index(): DataSource;\\n/**\\n * Turns an array into a string, separated by X.\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", function(state) {\\n * return arrayToString(dataValue(\"path_of_array\")(state), \\', \\')\\n * })\\n * @param {array} arr - Array of toString\\'able primatives.\\n * @param {string} separator - Separator string.\\n * @returns {string}\\n */\\nexport function arrayToString(arr: any[], separator: string): string;\\n/**\\n * Ensures primitive data types are wrapped in an array.\\n * Does not affect array objects.\\n * @public\\n * @function\\n * @example\\n * each(function(state) {\\n * return toArray( dataValue(\"path_of_array\")(state) )\\n * }, ...)\\n * @param {any} arg - Data required to be in an array\\n * @returns {array}\\n */\\nexport function toArray(arg: any): any[];\\n/**\\n * Prepares next state\\n * @public\\n * @function\\n * @example\\n * composeNextState(state, response)\\n * @param {State} state - state\\n * @param {Object} response - Response to be added\\n * @returns {State}\\n */\\nexport function composeNextState(state: State, response: any): State;\\n/**\\n * Substitutes underscores for spaces and proper-cases a string\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", humanProper(state.data.path_to_string))\\n * @param {string} str - String that needs converting\\n * @returns {string}\\n */\\nexport function humanProper(str: string): string;\\n/**\\n * Splits an object into two objects based on a list of keys.\\n * The first object contains the keys that are not in the list,\\n * and the second contains the keys that are.\\n * @public\\n * @function\\n * @param {Object} obj - The object to split.\\n * @param {string[]} keys - List of keys to split on.\\n * @returns {Object[]} - Tuple of objects, first object contains keys not in list, second contains keys that are.\\n */\\nexport function splitKeys(obj: any, keys: string[]): any[];\\n/**\\n * Replaces emojis in a string.\\n * @public\\n * @function\\n * @example\\n * scrubEmojis(\\'Dove🕊️⭐ 29\\')\\n * @param {string} text - String that needs to be cleaned\\n * @param {string} replacementChars - Characters that replace the emojis\\n * @returns {string}\\n */\\nexport function scrubEmojis(text: string, replacementChars: string): string;\\n/**\\n * Chunks an array into an array of arrays, each with no more than a certain size.\\n * @public\\n * @function\\n * @example\\n * chunk([1,2,3,4,5], 2)\\n * @param {Object} array - Array to be chunked\\n * @param {Integer} chunkSize - The maxiumum size of each chunks\\n * @returns {Object}\\n */\\nexport function chunk(array: any, chunkSize: Integer): any;\\n/**\\n * Takes a CSV file string or stream and parsing options as input, and returns a promise that\\n * resolves to the parsed CSV data as an array of objects.\\n * Options for `parsingOptions` include:\\n * - `delimiter` {string/Buffer/[string/Buffer]} - Defines the character(s) used to delineate the fields inside a record. Default: `\\',\\'`\\n * - `quote` {string/Buffer/[string/Buffer]} - Defines the characters used to surround a field. Default: `\\'\"\\'`\\n * - `escape` {Buffer/string/null/boolean} - Set the escape character as one character/byte only. Default: `\"`\\n * - `columns` {boolean / array / function} - Generates record in the form of object literals. Default: `true`\\n * - `bom` {boolean} - Strips the {@link https://en.wikipedia.org/wiki/Byte_order_mark byte order mark (BOM)} from the input string or buffer. Default: `true`\\n * - `trim` {boolean} - Ignore whitespace characters immediately around the `delimiter`. Default: `true`\\n * - `ltrim` {boolean} - Ignore whitespace characters from the left side of a CSV field. Default: `true`\\n * - `rtrim` {boolean} - Ignore whitespace characters from the right side of a CSV field. Default: `true`\\n * - `chunkSize` {number} - The size of each chunk of CSV data. Default: `Infinity`\\n * - `skip_empty_lines` {boolean} - Ignore empty lines in the CSV file. Default: `true`\\n * @public\\n * @function\\n * @param {String | Stream} csvData - A CSV string or a readable stream\\n * @param {Object} [parsingOptions] - Optional. Parsing options for converting CSV to JSON.\\n * @param {function} [callback] - (Optional) callback function. If used it will be called state and an array of rows.\\n * @returns {Operation} The function returns a Promise that resolves to the result of parsing a CSV `stringOrStream`.\\n */\\nexport function parseCsv(csvData: string | Stream, parsingOptions?: any, callback?: Function): Operation;\\n/**\\n * Validate against a JSON schema. Any erors are written to an array at `state.validationErrors`.\\n * Schema can be passed directly, loaded as a JSON path from state, or loaded from a URL\\n * Data can be passed directly or loaded as a JSON path from state.\\n * By default, schema is loaded from `state.schema` and data from `state.data`.\\n * @pubic\\n * @function\\n * @param {string|object} schema - The schema, path or URL to validate against\\n * @param {string|object} data - The data or path to validate\\n * @example Validate `state.data` with `state.schema`\\n * validate()\\n * @example Validate form data at `state.form` with a schema from a URL\\n * validate(\"https://www.example.com/schema/record\", \"form\")\\n * @example Validate the each item in `state.records` with a schema from a URL\\n * each(\"records[*]\", validate(\"https://www.example.com/schema/record\"))\\n * @returns {Operation}\\n */\\nexport function validate(schema?: string | object, data?: string | object): Operation;\\n/**\\n * Sets a cursor property on state.\\n * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\\n * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones\\n * are not yet supported.\\n * You can provide a formatter to customise the final cursor value, which is useful for normalising\\n * different inputs. The custom formatter runs after natural language date conversion.\\n * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}\\n * @public\\n * @function\\n * @example Use a cursor from state if present, or else use the default value\\n * cursor($.cursor, { defaultValue: \\'today\\' })\\n * @example Use a pagination cursor\\n * cursor(22)\\n * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number\\n * @param {object} options - options to control the cursor.\\n * @param {string} options.key - set the cursor key. Will persist through the whole run.\\n * @param {any} options.defaultValue - the value to use if value is falsy\\n * @param {Function} options.format - custom formatter for the final cursor value\\n * @returns {Operation}\\n */\\nexport function cursor(value: any, options?: {\\n key: string;\\n defaultValue: any;\\n format: Function;\\n}): Operation;\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n * @public\\n * @function\\n * @example\\n * map(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\"))\\n * )\\n * )\\n * @param {string} path - JSONPath referencing a point in `state.data`.\\n * @param {function} operation - The operation needed to be repeated.\\n * @param {State} state - Runtime state.\\n * @returns {State}\\n */\\nexport const map: any;\\n\\n\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n *\\n * It also ensures the results of an operation make their way back into\\n * the state\\'s references.\\n * @public\\n * @example\\n * each(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\")))\\n * )\\n * @function\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n\\n\\nexport { parse, format } from \"date-fns\";\\n\\n\\n/**\\n * Builder function to create request options. Returns an object with helpers to\\n * easily add commonly used options. The return object is chainable so you can set\\n * as many options as you want.\\n * Pass an object to set your own options.\\n * @param {CommonRequestOptions} options - options to pass to the request\\n * @returns {OptionsHelpers}\\n * @function\\n * @public\\n * @example Get with a query an oath token\\n * get($.data.url, http.options({ query: $.query }).oath($.configuration.access_token)\\n */\\nexport function options(opts?: {}): any;\\n/**\\n * Make a GET request.\\n * @public\\n * @function\\n * @example Request a resource\\n * http.get(\\'https://jsonplaceholder.typicode.com/todos\\')\\n * @example Request a resource with basic auth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().basic(\\'user\\', \\'pass\\')\\n * )\\n * @example Request a resource with oauth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().oauth($.configuration.access_token)\\n * )\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function get(url: string, options: CommonRequestOptions): Operation;\\n/**\\n * Make a POST request.\\n * @public\\n * @function\\n * @example Post a JSON object (setting the content-type header)\\n * http.post(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * $.data,\\n * options().json(),\\n * })\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function post(path: any, data: any, options: CommonRequestOptions): Operation;\\nexport { req as request };\\n/**\\n * Helper functions provided by `http.options`.\\n */\\nexport type OptionsHelpers = any;\\n/**\\n * Options provided to the HTTP request\\n */\\nexport type CommonRequestOptions = {\\n /**\\n * - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n */\\n errors: object;\\n /**\\n * - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n */\\n form: object;\\n /**\\n * - An object of query parameters to be encoded into the URL.\\n */\\n query: object;\\n /**\\n * - An object of headers to append to the request.\\n */\\n headers: object;\\n /**\\n * - Parse the response body as json, text or stream. By default will use the response headers.\\n */\\n parseAs: string;\\n /**\\n * - Request timeout in ms. Default: 300 seconds.\\n */\\n timeout: number;\\n /**\\n * - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n tls: object;\\n};\\n/**\\n * State object\\n */\\nexport type CommonHttpState = any;\\n/**\\n * Options provided to the HTTP request\\n * @typedef {Object} CommonRequestOptions\\n * @property {object} errors - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n * @property {object} form - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n * @property {object} query - An object of query parameters to be encoded into the URL.\\n * @property {object} headers - An object of headers to append to the request.\\n * @property {string} parseAs - Parse the response body as json, text or stream. By default will use the response headers.\\n * @property {number} timeout - Request timeout in ms. Default: 300 seconds.\\n * @property {object} tls - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n/**\\n * State object\\n * @typedef {Object} CommonHttpState\\n * @private\\n * @property data - the parsed response body\\n * @property response - the response from the HTTP server, including headers, statusCode, body, etc\\n * @property references - an array of all previous data objects used in the Job\\n **/\\n/**\\n * Make a HTTP request.\\n * @public\\n * @function\\n * @example\\n * http.request(\\n * \\'GET\\',\\n * \\'https://jsonplaceholder.typicode.com/todos\\'\\n * )\\n * @name request\\n * @param {string} method - The HTTP method to use.\\n * @param {string} url - URL to resource.\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\ndeclare function req(method: string, url: string, options: CommonRequestOptions): Operation;\\n\\n\\nimport * as Adaptor from \\'./Adaptor\\';\\nexport default Adaptor;\\nexport * from \\'./Adaptor\\';\\nexport * as beta from \\'./beta\\';\\nexport * as http from \\'./http\\';\\nexport * as dateFns from \\'./dateFns\\';\\nimport * as metadata from \\'./metadata\\';\\nexport { metadata };\\n\\n\\ndeclare type Entity = {\\n name: string;\\n type: string;\\n label?: string;\\n datatype?: string;\\n desc?: string;\\n children?: Entity[] | Record;\\n meta?: Record;\\n addChild: (e: Entity, name?: string) => void;\\n};\\ndeclare type DataType = \\'string\\' | \\'boolean\\' | \\'date\\';\\n\\n\\nexport function encode(data: string): string;\\nexport function decode(base64Data: string): string;\\nexport function uuid(): string;\\n\\n\\n/**\\n * `request` is a helper function that sends HTTP requests and returns the response\\n * body, headers, and status code.\\n * Use the error map to provide custom error messages or get hold of the response in case of errors.\\n * @param method - The HTTP method to use for the request (e.g., \"GET\", \"POST\", \"PUT\", \"DELETE\", etc.).\\n * @param fullUrlOrPath - The full or partial URL for the request.\\n * @param [options] - The `options` parameter is an object that contains additional configuration\\n * options for the request.\\n * @returns an object with the following properties:\\n * - method: the request method\\n * - url: the request url\\n * - code: the status code of the response\\n * - headers: the headers of the response\\n * - body: the body of the response\\n * - message: the status text of the response\\n * - duration: the response time\\n */\\nexport function request(method: any, fullUrlOrPath: any, options?: {}): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function makeBasicAuthHeader(username: any, password: any): {\\n Authorization: string;\\n};\\nexport function logResponse(response: any): any;\\nexport function enableMockClient(baseUrl: any): import(\"undici/types/mock-interceptor\").Interceptable;\\nexport const ERROR_ABSOLUTE_URL: \"Absolute URLs not suppored\";\\nexport function assertRelativeUrl(path: any): void;\\nexport const ERROR_URL_MISMATCH: \"Target origin does not match baseUrl origin\";\\nexport function parseUrl(pathOrUrl: string, baseUrl: any): {\\n url: string;\\n baseUrl: string;\\n path: string;\\n query: any;\\n};\\nexport function get(url: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function post(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function put(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function del(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\n\\n\\nexport * from \"./http\";\\nexport * from \"./helpers\";\\nexport * from \"./references\";\\nimport parseDate from \"./parse-date\";\\nimport throwError from \"./throw-error\";\\nexport { parseDate, throwError };\\n\\n\\ndeclare function _default(d: any, startDate: any): any;\\nexport default _default;\\n\\n\\nexport function expandReferences(state: any, ...args: any[]): any[];\\nexport function normalizeOauthConfig(configuration: any): any;\\n\\n\\ndeclare function _default(code: any, { description, fix, ...extras }?: {\\n description: any;\\n fix: any;\\n}): never;\\nexport default _default;\\nTypescript definitions for doc @openfn/language-googlesheets/**\\n * Execute a sequence of oper.\\n * Wraps `language-common/execute`, and prepends initial state for http.\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )(state)\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function execute(...operations: Operations): Operation;\\n/**\\n * Add an array of rows to the spreadsheet.\\n * https://developers.google.com/sheets/api/samples/writing#append_values\\n * @public\\n * @example\\n * appendValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function appendValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Batch update values in a Spreadsheet.\\n * @example\\n * batchUpdateValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @public\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {string} [params.valueInputOption] (Optional) Value update options. Defaults to \\'USER_ENTERED\\'\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function batchUpdateValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n valueInputOption?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Gets cell values from a Spreadsheet.\\n * @public\\n * @example\\n * getValues(\\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\'Sheet1!A1:E1\\')\\n * @function\\n * @param {string} spreadsheetId The spreadsheet ID.\\n * @param {string} range The sheet range.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function getValues(spreadsheetId: string, range: string, callback?: Function): Operation;\\nexport { alterState, combine, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, lastReferenceValue, merge, sourceValue } from \"@openfn/language-common\";\\n\\n\\nexport default Adaptor;\\nexport * from \"./Adaptor\";\\nimport * as Adaptor from \"./Adaptor\";\\n'}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': '// write your job code here'}]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# Import tenacity to implement waiting after failed api calls\n", + "\n", + "from tenacity import (\n", + " retry,\n", + " stop_after_attempt,\n", + " wait_random_exponential,\n", + ") # for exponential backoff" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", + "def generate_apollo_format(content, history=\"\", adaptor=\"@openfn/language-googlesheets@3.0.5\", model=\"claude-3-5-sonnet-20240620\"):\n", + " \"\"\"Use Apollo prompt formatting templates to generate a Claude answer.\"\"\"\n", + "\n", + " context = DictObj({\n", + " \"expression\": \"// write your job code here\",\n", + " \"adaptor\": adaptor\n", + " })\n", + " print(content)\n", + " system_message, prompt = build_prompt(content, history, context)\n", + " # print(system_message)\n", + " # print(prompt)\n", + "\n", + " message = client.beta.prompt_caching.messages.create(\n", + " max_tokens=1024, messages=prompt, model=model, system=system_message\n", + " )\n", + " return message.content[0].text" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How do I transform a Google Sheets row into a CommCare case update?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "data": { + "text/plain": [ + "'To transform a Google Sheets row into a CommCare case update, you\\'ll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format required by CommCare\\n3. Send the transformed data to CommCare (using the CommCare adaptor)\\n\\nHere\\'s a high-level example of how you might structure this job:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues(\"your-spreadsheet-id\", \"Sheet1!A2:E\")\\n\\n// Transform the data\\nfn(state => {\\n const rows = state.data.values;\\n const transformedData = rows.map(row => ({\\n case_id: row[0],\\n properties: {\\n name: row[1],\\n age: row[2],\\n // Add more properties as needed\\n }\\n }));\\n return { ...state, commcareData: transformedData };\\n})\\n\\n// Then use the CommCare adaptor to update cases\\n// Note: This part would require the CommCare adaptor, not included in this example\\n// each(\\n// \"$.commcareData[*]\",\\n// updateCase({\\n// caseId: dataValue(\"case_id\"),\\n// properties: dataValue(\"properties\")\\n// })\\n// )\\n```\\n\\nThis example assumes:\\n1. Your Google Sheet has case IDs in column A, names in B, ages in C, etc.\\n2. You\\'re starting from row 2 (A2) to skip headers\\n3. You\\'ll need to adjust the range and property mapping based on your actual sheet structure\\n\\nRemember, to complete this job, you\\'d need to:\\n1. Configure your Google Sheets credentials properly\\n2. Install and configure the CommCare adaptor\\n3. Adjust the data transformation to match your exact needs\\n\\nIs there a specific part of this process you\\'d like more details on?'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "generate_apollo_format(\"How do I transform a Google Sheets row into a CommCare case update?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# Generate answers with the new system prompt\n", + "# @retry(wait=wait_random_exponential(min=1, max=90), stop=stop_after_attempt(20))\n", + "\n", + "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", + "def generate_new_format(question, new_system_prompt, model=\"claude-3-5-sonnet-20240620\"):\n", + " \"\"\"Use the apollo format to call claude with a custom system prompt.\"\"\"\n", + " prompt = [{'role': 'user', 'content': question}]\n", + " print(question)\n", + " message = client.beta.prompt_caching.messages.create(\n", + " max_tokens=1024, messages=prompt, model=model, system=new_system_prompt\n", + " )\n", + " return message.content[0].text" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the best way to transform an array of CommCare referrals into Satu Sehat encounters?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I split a single FHIR Bundle resource into multiple Google Sheets rows?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the correct way to map nested CommCare form data to FHIR Observation resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I create multiple Google Sheets worksheets from a single OpenMRS patient query?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I combine data from multiple Satu Sehat resources to create a single CommCare case?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the best way to transform an array of CommCare referrals into Satu Sehat encounters?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I split a single FHIR Bundle resource into multiple Google Sheets rows?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the correct way to map nested CommCare form data to FHIR Observation resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How do I create multiple Google Sheets worksheets from a single OpenMRS patient query?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "How can I combine data from multiple Satu Sehat resources to create a single CommCare case?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", + "What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy?\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" + ] + } + ], + "source": [ + "# Get answers to generated questions\n", + "qa_pairs = []\n", + "for question in generated_questions[:30]:\n", + " answer = generate_apollo_format(question)\n", + " qa_pairs.append((question, answer, \"v1\"))\n", + "for question in generated_questions[:30]: # two loops to make use of prompt caching\n", + " new_answer = generate_new_format(question, system_prompt_v2)\n", + " qa_pairs.append((question, new_answer, \"v2\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "evaluate_answer_template = \"\"\"\n", + "For the given Question and Answer pair, evaluate whether the answer meets the expectations based on the question's requirements.\n", + "\n", + "AThe programmatic evaluation will handle checks for patterns like `)(state)`. Your role is to ensure the provided answer logically aligns with the question's requirements and is technically valid. Do not validate for patterns such as `)(state)`; this is handled separately.\n", + "\n", + "\n", + "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", + "A: \"get('Patient)\"\n", + "True\n", + "\n", + "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", + "A: \"fn(state => {{ return {{ ...state, people: [] }}; }})(state);\"\n", + "False\n", + "\n", + "Now, following Q/A pair, by ONLY outputting \"True\" or \"False\":\n", + "Q: {question}\n", + "A: {answer}\n", + "\"\"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import re\n", + "\n", + "# Function to extract JavaScript code\n", + "def extract_code(answer_text):\n", + " \"\"\"\n", + " Extracts JavaScript code from an answer text enclosed in ```javascript ... ```.\n", + " \"\"\"\n", + " match = re.search(r\"```javascript\\s*([\\s\\S]*?)\\s*```\", answer_text)\n", + " return match.group(1).strip() if match else None\n", + "\n", + "# Function to check for `)(state)` pattern\n", + "def check_nested_state(code):\n", + " \"\"\"\n", + " Checks for the presence of the `)(state)` pattern in the code.\n", + " \"\"\"\n", + " return bool(re.search(r\"\\)\\(state\\)\", code)) if code else False\n", + "\n", + "# Evaluate QA pairs\n", + "results = []\n", + "\n", + "for q, a, version in qa_pairs:\n", + " evaluate_answer_template_filled = evaluate_answer_template.format(question=q, answer=a)\n", + "\n", + " # Send to AI for evaluation\n", + " message = client.messages.create(\n", + " model=\"claude-3-5-sonnet-20241022\",\n", + " max_tokens=1000,\n", + " temperature=0,\n", + " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": evaluate_answer_template_filled\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + " )\n", + " \n", + " # AI response and programmatic check\n", + " ai_result = message.content[0].text.strip()\n", + " code = extract_code(a)\n", + " contains_nested_state = check_nested_state(code)\n", + " \n", + " # Append results\n", + " results.append([q, code, version, contains_nested_state])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "V1 success rate: 0.80, V2 success rate: 0.70\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
questioncodesystem_prompt_versioncontains_nested_statesuccess
0How can I map CommCare case data to create mul...each(\\n \"$.forms[*].cases[*]\",\\n fn(state =>...v1TrueFalse
1What's the syntax to create a new ODK form sub...fn(state => {\\n const commcareCases = state.d...v1FalseTrue
2How do I merge data from multiple Google Sheet...// Get data from Sheet 1\\ngetValues(\\n '1O-a4...v1FalseTrue
3What's the best way to transform an array of C...each(\\n \"$.data.referrals[*]\",\\n fn(state =>...v1FalseTrue
4How can I split a single FHIR Bundle resource ...fn(state => {\\n const bundle = state.data.bun...v1FalseTrue
5What's the correct way to map nested CommCare ...each(\\n \"$.data.form.observations[*]\",\\n fn(...v1FalseTrue
6How do I create multiple Google Sheets workshe...fn(state => {\\n // Assume state.data contains...v1FalseTrue
7What's the syntax for transforming ODK repeat ...Nonev1FalseTrue
8How can I combine data from multiple Satu Seha...getValues('your_spreadsheet_id', 'Sheet1!A1:Z1...v1FalseTrue
9What's the proper way to map OpenMRS observati...each(\\n \"$.observations[*]\",\\n fn(state => {...v1TrueFalse
10How can I map CommCare case data to create mul...each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...v2TrueFalse
11What's the syntax to create a new ODK form sub...// Assuming you have data from CommCare cases ...v2FalseTrue
12How do I merge data from multiple Google Sheet...// Fetch data from Google Sheets tabs\\ngetValu...v2FalseTrue
13What's the best way to transform an array of C...// Assuming referrals are in state.data.referr...v2FalseTrue
14How can I split a single FHIR Bundle resource ...fn(state => {\\n const bundle = state.data.bun...v2FalseTrue
15What's the correct way to map nested CommCare ...each(\\n '$.data.form.observations[*]',\\n fn(...v2TrueFalse
16How do I create multiple Google Sheets workshe...fn(state => {\\n const patients = state.data.p...v2FalseTrue
17What's the syntax for transforming ODK repeat ...Nonev2FalseTrue
18How can I combine data from multiple Satu Seha...// Fetch data from Satu Sehat resources (assum...v2FalseTrue
19What's the proper way to map OpenMRS observati...each(\\n \"$.observations[*]\",\\n fn(state => {...v2TrueFalse
\n", + "
" + ], + "text/plain": [ + " question \\\n", + "0 How can I map CommCare case data to create mul... \n", + "1 What's the syntax to create a new ODK form sub... \n", + "2 How do I merge data from multiple Google Sheet... \n", + "3 What's the best way to transform an array of C... \n", + "4 How can I split a single FHIR Bundle resource ... \n", + "5 What's the correct way to map nested CommCare ... \n", + "6 How do I create multiple Google Sheets workshe... \n", + "7 What's the syntax for transforming ODK repeat ... \n", + "8 How can I combine data from multiple Satu Seha... \n", + "9 What's the proper way to map OpenMRS observati... \n", + "10 How can I map CommCare case data to create mul... \n", + "11 What's the syntax to create a new ODK form sub... \n", + "12 How do I merge data from multiple Google Sheet... \n", + "13 What's the best way to transform an array of C... \n", + "14 How can I split a single FHIR Bundle resource ... \n", + "15 What's the correct way to map nested CommCare ... \n", + "16 How do I create multiple Google Sheets workshe... \n", + "17 What's the syntax for transforming ODK repeat ... \n", + "18 How can I combine data from multiple Satu Seha... \n", + "19 What's the proper way to map OpenMRS observati... \n", + "\n", + " code system_prompt_version \\\n", + "0 each(\\n \"$.forms[*].cases[*]\",\\n fn(state =>... v1 \n", + "1 fn(state => {\\n const commcareCases = state.d... v1 \n", + "2 // Get data from Sheet 1\\ngetValues(\\n '1O-a4... v1 \n", + "3 each(\\n \"$.data.referrals[*]\",\\n fn(state =>... v1 \n", + "4 fn(state => {\\n const bundle = state.data.bun... v1 \n", + "5 each(\\n \"$.data.form.observations[*]\",\\n fn(... v1 \n", + "6 fn(state => {\\n // Assume state.data contains... v1 \n", + "7 None v1 \n", + "8 getValues('your_spreadsheet_id', 'Sheet1!A1:Z1... v1 \n", + "9 each(\\n \"$.observations[*]\",\\n fn(state => {... v1 \n", + "10 each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... v2 \n", + "11 // Assuming you have data from CommCare cases ... v2 \n", + "12 // Fetch data from Google Sheets tabs\\ngetValu... v2 \n", + "13 // Assuming referrals are in state.data.referr... v2 \n", + "14 fn(state => {\\n const bundle = state.data.bun... v2 \n", + "15 each(\\n '$.data.form.observations[*]',\\n fn(... v2 \n", + "16 fn(state => {\\n const patients = state.data.p... v2 \n", + "17 None v2 \n", + "18 // Fetch data from Satu Sehat resources (assum... v2 \n", + "19 each(\\n \"$.observations[*]\",\\n fn(state => {... v2 \n", + "\n", + " contains_nested_state success \n", + "0 True False \n", + "1 False True \n", + "2 False True \n", + "3 False True \n", + "4 False True \n", + "5 False True \n", + "6 False True \n", + "7 False True \n", + "8 False True \n", + "9 True False \n", + "10 True False \n", + "11 False True \n", + "12 False True \n", + "13 False True \n", + "14 False True \n", + "15 True False \n", + "16 False True \n", + "17 False True \n", + "18 False True \n", + "19 True False " + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results_df = pd.DataFrame(results, columns=[\"question\", \"code\", \"system_prompt_version\", \"contains_nested_state\"])\n", + "\n", + "results_df['success'] = ~results_df['contains_nested_state']\n", + "\n", + "v1_mean = results_df[results_df['system_prompt_version'] == 'v1']['success'].mean()\n", + "v2_mean = results_df[results_df['system_prompt_version'] == 'v2']['success'].mean()\n", + "\n", + "print(f\"V1 success rate: {v1_mean:.2f}, V2 success rate: {v2_mean:.2f}\")\n", + "\n", + "downloads_path = os.path.join(os.path.expanduser(\"~\"), \"Downloads\", \"code_check_results.csv\")\n", + "results_df.to_csv(downloads_path, index=False)\n", + "\n", + "results_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "apollo", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb b/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb deleted file mode 100644 index 90f48ff..0000000 --- a/services/job_chat/evaluation/test_prompts_issue_97_a.ipynb +++ /dev/null @@ -1,966 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test Prompts for issue 97:a\n", - "\n", - "This notebook will be used to test prompt versions on the AI Assistant to address problem a on issue 97" - ] - }, - { - "cell_type": "code", - "execution_count": 321, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "688457.10s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: python-dotenv in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (1.0.1)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "688463.81s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: anthropic in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (0.39.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.6.2.post1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.9.0)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.27.2)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (0.8.0)\n", - "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.10.19)\n", - "Requirement already satisfied: sniffio in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (1.3.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anthropic) (4.12.2)\n", - "Requirement already satisfied: idna>=2.8 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from anyio<5,>=3.5.0->anthropic) (3.10)\n", - "Requirement already satisfied: certifi in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpx<1,>=0.23.0->anthropic) (1.0.7)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->anthropic) (0.14.0)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "688469.64s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: tenacity in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (9.0.0)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "688475.29s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: apollo in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (4.2.13)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (2.32.3)\n", - "Requirement already satisfied: biopython>=1.77 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.84)\n", - "Requirement already satisfied: cachetools<4 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (3.1.1)\n", - "Requirement already satisfied: click>=6.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (8.1.7)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (1.17.0)\n", - "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (6.0.2)\n", - "Requirement already satisfied: decorator in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (5.1.1)\n", - "Requirement already satisfied: bcbio-gff>=0.6.6 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from apollo) (0.7.1)\n", - "Requirement already satisfied: six in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from bcbio-gff>=0.6.6->apollo) (1.16.0)\n", - "Requirement already satisfied: numpy in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from biopython>=1.77->apollo) (2.1.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from requests->apollo) (2024.8.30)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "688481.03s - pydevd: Sending message related to process being replaced timed-out after 5 seconds\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting pandas\n", - " Downloading pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl.metadata (89 kB)\n", - "Requirement already satisfied: numpy>=1.23.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.1.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", - "Collecting pytz>=2020.1 (from pandas)\n", - " Downloading pytz-2024.2-py2.py3-none-any.whl.metadata (22 kB)\n", - "Collecting tzdata>=2022.7 (from pandas)\n", - " Using cached tzdata-2024.2-py2.py3-none-any.whl.metadata (1.4 kB)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n", - "Downloading pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl (11.3 MB)\n", - "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/11.3 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\n", - "\u001b[?25hDownloading pytz-2024.2-py2.py3-none-any.whl (508 kB)\n", - "Using cached tzdata-2024.2-py2.py3-none-any.whl (346 kB)\n", - "Installing collected packages: pytz, tzdata, pandas\n", - "Successfully installed pandas-2.2.3 pytz-2024.2 tzdata-2024.2\n" - ] - } - ], - "source": [ - "!pip install python-dotenv\n", - "!pip install anthropic\n", - "!pip install tenacity\n", - "!pip install apollo\n", - "!pip install pandas\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from dotenv import load_dotenv\n", - "import anthropic\n", - "\n", - "load_dotenv()\n", - "ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\")\n", - "\n", - "client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)" - ] - }, - { - "cell_type": "code", - "execution_count": 301, - "metadata": {}, - "outputs": [], - "source": [ - "# Sample user prompt\n", - "user_prompt = \"\"\"\n", - "Generate 10 short code questions that a user might ask about openFn job writing that makes use of different adaptor functions and mappings. \n", - "\n", - "The questions can span around different platforms such as:\n", - "\n", - "commcare\n", - "satusehat\n", - "googlesheets\n", - "fhir\n", - "openmrs\n", - "odk\n", - "\n", - "Output your answers with new lines and without numbers or bullet points.\n", - "\n", - "\n", - "Using an array of people, map it to create a single sheet for each person\n", - "\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 302, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", - "\n", - "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", - "\n", - "How do I transform a Google Sheets row into a CommCare case update?\n", - "\n", - "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", - "\n", - "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", - "\n", - "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", - "\n", - "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", - "\n", - "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", - "\n", - "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", - "\n", - "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n" - ] - } - ], - "source": [ - "# Generate job code\n", - "\n", - "message = client.messages.create(\n", - " model=\"claude-3-5-sonnet-20241022\", # TODO change to cheaper model\n", - " max_tokens=1000,\n", - " temperature=0,\n", - " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": user_prompt\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - ")\n", - "print(message.content[0].text)" - ] - }, - { - "cell_type": "code", - "execution_count": 303, - "metadata": {}, - "outputs": [], - "source": [ - "# Create list from generated questions\n", - "\n", - "generated_questions = [q for q in message.content[0].text.split(\"\\n\") if q]" - ] - }, - { - "cell_type": "code", - "execution_count": 309, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/Users/hunter/apollo/services\n" - ] - } - ], - "source": [ - "import sys\n", - "import os\n", - "\n", - "# Add the parent directory to sys.path\n", - "parent_dir = os.path.abspath(\"../../\")\n", - "print(parent_dir)\n", - "sys.path.append(parent_dir)\n", - "\n", - "from prompt import job_writing_summary, system_role, build_prompt\n", - "from util import DictObj\n", - "\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 331, - "metadata": {}, - "outputs": [], - "source": [ - "# New edited system prompt\n", - "\n", - "\n", - "\n", - "# formatted like in apollo\n", - "system_prompt_v2 = [{'type': 'text', 'text': system_role}, {'type': 'text', 'text': job_writing_summary}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': 'The user is using the OpenFn @openfn/language-googlesheets@3.0.5 adaptor. Use functions provided by its API.Typescript definitions for doc @openfn/language-common/**\\n * Execute a sequence of operations.\\n * Main outer API for executing expressions.\\n * @public\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Promise}\\n */\\nexport function execute(...operations: Operations): Promise;\\n/**\\n * alias for \"fn()\"\\n * @function\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function alterState(func: Function): Operation;\\n/**\\n * Creates a custom step (or operation) for more flexible job writing.\\n * @public\\n * @function\\n * @example\\n * fn(state => {\\n * // do some things to state\\n * return state;\\n * });\\n * @param {Function} func is the function\\n * @returns {Operation}\\n */\\nexport function fn(func: Function): Operation;\\n/**\\n * A custom operation that will only execute the function if the condition returns true\\n * @public\\n * @function\\n * @example\\n * fnIf((state) => state?.data?.name, get(\"https://example.com\"));\\n * @param {Boolean} condition - The condition that returns true\\n * @param {Operation} operation - The operation needed to be executed.\\n * @returns {Operation}\\n */\\nexport function fnIf(condition: boolean, operation: Operation): Operation;\\n/**\\n * Picks out a single value from a JSON object.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * jsonValue({ a:1 }, \\'a\\')\\n * @param {object} obj - A valid JSON object.\\n * @param {String} path - JSONPath referencing a point in given JSON object.\\n * @returns {Operation}\\n */\\nexport function jsonValue(obj: object, path: string): Operation;\\n/**\\n * Picks out a single value from source data.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * sourceValue(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Operation}\\n */\\nexport function sourceValue(path: string): Operation;\\n/**\\n * Picks out a value from source data.\\n * Will return whatever JSONPath returns, which will always be an array.\\n * If you need a single value use `sourceValue` instead.\\n * @public\\n * @function\\n * @example\\n * source(\\'$.key\\')\\n * @param {String} path - JSONPath referencing a point in `state`.\\n * @returns {Array.}\\n */\\nexport function source(path: string): Array;\\n/**\\n * Ensures a path points at the data.\\n * @public\\n * @function\\n * @example\\n * dataPath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `data`.\\n * @returns {string}\\n */\\nexport function dataPath(path: string): string;\\n/**\\n * Picks out a single value from the source data object—usually `state.data`.\\n * If a JSONPath returns more than one value for the reference, the first\\n * item will be returned.\\n * @public\\n * @function\\n * @example\\n * dataValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `data`.\\n * @returns {Operation}\\n */\\nexport function dataValue(path: string): Operation;\\n/**\\n * Ensures a path points at references.\\n * @public\\n * @function\\n * @example\\n * referencePath(\\'key\\')\\n * @param {string} path - JSONPath referencing a point in `references`.\\n * @returns {string}\\n */\\nexport function referencePath(path: string): string;\\n/**\\n * Picks out the last reference value from source data.\\n * @public\\n * @function\\n * @example\\n * lastReferenceValue(\\'key\\')\\n * @param {String} path - JSONPath referencing a point in `references`.\\n * @returns {Operation}\\n */\\nexport function lastReferenceValue(path: string): Operation;\\n/**\\n * Simple switcher allowing other expressions to use either a JSONPath or\\n * object literals as a data source.\\n * - JSONPath referencing a point in `state`\\n * - Object Literal of the data itself.\\n * - Function to be called with state.\\n * @public\\n * @function\\n * @example\\n * asData(\\'$.key\\'| key | callback)\\n * @param {String|object|function} data\\n * @param {object} state - The current state.\\n * @returns {array}\\n */\\nexport function asData(data: string | object | Function, state: object): any[];\\n/**\\n * Iterates over an array of items and invokes an operation upon each one, where the state\\n * object is _scoped_ so that state.data is the item under iteration.\\n * The rest of the state object is untouched and can be referenced as usual.\\n * You can pass an array directly, or use lazy state or a JSONPath string to\\n * reference a slice of state.\\n * @public\\n * @function\\n * @example Using lazy state ($) to iterate over items in state.data and pass each into an \"insert\" operation\\n * each(\\n * $.data,\\n * // Inside the callback operation, `$.data` is scoped to the item under iteration\\n * insert(\"patient\", {\\n * patient_name: $.data.properties.case_name,\\n * patient_id: $.data.case_id,\\n * })\\n * );\\n * @example Iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * $.data,\\n * insert(\"patient\", (state) => ({\\n * patient_id: state.data.case_id,\\n * ...state.data\\n * }))\\n * );\\n * @example Using JSON path to iterate over items in state.data and pass each one into an \"insert\" operation\\n * each(\\n * \"$.data[*]\",\\n * insert(\"patient\", (state) => ({\\n * patient_name: state.data.properties.case_name,\\n * patient_id: state.data.case_id,\\n * }))\\n * );\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n/**\\n * Combines two operations into one\\n * @public\\n * @function\\n * @example\\n * combine(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function combine(...operations: Operations): Operation;\\n/**\\n * Adds data from a target object\\n * @public\\n * @function\\n * @example\\n * join(\\'$.key\\',\\'$.data\\',\\'newKey\\')\\n * @param {String} targetPath - Target path\\n * @param {String} sourcePath - Source path\\n * @param {String} targetKey - Target Key\\n * @returns {Operation}\\n */\\nexport function join(targetPath: string, sourcePath: string, targetKey: string): Operation;\\n/**\\n * Recursively resolves objects that have resolvable values (functions).\\n * @public\\n * @function\\n * @param {object} value - data\\n * @param {Function} [skipFilter] - a function which returns true if a value should be skipped\\n * @returns {Operation}\\n */\\nexport function expandReferences(value: object, skipFilter?: Function): Operation;\\n/**\\n * Returns a key, value pair in an array.\\n * @public\\n * @function\\n * @example\\n * field(\\'destination_field_name__c\\', \\'value\\')\\n * @param {string} key - Name of the field\\n * @param {Value} value - The value itself or a sourceable operation.\\n * @returns {Field}\\n */\\nexport function field(key: string, value: Value): Field;\\n/**\\n * Zips key value pairs into an object.\\n * @public\\n * @function\\n * @example\\n * fields(list_of_fields)\\n * @param {Fields} fields - a list of fields\\n * @returns {Object}\\n */\\nexport function fields(...fields: Fields): any;\\n/**\\n * Merges fields into each item in an array.\\n * @public\\n * @example\\n * merge(\\n * \"$.books[*]\",\\n * fields(\\n * field( \"publisher\", sourceValue(\"$.publisher\") )\\n * )\\n * )\\n * @function\\n * @public\\n * @param {DataSource} dataSource\\n * @param {Object} fields - Group of fields to merge in.\\n * @returns {DataSource}\\n */\\nexport function merge(dataSource: DataSource, fields: any): DataSource;\\n/**\\n * Groups an array of objects by a specified key path.\\n * @public\\n * @example\\n * const users = [\\n * { name: \\'Alice\\', age: 25, city: \\'New York\\' },\\n * { name: \\'Bob\\', age: 30, city: \\'San Francisco\\' },\\n * { name: \\'Charlie\\', age: 25, city: \\'New York\\' },\\n * { name: \\'David\\', age: 30, city: \\'San Francisco\\' }\\n * ];\\n * group(users, \\'city\\');\\n * // state is { data: { \\'New York\\': [/Alice, Charlie/], \\'San Francisco\\': [ /Bob, David / ] }\\n * @function\\n * @public\\n * @param {Object[]} arrayOfObjects - The array of objects to be grouped.\\n * @param {string} keyPath - The key path to group by.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function group(arrayOfObjects: any[], keyPath: string, callback?: Function): Operation;\\n/**\\n * Returns the index of the current array being iterated.\\n * To be used with `each` as a data source.\\n * @public\\n * @function\\n * @example\\n * index()\\n * @returns {DataSource}\\n */\\nexport function index(): DataSource;\\n/**\\n * Turns an array into a string, separated by X.\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", function(state) {\\n * return arrayToString(dataValue(\"path_of_array\")(state), \\', \\')\\n * })\\n * @param {array} arr - Array of toString\\'able primatives.\\n * @param {string} separator - Separator string.\\n * @returns {string}\\n */\\nexport function arrayToString(arr: any[], separator: string): string;\\n/**\\n * Ensures primitive data types are wrapped in an array.\\n * Does not affect array objects.\\n * @public\\n * @function\\n * @example\\n * each(function(state) {\\n * return toArray( dataValue(\"path_of_array\")(state) )\\n * }, ...)\\n * @param {any} arg - Data required to be in an array\\n * @returns {array}\\n */\\nexport function toArray(arg: any): any[];\\n/**\\n * Prepares next state\\n * @public\\n * @function\\n * @example\\n * composeNextState(state, response)\\n * @param {State} state - state\\n * @param {Object} response - Response to be added\\n * @returns {State}\\n */\\nexport function composeNextState(state: State, response: any): State;\\n/**\\n * Substitutes underscores for spaces and proper-cases a string\\n * @public\\n * @function\\n * @example\\n * field(\"destination_string__c\", humanProper(state.data.path_to_string))\\n * @param {string} str - String that needs converting\\n * @returns {string}\\n */\\nexport function humanProper(str: string): string;\\n/**\\n * Splits an object into two objects based on a list of keys.\\n * The first object contains the keys that are not in the list,\\n * and the second contains the keys that are.\\n * @public\\n * @function\\n * @param {Object} obj - The object to split.\\n * @param {string[]} keys - List of keys to split on.\\n * @returns {Object[]} - Tuple of objects, first object contains keys not in list, second contains keys that are.\\n */\\nexport function splitKeys(obj: any, keys: string[]): any[];\\n/**\\n * Replaces emojis in a string.\\n * @public\\n * @function\\n * @example\\n * scrubEmojis(\\'Dove🕊️⭐ 29\\')\\n * @param {string} text - String that needs to be cleaned\\n * @param {string} replacementChars - Characters that replace the emojis\\n * @returns {string}\\n */\\nexport function scrubEmojis(text: string, replacementChars: string): string;\\n/**\\n * Chunks an array into an array of arrays, each with no more than a certain size.\\n * @public\\n * @function\\n * @example\\n * chunk([1,2,3,4,5], 2)\\n * @param {Object} array - Array to be chunked\\n * @param {Integer} chunkSize - The maxiumum size of each chunks\\n * @returns {Object}\\n */\\nexport function chunk(array: any, chunkSize: Integer): any;\\n/**\\n * Takes a CSV file string or stream and parsing options as input, and returns a promise that\\n * resolves to the parsed CSV data as an array of objects.\\n * Options for `parsingOptions` include:\\n * - `delimiter` {string/Buffer/[string/Buffer]} - Defines the character(s) used to delineate the fields inside a record. Default: `\\',\\'`\\n * - `quote` {string/Buffer/[string/Buffer]} - Defines the characters used to surround a field. Default: `\\'\"\\'`\\n * - `escape` {Buffer/string/null/boolean} - Set the escape character as one character/byte only. Default: `\"`\\n * - `columns` {boolean / array / function} - Generates record in the form of object literals. Default: `true`\\n * - `bom` {boolean} - Strips the {@link https://en.wikipedia.org/wiki/Byte_order_mark byte order mark (BOM)} from the input string or buffer. Default: `true`\\n * - `trim` {boolean} - Ignore whitespace characters immediately around the `delimiter`. Default: `true`\\n * - `ltrim` {boolean} - Ignore whitespace characters from the left side of a CSV field. Default: `true`\\n * - `rtrim` {boolean} - Ignore whitespace characters from the right side of a CSV field. Default: `true`\\n * - `chunkSize` {number} - The size of each chunk of CSV data. Default: `Infinity`\\n * - `skip_empty_lines` {boolean} - Ignore empty lines in the CSV file. Default: `true`\\n * @public\\n * @function\\n * @param {String | Stream} csvData - A CSV string or a readable stream\\n * @param {Object} [parsingOptions] - Optional. Parsing options for converting CSV to JSON.\\n * @param {function} [callback] - (Optional) callback function. If used it will be called state and an array of rows.\\n * @returns {Operation} The function returns a Promise that resolves to the result of parsing a CSV `stringOrStream`.\\n */\\nexport function parseCsv(csvData: string | Stream, parsingOptions?: any, callback?: Function): Operation;\\n/**\\n * Validate against a JSON schema. Any erors are written to an array at `state.validationErrors`.\\n * Schema can be passed directly, loaded as a JSON path from state, or loaded from a URL\\n * Data can be passed directly or loaded as a JSON path from state.\\n * By default, schema is loaded from `state.schema` and data from `state.data`.\\n * @pubic\\n * @function\\n * @param {string|object} schema - The schema, path or URL to validate against\\n * @param {string|object} data - The data or path to validate\\n * @example Validate `state.data` with `state.schema`\\n * validate()\\n * @example Validate form data at `state.form` with a schema from a URL\\n * validate(\"https://www.example.com/schema/record\", \"form\")\\n * @example Validate the each item in `state.records` with a schema from a URL\\n * each(\"records[*]\", validate(\"https://www.example.com/schema/record\"))\\n * @returns {Operation}\\n */\\nexport function validate(schema?: string | object, data?: string | object): Operation;\\n/**\\n * Sets a cursor property on state.\\n * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,\\n * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones\\n * are not yet supported.\\n * You can provide a formatter to customise the final cursor value, which is useful for normalising\\n * different inputs. The custom formatter runs after natural language date conversion.\\n * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}\\n * @public\\n * @function\\n * @example Use a cursor from state if present, or else use the default value\\n * cursor($.cursor, { defaultValue: \\'today\\' })\\n * @example Use a pagination cursor\\n * cursor(22)\\n * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number\\n * @param {object} options - options to control the cursor.\\n * @param {string} options.key - set the cursor key. Will persist through the whole run.\\n * @param {any} options.defaultValue - the value to use if value is falsy\\n * @param {Function} options.format - custom formatter for the final cursor value\\n * @returns {Operation}\\n */\\nexport function cursor(value: any, options?: {\\n key: string;\\n defaultValue: any;\\n format: Function;\\n}): Operation;\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n * @public\\n * @function\\n * @example\\n * map(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\"))\\n * )\\n * )\\n * @param {string} path - JSONPath referencing a point in `state.data`.\\n * @param {function} operation - The operation needed to be repeated.\\n * @param {State} state - Runtime state.\\n * @returns {State}\\n */\\nexport const map: any;\\n\\n\\n/**\\n * Scopes an array of data based on a JSONPath.\\n * Useful when the source data has `n` items you would like to map to\\n * an operation.\\n * The operation will receive a slice of the data based of each item\\n * of the JSONPath provided.\\n *\\n * It also ensures the results of an operation make their way back into\\n * the state\\'s references.\\n * @public\\n * @example\\n * each(\"$.[*]\",\\n * create(\"SObject\",\\n * field(\"FirstName\", sourceValue(\"$.firstName\")))\\n * )\\n * @function\\n * @param {DataSource} dataSource - JSONPath referencing a point in `state`.\\n * @param {Operation} operation - The operation needed to be repeated.\\n * @returns {Operation}\\n */\\nexport function each(dataSource: DataSource, operation: Operation): Operation;\\n\\n\\nexport { parse, format } from \"date-fns\";\\n\\n\\n/**\\n * Builder function to create request options. Returns an object with helpers to\\n * easily add commonly used options. The return object is chainable so you can set\\n * as many options as you want.\\n * Pass an object to set your own options.\\n * @param {CommonRequestOptions} options - options to pass to the request\\n * @returns {OptionsHelpers}\\n * @function\\n * @public\\n * @example Get with a query an oath token\\n * get($.data.url, http.options({ query: $.query }).oath($.configuration.access_token)\\n */\\nexport function options(opts?: {}): any;\\n/**\\n * Make a GET request.\\n * @public\\n * @function\\n * @example Request a resource\\n * http.get(\\'https://jsonplaceholder.typicode.com/todos\\')\\n * @example Request a resource with basic auth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().basic(\\'user\\', \\'pass\\')\\n * )\\n * @example Request a resource with oauth\\n * http.get(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * http.options().oauth($.configuration.access_token)\\n * )\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function get(url: string, options: CommonRequestOptions): Operation;\\n/**\\n * Make a POST request.\\n * @public\\n * @function\\n * @example Post a JSON object (setting the content-type header)\\n * http.post(\\n * \\'https://jsonplaceholder.typicode.com/todos\\',\\n * $.data,\\n * options().json(),\\n * })\\n * @param {string} url - URL to access\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\nexport function post(path: any, data: any, options: CommonRequestOptions): Operation;\\nexport { req as request };\\n/**\\n * Helper functions provided by `http.options`.\\n */\\nexport type OptionsHelpers = any;\\n/**\\n * Options provided to the HTTP request\\n */\\nexport type CommonRequestOptions = {\\n /**\\n * - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n */\\n errors: object;\\n /**\\n * - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n */\\n form: object;\\n /**\\n * - An object of query parameters to be encoded into the URL.\\n */\\n query: object;\\n /**\\n * - An object of headers to append to the request.\\n */\\n headers: object;\\n /**\\n * - Parse the response body as json, text or stream. By default will use the response headers.\\n */\\n parseAs: string;\\n /**\\n * - Request timeout in ms. Default: 300 seconds.\\n */\\n timeout: number;\\n /**\\n * - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n tls: object;\\n};\\n/**\\n * State object\\n */\\nexport type CommonHttpState = any;\\n/**\\n * Options provided to the HTTP request\\n * @typedef {Object} CommonRequestOptions\\n * @property {object} errors - Map of errorCodes -> error messages, ie, `{ 404: \\'Resource not found;\\' }`. Pass `false` to suppress errors.\\n * @property {object} form - Pass a JSON object to be serialised into a multipart HTML form (as FormData) in the body.\\n * @property {object} query - An object of query parameters to be encoded into the URL.\\n * @property {object} headers - An object of headers to append to the request.\\n * @property {string} parseAs - Parse the response body as json, text or stream. By default will use the response headers.\\n * @property {number} timeout - Request timeout in ms. Default: 300 seconds.\\n * @property {object} tls - TLS/SSL authentication options. See https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions\\n */\\n/**\\n * State object\\n * @typedef {Object} CommonHttpState\\n * @private\\n * @property data - the parsed response body\\n * @property response - the response from the HTTP server, including headers, statusCode, body, etc\\n * @property references - an array of all previous data objects used in the Job\\n **/\\n/**\\n * Make a HTTP request.\\n * @public\\n * @function\\n * @example\\n * http.request(\\n * \\'GET\\',\\n * \\'https://jsonplaceholder.typicode.com/todos\\'\\n * )\\n * @name request\\n * @param {string} method - The HTTP method to use.\\n * @param {string} url - URL to resource.\\n * @param {CommonRequestOptions} options - Request options\\n * @state {CommonHttpState}\\n * @returns {Operation}\\n */\\ndeclare function req(method: string, url: string, options: CommonRequestOptions): Operation;\\n\\n\\nimport * as Adaptor from \\'./Adaptor\\';\\nexport default Adaptor;\\nexport * from \\'./Adaptor\\';\\nexport * as beta from \\'./beta\\';\\nexport * as http from \\'./http\\';\\nexport * as dateFns from \\'./dateFns\\';\\nimport * as metadata from \\'./metadata\\';\\nexport { metadata };\\n\\n\\ndeclare type Entity = {\\n name: string;\\n type: string;\\n label?: string;\\n datatype?: string;\\n desc?: string;\\n children?: Entity[] | Record;\\n meta?: Record;\\n addChild: (e: Entity, name?: string) => void;\\n};\\ndeclare type DataType = \\'string\\' | \\'boolean\\' | \\'date\\';\\n\\n\\nexport function encode(data: string): string;\\nexport function decode(base64Data: string): string;\\nexport function uuid(): string;\\n\\n\\n/**\\n * `request` is a helper function that sends HTTP requests and returns the response\\n * body, headers, and status code.\\n * Use the error map to provide custom error messages or get hold of the response in case of errors.\\n * @param method - The HTTP method to use for the request (e.g., \"GET\", \"POST\", \"PUT\", \"DELETE\", etc.).\\n * @param fullUrlOrPath - The full or partial URL for the request.\\n * @param [options] - The `options` parameter is an object that contains additional configuration\\n * options for the request.\\n * @returns an object with the following properties:\\n * - method: the request method\\n * - url: the request url\\n * - code: the status code of the response\\n * - headers: the headers of the response\\n * - body: the body of the response\\n * - message: the status text of the response\\n * - duration: the response time\\n */\\nexport function request(method: any, fullUrlOrPath: any, options?: {}): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function makeBasicAuthHeader(username: any, password: any): {\\n Authorization: string;\\n};\\nexport function logResponse(response: any): any;\\nexport function enableMockClient(baseUrl: any): import(\"undici/types/mock-interceptor\").Interceptable;\\nexport const ERROR_ABSOLUTE_URL: \"Absolute URLs not suppored\";\\nexport function assertRelativeUrl(path: any): void;\\nexport const ERROR_URL_MISMATCH: \"Target origin does not match baseUrl origin\";\\nexport function parseUrl(pathOrUrl: string, baseUrl: any): {\\n url: string;\\n baseUrl: string;\\n path: string;\\n query: any;\\n};\\nexport function get(url: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function post(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function put(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\nexport function del(url: any, body: any, options: any): Promise<{\\n url: string;\\n method: any;\\n statusCode: any;\\n statusMessage: string;\\n headers: any;\\n body: any;\\n duration: number;\\n}>;\\n\\n\\nexport * from \"./http\";\\nexport * from \"./helpers\";\\nexport * from \"./references\";\\nimport parseDate from \"./parse-date\";\\nimport throwError from \"./throw-error\";\\nexport { parseDate, throwError };\\n\\n\\ndeclare function _default(d: any, startDate: any): any;\\nexport default _default;\\n\\n\\nexport function expandReferences(state: any, ...args: any[]): any[];\\nexport function normalizeOauthConfig(configuration: any): any;\\n\\n\\ndeclare function _default(code: any, { description, fix, ...extras }?: {\\n description: any;\\n fix: any;\\n}): never;\\nexport default _default;\\nTypescript definitions for doc @openfn/language-googlesheets/**\\n * Execute a sequence of oper.\\n * Wraps `language-common/execute`, and prepends initial state for http.\\n * @example\\n * execute(\\n * create(\\'foo\\'),\\n * delete(\\'bar\\')\\n * )(state)\\n * @private\\n * @param {Operations} operations - Operations to be performed.\\n * @returns {Operation}\\n */\\nexport function execute(...operations: Operations): Operation;\\n/**\\n * Add an array of rows to the spreadsheet.\\n * https://developers.google.com/sheets/api/samples/writing#append_values\\n * @public\\n * @example\\n * appendValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) Callback function\\n * @returns {Operation}\\n */\\nexport function appendValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Batch update values in a Spreadsheet.\\n * @example\\n * batchUpdateValues({\\n * spreadsheetId: \\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\n * range: \\'Sheet1!A1:E1\\',\\n * values: [\\n * [\\'From expression\\', \\'$15\\', \\'2\\', \\'3/15/2016\\'],\\n * [\\'Really now!\\', \\'$100\\', \\'1\\', \\'3/20/2016\\'],\\n * ],\\n * })\\n * @function\\n * @public\\n * @param {Object} params - Data object to add to the spreadsheet.\\n * @param {string} [params.spreadsheetId] The spreadsheet ID.\\n * @param {string} [params.range] The range of values to update.\\n * @param {string} [params.valueInputOption] (Optional) Value update options. Defaults to \\'USER_ENTERED\\'\\n * @param {array} [params.values] A 2d array of values to update.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function batchUpdateValues(params: {\\n spreadsheetId?: string;\\n range?: string;\\n valueInputOption?: string;\\n values?: any[];\\n}, callback?: Function): Operation;\\n/**\\n * Gets cell values from a Spreadsheet.\\n * @public\\n * @example\\n * getValues(\\'1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos\\',\\'Sheet1!A1:E1\\')\\n * @function\\n * @param {string} spreadsheetId The spreadsheet ID.\\n * @param {string} range The sheet range.\\n * @param {function} callback - (Optional) callback function\\n * @returns {Operation} spreadsheet information\\n */\\nexport function getValues(spreadsheetId: string, range: string, callback?: Function): Operation;\\nexport { alterState, combine, cursor, dataPath, dataValue, each, field, fields, fn, fnIf, http, lastReferenceValue, merge, sourceValue } from \"@openfn/language-common\";\\n\\n\\nexport default Adaptor;\\nexport * from \"./Adaptor\";\\nimport * as Adaptor from \"./Adaptor\";\\n'}, {'type': 'text', 'text': '.', 'cache_control': {'type': 'ephemeral'}}, {'type': 'text', 'text': '// write your job code here'}]" - ] - }, - { - "cell_type": "code", - "execution_count": 332, - "metadata": {}, - "outputs": [], - "source": [ - "# Import tenacity to implement waiting after failed api calls\n", - "\n", - "from tenacity import (\n", - " retry,\n", - " stop_after_attempt,\n", - " wait_random_exponential,\n", - ") # for exponential backoff" - ] - }, - { - "cell_type": "code", - "execution_count": 333, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "\n", - "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", - "def generate_apollo_format(content, history=\"\", adaptor=\"@openfn/language-googlesheets@3.0.5\", model=\"claude-3-5-sonnet-20240620\"):\n", - " \"\"\"Use Apollo prompt formatting templates to generate a Claude answer.\"\"\"\n", - "\n", - " context = DictObj({\n", - " \"expression\": \"// write your job code here\",\n", - " \"adaptor\": adaptor\n", - " })\n", - " print(content)\n", - " system_message, prompt = build_prompt(content, history, context)\n", - " # print(system_message)\n", - " # print(prompt)\n", - "\n", - " message = client.beta.prompt_caching.messages.create(\n", - " max_tokens=1024, messages=prompt, model=model, system=system_message\n", - " )\n", - " return message.content[0].text" - ] - }, - { - "cell_type": "code", - "execution_count": 334, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "How do I transform a Google Sheets row into a CommCare case update?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" - ] - }, - { - "data": { - "text/plain": [ - "\"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format CommCare expects\\n3. Send the update to CommCare\\n\\nHere's a basic outline of how you might structure this job:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E10')\\n\\n// Transform the data\\nfn(state => {\\n const rows = state.data.values;\\n const caseUpdates = rows.map(row => ({\\n case_id: row[0], // Assuming case ID is in the first column\\n properties: {\\n name: row[1],\\n age: row[2],\\n // Add more properties as needed\\n }\\n }));\\n return { ...state, caseUpdates };\\n})\\n\\n// Send updates to CommCare\\neach(\\n '$.caseUpdates[*]',\\n post(\\n 'https://www.commcarehq.org/a/your-domain/api/v0.5/case/',\\n state => ({\\n case_id: state.data.case_id,\\n properties: state.data.properties\\n }),\\n {\\n headers: {\\n 'Authorization': 'ApiKey your-api-key',\\n 'Content-Type': 'application/json'\\n }\\n }\\n )\\n)\\n```\\n\\nThis is a basic example and you'll need to adjust it based on your specific Google Sheets structure and CommCare case properties. Also, you'll need to replace 'your-spreadsheet-id', 'your-domain', and 'your-api-key' with your actual values.\\n\\nRemember to handle errors and edge cases in a production environment. You might also want to add logging or error reporting.\"" - ] - }, - "execution_count": 334, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "generate_apollo_format(\"How do I transform a Google Sheets row into a CommCare case update?\")" - ] - }, - { - "cell_type": "code", - "execution_count": 335, - "metadata": {}, - "outputs": [], - "source": [ - "# Generate answers with the new system prompt\n", - "# @retry(wait=wait_random_exponential(min=1, max=90), stop=stop_after_attempt(20))\n", - "\n", - "@retry(wait=wait_random_exponential(multiplier=1, max=60), stop=stop_after_attempt(20))\n", - "def generate_new_format(question, new_system_prompt, model=\"claude-3-5-sonnet-20240620\"):\n", - " \"\"\"Use the apollo format to call claude with a custom system prompt.\"\"\"\n", - " prompt = [{'role': 'user', 'content': question}]\n", - " print(question)\n", - " message = client.beta.prompt_caching.messages.create(\n", - " max_tokens=1024, messages=prompt, model=model, system=new_system_prompt\n", - " )\n", - " return message.content[0].text" - ] - }, - { - "cell_type": "code", - "execution_count": 336, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How do I transform a Google Sheets row into a CommCare case update?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How can I map CommCare case data to create multiple FHIR patients in a single job?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How do I transform a Google Sheets row into a CommCare case update?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How can I merge data from multiple OpenMRS observations into a single Google Sheet row?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "How can I use combine() to merge data from multiple sources before creating a FHIR resource?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n", - "What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages?beta=prompt_caching \"HTTP/1.1 200 OK\"\n" - ] - } - ], - "source": [ - "# Get answers to generated questions\n", - "qa_pairs = []\n", - "for question in generated_questions[:30]:\n", - " answer = generate_apollo_format(question)\n", - " qa_pairs.append((question, answer, \"v1\"))\n", - "for question in generated_questions[:30]: # two loops to make use of prompt caching\n", - " new_answer = generate_new_format(question, system_prompt_v2)\n", - " qa_pairs.append((question, new_answer, \"v2\"))" - ] - }, - { - "cell_type": "code", - "execution_count": 337, - "metadata": {}, - "outputs": [], - "source": [ - "evaluate_answer_template = \"\"\"\n", - "For the given Question and Answer pair, evaluate whether the answer at least briefly answers the question (True), or if it entirely fails to answer the question (False).\n", - "\n", - "Additionally, check the content of the `answer` variable for the presence of the pattern `)(state)` using regex. If this pattern is found in the `answer`, consider the response invalid and mark it as `Success: False`.\n", - "\n", - "\n", - "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", - "A: \"get('Patient)\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "Q: \"How can I map CommCare case data to create multiple FHIR patients in a single job?\"\n", - "A: \"fn(state => {{ return {{ ...state, people: [] }}; }})(state);\"\n", - "Regex Match: True\n", - "Success: False\n", - "\n", - "Now, following Q/A pair, by ONLY outputting \"True\" or \"False\":\n", - "Q: {question}\n", - "A: {answer}\n", - "Regex Match: \n", - "Success:\n", - "\"\"\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": 338, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: False\n", - "\n", - "The answer provides information about using GoogleSheets adaptor and appendValues function, but does not address how to create an ODK form submission using CommCare case data. The answer is completely off-topic from what was asked in the question.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to merge OpenMRS observations data into a Google Sheet row, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer successfully addresses the question, so Success is True.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to use each() to process FHIR resources and create CommCare cases, including code examples and step-by-step instructions. While the answer contains state in function parameters, it doesn't match the specific pattern `)(state)`, so there's no regex match. The answer successfully addresses the question asked.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to map ODK repeat groups to create multiple OpenMRS encounters using Google Sheets adaptor, including code examples and step-by-step instructions. While it uses Google Sheets as an intermediary rather than direct OpenMRS integration, it still addresses the core concept of mapping repeat groups to multiple entries, making it a valid response to the question.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Let me evaluate this:\n", - "\n", - "1. First, checking for the regex pattern `)(state)` in the answer:\n", - " - The answer contains code examples but doesn't have the exact pattern `)(state)`\n", - " - Regex Match: False\n", - "\n", - "2. Evaluating if the answer addresses the question about using combine() to merge data:\n", - " - The answer directly explains how to use combine()\n", - " - Provides a detailed example with steps\n", - " - Shows how to merge data from multiple sources\n", - " - Demonstrates creating a FHIR resource with the combined data\n", - " - Success: True\n", - "\n", - "Therefore, the final answer is: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to map CommCare case properties to OpenMRS patient attributes, including code examples and key steps. It directly addresses the question asked, and it doesn't contain the invalid pattern `)(state)`.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to create ODK form submissions using CommCare case data, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer addresses the question directly, so it's successful.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of a general approach to mapping data from CommCare to FHIR resources, including specific steps and code examples. While it acknowledges not having specific information about SatuSehat's implementation, it still provides a relevant and helpful response to the question about mapping nested CommCare form data to FHIR resources.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The answer provides a detailed explanation of how to filter Google Sheets data, which addresses the core question about filtering Google Sheets data, even though it clarifies that direct integration with SatuSehat isn't built-in. The pattern `)(state)` is not present in the answer, and the response provides useful, relevant information about data filtering in Google Sheets.\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Let me evaluate this:\n", - "\n", - "1. First, checking if the answer addresses the question about using combine() to merge data:\n", - "The answer provides a detailed explanation of using combine() with multiple data sources and includes a code example, so it does answer the question.\n", - "\n", - "2. Checking for the regex pattern `)(state)`:\n", - "Looking through the answer, while it contains `state` in various places, it doesn't contain the specific pattern `)(state)`.\n", - "\n", - "Therefore:\n", - "Regex Match: False\n", - "Success: True\n", - "\n", - "The final answer is: True\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "Regex Match: True (the pattern `)(state)` appears in the code)\n", - "Success: False\n" - ] - } - ], - "source": [ - "# Evaluate QA pairs\n", - "\n", - "results = []\n", - "\n", - "for q, a, version in qa_pairs:\n", - " evaluate_answer_template_filled = evaluate_answer_template.format(question=q, answer=a)\n", - "\n", - " message = client.messages.create(\n", - " model=\"claude-3-5-sonnet-20241022\",\n", - " max_tokens=1000, #TODO limit to 1\n", - " temperature=0,\n", - " system=\"You are an AI programming assistant. Follow the user's requirements carefully and to the letter.\",\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\": evaluate_answer_template_filled\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - " )\n", - " result = message.content[0].text\n", - " print(result)\n", - " results.append([q, a, version, result])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 339, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[['How can I map CommCare case data to create multiple FHIR patients in a single job?', 'To map CommCare case data to create multiple FHIR patients in a single job, you can use the `each` function to iterate over the CommCare cases and create a FHIR patient for each one. Here\\'s an example of how you could structure your job:\\n\\n```javascript\\neach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [{ value: caseData.case_id }],\\n name: [{ \\n given: [caseData.first_name],\\n family: caseData.last_name\\n }],\\n gender: caseData.gender,\\n birthDate: caseData.dob\\n };\\n \\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Patients!A:E\",\\n values: [[\\n patientData.identifier[0].value,\\n patientData.name[0].given[0],\\n patientData.name[0].family,\\n patientData.gender,\\n patientData.birthDate\\n ]]\\n })(state);\\n })\\n);\\n```\\n\\nThis job does the following:\\n\\n1. Uses `each` to iterate over all cases in the CommCare data.\\n2. For each case, it creates a FHIR Patient resource structure.\\n3. It then appends a row to a Google Sheet with the patient data.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with the actual ID of your Google Sheet.\\n\\nNote that this is a simplified example. You might need to adjust the field mappings based on your specific CommCare and FHIR data structures. Also, you may want to add error handling and additional data transformations as needed.', 'v1', 'True'], [\"What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\", 'To create a new ODK form submission using data from multiple CommCare cases, you\\'ll need to use a combination of functions from the GoogleSheets adaptor. Here\\'s a general approach:\\n\\n1. First, you\\'d fetch the CommCare case data (this would be done in a previous step or job).\\n2. Then, you\\'d use the `appendValues` function to add the data to a Google Sheet.\\n\\nHere\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.cases[*]\",\\n appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:E\",\\n values: state => [\\n [\\n state.data.case_id,\\n state.data.properties.name,\\n state.data.properties.age,\\n state.data.properties.village,\\n state.data.properties.status\\n ]\\n ]\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. It uses `each` to iterate over an array of cases stored in `state.cases`.\\n2. For each case, it calls `appendValues` to add a new row to the Google Sheet.\\n3. The `values` property is a function that returns an array of arrays (in this case, just one inner array per case). Each inner array represents a row in the spreadsheet.\\n4. We\\'re assuming the CommCare case data has properties like `case_id`, `name`, `age`, `village`, and `status`. You\\'ll need to adjust these to match your actual case structure.\\n\\nRemember to replace \"YOUR_SPREADSHEET_ID_HERE\" with your actual Google Spreadsheet ID.\\n\\nAlso, make sure you\\'ve correctly set up your configuration to authenticate with Google Sheets API.\\n\\nIf you need to do any data transformation or combine data from multiple cases into a single row, you can do that within the `values` function.', 'v1', 'Regex Match: False\\nSuccess: False\\n\\nThe answer provides information about using GoogleSheets adaptor and appendValues function, but does not address how to create an ODK form submission using CommCare case data. The answer is completely off-topic from what was asked in the question.'], ['How do I transform a Google Sheets row into a CommCare case update?', \"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format CommCare expects\\n3. Send the transformed data to CommCare\\n\\nHere's a basic example of how you might structure this:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A2:E', state => {\\n const [name, caseId, age, status] = state.data.values[0];\\n \\n // Transform data for CommCare\\n const caseData = {\\n case_id: caseId,\\n properties: {\\n name: name,\\n age: age,\\n status: status\\n }\\n };\\n \\n // Here you would use a CommCare adaptor to send the data\\n // This is just a placeholder since we're using Google Sheets adaptor\\n console.log('Case data to send to CommCare:', caseData);\\n \\n return state;\\n});\\n```\\n\\nNote that this example assumes:\\n1. Your Google Sheet has columns for name, case ID, age, and status\\n2. You're only processing one row (the first row of data)\\n\\nTo process multiple rows, you'd need to use `each()` or map over `state.data.values`.\\n\\nRemember, this job doesn't actually send data to CommCare - you'd need to use the CommCare adaptor for that part. But this shows how you can fetch and transform the data from Google Sheets.\", 'v1', 'Regex Match: False\\nSuccess: True'], [\"What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\", \"I apologize, but I can't provide specific advice about mapping CommCare form data to SatuSehat's FHIR resources, as that's outside the scope of the OpenFn job writing assistance I'm configured to provide. \\n\\nHowever, I can offer some general guidance on working with nested data in OpenFn jobs:\\n\\n1. Use the `each()` function to iterate over nested arrays of data.\\n\\n2. Use dot notation or bracket notation to access nested properties, e.g. `state.data.form.patient.name`.\\n\\n3. The `dataValue()` function can be useful for extracting values from nested paths.\\n\\n4. You may need to use `fn()` blocks to perform more complex data transformations.\\n\\n5. Consider breaking down complex mappings into multiple steps for readability.\\n\\nIf you have a specific OpenFn job writing question related to Google Sheets, I'd be happy to assist with that. Otherwise, you may want to consult documentation or experts familiar with CommCare and SatuSehat's FHIR implementation for the specific mapping details.\", 'v1', 'Regex Match: False\\nSuccess: True'], ['How can I merge data from multiple OpenMRS observations into a single Google Sheet row?', \"To merge data from multiple OpenMRS observations into a single Google Sheet row, you can follow these steps:\\n\\n1. Fetch the OpenMRS observations data (this step would be done in a separate OpenMRS adaptor job).\\n\\n2. Transform the data to create a single row per patient with all observations.\\n\\n3. Use the Google Sheets adaptor to append this data as a new row.\\n\\nHere's an example of how you might structure your Google Sheets job to accomplish this:\\n\\n```javascript\\nfn(state => {\\n const patients = state.data.map(patient => {\\n // Assuming patient data is structured with observations\\n return {\\n patientId: patient.id,\\n name: patient.name,\\n weight: patient.observations.find(obs => obs.concept === 'weight')?.value,\\n height: patient.observations.find(obs => obs.concept === 'height')?.value,\\n bloodPressure: patient.observations.find(obs => obs.concept === 'bloodPressure')?.value\\n };\\n });\\n\\n return { ...state, transformedData: patients };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.transformedData.map(patient => [\\n patient.patientId,\\n patient.name,\\n patient.weight,\\n patient.height,\\n patient.bloodPressure\\n ])\\n});\\n```\\n\\nThis code does the following:\\n\\n1. Uses `fn()` to transform the data from the OpenMRS format into a flat structure for each patient.\\n\\n2. Uses `appendValues()` to add the transformed data to your Google Sheet.\\n\\nMake sure to replace the `spreadsheetId` with your actual Google Sheet ID, and adjust the `range` as needed. Also, modify the data transformation logic to match your specific OpenMRS data structure and the desired Google Sheets format.\", 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to merge OpenMRS observations data into a Google Sheet row, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer successfully addresses the question, so Success is True.\"], [\"What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\", 'To use `each()` to process multiple FHIR resources and create CommCare cases, you\\'ll want to structure your code like this:\\n\\n```javascript\\neach(\\n \"$.bundle.entry[*].resource\",\\n fn(state => {\\n const resource = state.data;\\n // Process the FHIR resource and prepare data for CommCare\\n const caseData = {\\n case_type: \"patient\",\\n // Map FHIR fields to CommCare fields\\n name: resource.name[0].given[0],\\n gender: resource.gender,\\n // Add more fields as needed\\n };\\n \\n // Use appendValues to add the case data to a Google Sheet\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: [Object.values(caseData)]\\n })(state);\\n })\\n);\\n```\\n\\nThis code will:\\n\\n1. Use `each()` to iterate over each FHIR resource in the bundle.\\n2. For each resource, create a `caseData` object mapping FHIR fields to CommCare fields.\\n3. Use `appendValues()` to add the case data as a new row in your Google Sheet.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with your actual Google Sheets spreadsheet ID.\\n\\nYou may need to adjust the field mappings and add error handling based on your specific FHIR resources and CommCare case structure.', 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to use each() to process FHIR resources and create CommCare cases, including code examples and step-by-step instructions. While the answer contains state in function parameters, it doesn't match the specific pattern `)(state)`, so there's no regex match. The answer successfully addresses the question asked.\"], ['How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?', 'To map an array of ODK repeat groups to create multiple OpenMRS encounters using the Google Sheets adaptor, you\\'ll need to use a combination of `each()` and `appendValues()`. Here\\'s a general approach:\\n\\n1. First, access the repeat group data from your state.\\n2. Use `each()` to iterate over the repeat group array.\\n3. Inside the `each()` block, format each repeat group item into a row for Google Sheets.\\n4. Use `appendValues()` to add the formatted row to your Google Sheet.\\n\\nHere\\'s an example of how this might look:\\n\\n```javascript\\neach(\\n \"$.form.repeat_group[*]\",\\n fn(state => {\\n const repeatItem = state.data;\\n const formattedRow = [\\n repeatItem.encounter_date,\\n repeatItem.patient_id,\\n repeatItem.symptom,\\n repeatItem.severity\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:D\",\\n values: [formattedRow]\\n })(state);\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. `each(\"$.form.repeat_group[*]\", ...)` iterates over each item in the repeat group array.\\n2. Inside the `fn()`, we access the current repeat group item via `state.data`.\\n3. We format the repeat group item into a row array (`formattedRow`).\\n4. We use `appendValues()` to add this row to the Google Sheet.\\n\\nMake sure to replace \"YOUR_SPREADSHEET_ID\" with your actual Google Sheets spreadsheet ID, and adjust the range and column structure to match your specific needs.\\n\\nRemember that this approach will make one API call to Google Sheets for each repeat group item. If you have a large number of items, you might want to consider batching the updates for better performance.', 'v1', 'Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to map ODK repeat groups to create multiple OpenMRS encounters using Google Sheets adaptor, including code examples and step-by-step instructions. While it uses Google Sheets as an intermediary rather than direct OpenMRS integration, it still addresses the core concept of mapping repeat groups to multiple entries, making it a valid response to the question.'], [\"What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\", \"I apologize, but there's a misunderstanding in your question. The @openfn/language-googlesheets adaptor doesn't have built-in functionality to directly filter data and send it to SatuSehat. \\n\\nHowever, I can explain how you might approach this task using OpenFn:\\n\\n1. First, you'd use the `getValues` function to retrieve data from Google Sheets.\\n2. Then you'd use `fn()` to filter that data.\\n3. Finally, you'd use a different adaptor (not Google Sheets) to send the filtered data to SatuSehat.\\n\\nHere's a conceptual example:\\n\\n```javascript\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E10')\\nfn(state => {\\n const filteredData = state.data.values.filter(row => {\\n // Your filtering logic here\\n return true; // or false\\n });\\n return { ...state, filteredData };\\n})\\n// Here you'd use another adaptor to send filteredData to SatuSehat\\n```\\n\\nRemember, you'll need to use the appropriate adaptor and functions to send data to SatuSehat after filtering. The Google Sheets adaptor is only for interacting with Google Sheets.\", 'v1', 'Regex Match: False\\nSuccess: True'], ['How can I use combine() to merge data from multiple sources before creating a FHIR resource?', \"To use `combine()` to merge data from multiple sources before creating a FHIR resource, you can follow this pattern:\\n\\n1. Use `getValues()` to fetch data from different Google Sheets.\\n2. Use `combine()` to merge the results of those operations.\\n3. Use `fn()` to process the combined data and create your FHIR resource.\\n\\nHere's an example:\\n\\n```javascript\\ncombine(\\n getValues('spreadsheetId1', 'Sheet1!A1:C10'),\\n getValues('spreadsheetId2', 'Sheet2!A1:D5')\\n)\\n\\nfn(state => {\\n const [data1, data2] = state.references;\\n \\n // Process and merge the data\\n const mergedData = {\\n // ... combine data1 and data2 as needed\\n };\\n\\n // Create FHIR resource\\n const fhirResource = {\\n resourceType: 'Patient',\\n // ... populate with mergedData\\n };\\n\\n return { ...state, fhirResource };\\n})\\n\\n// You could then use this data in a subsequent operation\\n// For example, to append it to another sheet:\\nappendValues({\\n spreadsheetId: 'outputSpreadsheetId',\\n range: 'OutputSheet!A1',\\n values: [JSON.stringify(state.fhirResource)]\\n})\\n```\\n\\nThis approach allows you to fetch data from multiple sources, combine it, process it into a FHIR resource, and then use that resource in subsequent operations.\", 'v1', \"Let me evaluate this:\\n\\n1. First, checking for the regex pattern `)(state)` in the answer:\\n - The answer contains code examples but doesn't have the exact pattern `)(state)`\\n - Regex Match: False\\n\\n2. Evaluating if the answer addresses the question about using combine() to merge data:\\n - The answer directly explains how to use combine()\\n - Provides a detailed example with steps\\n - Shows how to merge data from multiple sources\\n - Demonstrates creating a FHIR resource with the combined data\\n - Success: True\\n\\nTherefore, the final answer is: True\"], [\"What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\", 'To map CommCare case properties to custom OpenMRS patient attributes in an OpenFn job, you\\'ll typically use a combination of the `each()` function to iterate over cases, and then use `field()` functions to map specific properties. Here\\'s a general approach:\\n\\n1. Use `each()` to iterate over CommCare cases\\n2. Inside `each()`, use `create()` or `upsert()` to create/update OpenMRS patients\\n3. Use `field()` functions to map CommCare properties to OpenMRS attributes\\n\\nHere\\'s an example structure:\\n\\n```javascript\\neach(\\n \"$.forms[*]\", // or wherever your CommCare cases are in state\\n create(\"patient\", {\\n attributes: [\\n field(\"attributeType\", \"CommCare_ID\"),\\n field(\"value\", dataValue(\"case.case_id\")),\\n ],\\n person: {\\n names: [\\n {\\n givenName: dataValue(\"case.first_name\"),\\n familyName: dataValue(\"case.last_name\"),\\n },\\n ],\\n gender: dataValue(\"case.gender\"),\\n birthdate: dataValue(\"case.dob\"),\\n },\\n })\\n);\\n```\\n\\nThis example assumes you\\'re using the OpenMRS adaptor. You\\'d need to adjust the paths (e.g., `case.first_name`) to match your actual CommCare data structure.\\n\\nRemember to configure your OpenMRS instance to have custom attributes that match what you\\'re sending from CommCare. The `attributeType` should match the name or UUID of the custom attribute in OpenMRS.', 'v1', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to map CommCare case properties to OpenMRS patient attributes, including code examples and key steps. It directly addresses the question asked, and it doesn't contain the invalid pattern `)(state)`.\"], ['How can I map CommCare case data to create multiple FHIR patients in a single job?', 'To map CommCare case data to create multiple FHIR patients in a single job, you can use the `each()` function to iterate over the CommCare cases and create a FHIR patient for each one. Here\\'s an example of how you could structure your job:\\n\\n```javascript\\neach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patient = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcare.org/case-id\",\\n value: caseData.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }\\n ],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n { body: patient },\\n {\\n headers: {\\n \"Content-Type\": \"application/fhir+json\"\\n }\\n }\\n )(state);\\n })\\n);\\n```\\n\\nThis job does the following:\\n\\n1. Uses `each()` to iterate over all cases in the CommCare data.\\n2. For each case, it creates a FHIR Patient resource object.\\n3. It then uses a POST request to create the patient in your FHIR server.\\n\\nMake sure to replace \"https://your-fhir-server.com/Patient\" with the actual URL of your FHIR server\\'s Patient endpoint.\\n\\nYou may need to adjust the mapping based on your specific CommCare case structure and the FHIR Patient resource requirements. Also, consider adding error handling and logging as needed for your use case.', 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\", \"To create a new ODK form submission using data from multiple CommCare cases, you'll need to use a combination of functions from the Google Sheets adaptor and some data manipulation. Here's an example of how you might structure this:\\n\\n```javascript\\n// Assuming you have already fetched CommCare case data into state.data.cases\\n\\n// First, transform the CommCare case data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_Field_1': caseData.commcare_field_1,\\n 'ODK_Field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n \\n return { ...state, odkData };\\n});\\n\\n// Now append the transformed data to a Google Sheet\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData,\\n});\\n```\\n\\nThis code does the following:\\n\\n1. Uses `fn()` to transform the CommCare case data into the format expected by ODK.\\n2. Maps each CommCare case to an object with ODK field names.\\n3. Stores the transformed data in `state.odkData`.\\n4. Uses `appendValues()` to add the transformed data to a Google Sheet, which can then be used as a data source for ODK.\\n\\nRemember to adjust the field mappings, spreadsheet ID, and range according to your specific CommCare and ODK form structures.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to create ODK form submissions using CommCare case data, including code examples and step-by-step instructions. While it contains `fn(state =>`, it doesn't contain the exact pattern `)(state)`, so there's no regex match. The answer addresses the question directly, so it's successful.\"], ['How do I transform a Google Sheets row into a CommCare case update?', \"To transform a Google Sheets row into a CommCare case update, you'll need to follow these general steps:\\n\\n1. Fetch the data from Google Sheets\\n2. Transform the data into the format required by CommCare\\n3. Send the update to CommCare\\n\\nHere's a basic outline of how you might structure this job:\\n\\n```javascript\\n// Fetch data from Google Sheets\\ngetValues('your-spreadsheet-id', 'Sheet1!A2:E', state => {\\n const rows = state.data.values;\\n \\n // Transform and send each row to CommCare\\n return each(rows, state => {\\n const [caseId, name, age, status] = state.data;\\n \\n // Prepare the case update\\n const caseUpdate = {\\n case_id: caseId,\\n properties: {\\n name: name,\\n age: age,\\n status: status\\n }\\n };\\n \\n // Here you would use a CommCare adaptor function to send the update\\n // This is a placeholder as the actual function depends on the CommCare adaptor\\n // return updateCase(caseUpdate);\\n })(state);\\n});\\n```\\n\\nNote that this is a skeleton and you'll need to adjust it based on:\\n\\n1. The actual structure of your Google Sheet\\n2. The specific fields you're updating in CommCare\\n3. The actual function names and parameters for the CommCare adaptor\\n\\nAlso, you'll need to ensure you have the CommCare adaptor installed and configured in your project alongside the Google Sheets adaptor.\\n\\nIs there anything specific about your sheets structure or CommCare case properties that you'd like me to address?\", 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the best way to map nested CommCare form data to SatuSehat's FHIR resources?\", \"I apologize, but I don't have specific information about mapping CommCare form data to SatuSehat's FHIR resources. However, I can provide some general guidance on mapping data to FHIR resources using OpenFn and Google Sheets:\\n\\n1. First, you'll want to extract the relevant CommCare form data, likely using the CommCare adaptor for OpenFn.\\n\\n2. Then, you can use Google Sheets as an intermediary to map the CommCare data fields to the corresponding FHIR resource fields. \\n\\n3. You could create a Google Sheet with columns for the CommCare fields and corresponding FHIR fields.\\n\\n4. Use the Google Sheets adaptor to read this mapping information:\\n\\n```javascript\\ngetValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B100')\\n```\\n\\n5. Then use this mapping to transform your CommCare data into the proper FHIR format:\\n\\n```javascript\\nfn(state => {\\n const mappings = state.data.values;\\n const commcareData = state.data.commcareForm;\\n \\n const fhirResource = {};\\n \\n mappings.forEach(([commcareField, fhirField]) => {\\n if (commcareData[commcareField]) {\\n fhirResource[fhirField] = commcareData[commcareField];\\n }\\n });\\n \\n return { ...state, fhirResource };\\n})\\n```\\n\\n6. Finally, you'd use an HTTP adaptor or a specific FHIR adaptor (if available) to send this data to SatuSehat.\\n\\nThis is a general approach and would need to be adjusted based on the specific structure of your CommCare forms and the requirements of SatuSehat's FHIR implementation. You may need to handle nested data structures and data type conversions as well.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of a general approach to mapping data from CommCare to FHIR resources, including specific steps and code examples. While it acknowledges not having specific information about SatuSehat's implementation, it still provides a relevant and helpful response to the question about mapping nested CommCare form data to FHIR resources.\"], ['How can I merge data from multiple OpenMRS observations into a single Google Sheet row?', \"To merge data from multiple OpenMRS observations into a single Google Sheet row, you can follow these steps:\\n\\n1. Fetch the OpenMRS observations data (assuming you have this data in your state).\\n2. Process and transform the data to create a single row with the desired format.\\n3. Use the `appendValues` function to add the row to your Google Sheet.\\n\\nHere's an example of how you might structure your job:\\n\\n```javascript\\nfn(state => {\\n // Assume observations are in state.data.observations\\n const observations = state.data.observations;\\n\\n // Process observations to create a single row\\n const row = processObservations(observations);\\n\\n return { ...state, row };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:Z1', // Adjust the range based on your data\\n values: [state.row],\\n});\\n\\n// Helper function to process observations\\nfunction processObservations(observations) {\\n // Initialize an object to store the processed data\\n const processedData = {\\n patientId: observations[0].person.uuid,\\n encounterDate: observations[0].obsDatetime,\\n // Add more fields as needed\\n };\\n\\n // Loop through observations and add data to processedData\\n observations.forEach(obs => {\\n // Use the concept name as the key (you might want to map these to more readable names)\\n const conceptName = obs.concept.display;\\n processedData[conceptName] = obs.value;\\n });\\n\\n // Convert the processed data object to an array (for the spreadsheet row)\\n return Object.values(processedData);\\n}\\n```\\n\\nThis code does the following:\\n\\n1. It uses `fn()` to process the observations data, creating a single row.\\n2. The `processObservations` function takes the array of observations and merges them into a single object, then converts it to an array for the spreadsheet row.\\n3. It uses `appendValues()` to add the processed row to the Google Sheet.\\n\\nMake sure to adjust the spreadsheet ID, range, and the processing logic to match your specific data structure and requirements. Also, ensure that you have the necessary permissions and configuration set up for accessing both OpenMRS and Google Sheets.\", 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the correct way to use each() to process multiple FHIR resources and create CommCare cases?\", 'To use `each()` to process multiple FHIR resources and create CommCare cases, you can follow this pattern:\\n\\n```javascript\\neach(\\n \"$.bundle.entry[*].resource\",\\n fn(state => {\\n const resource = state.data;\\n \\n // Extract relevant data from the FHIR resource\\n const caseData = {\\n case_name: resource.name[0].text,\\n case_type: \"patient\",\\n owner_id: \"some_owner_id\",\\n // Add more fields as needed\\n };\\n\\n // Create a CommCare case using the extracted data\\n return getValues(\\n state.configuration.spreadsheetId,\\n \"Cases!A:Z\",\\n {\\n valueInputOption: \"USER_ENTERED\",\\n resource: [Object.values(caseData)]\\n }\\n )(state);\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. Uses `each()` to iterate over FHIR resources in the bundle.\\n2. Inside `each()`, we use `fn()` to process each resource.\\n3. We extract relevant data from the FHIR resource and create a `caseData` object.\\n4. We then use `getValues()` to add a new row to a Google Sheet, which can represent our CommCare cases.\\n\\nMake sure to adjust the field names and data extraction logic based on your specific FHIR resources and CommCare case structure.', 'v2', 'Regex Match: False\\nSuccess: True'], ['How do I map an array of ODK repeat groups to create multiple OpenMRS encounters?', 'To map an array of ODK repeat groups to create multiple OpenMRS encounters using the Google Sheets adaptor, you can use the `each` function to iterate over the repeat group data and create encounters for each item. Here\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.form.repeat_group[*]\",\\n fn(state => {\\n const repeatData = state.data;\\n \\n // Prepare the encounter data\\n const encounterData = {\\n spreadsheetId: \"your_spreadsheet_id_here\",\\n range: \"Encounters!A:Z\", // Adjust the range as needed\\n values: [\\n [\\n repeatData.encounter_date,\\n repeatData.patient_id,\\n repeatData.encounter_type,\\n // Add more fields as needed\\n ]\\n ]\\n };\\n\\n // Append the encounter data to the Google Sheet\\n return appendValues(encounterData)(state)\\n .then(state => {\\n console.log(\"Encounter created successfully\");\\n return state;\\n });\\n })\\n);\\n```\\n\\nThis code does the following:\\n\\n1. Uses `each` to iterate over items in the repeat group, which are assumed to be at `$.form.repeat_group[*]` in your source data.\\n\\n2. For each item in the repeat group, it prepares encounter data in a format suitable for Google Sheets.\\n\\n3. It then uses `appendValues` to add this data as a new row in your Google Sheet.\\n\\n4. The `fn` wrapper allows you to perform multiple operations and transformations for each item.\\n\\n5. After appending values, it logs a success message and returns the updated state.\\n\\nRemember to replace \"your_spreadsheet_id_here\" with your actual Google Sheets spreadsheet ID, and adjust the range and data fields to match your specific requirements.\\n\\nAlso, ensure that your Google Sheets is set up with appropriate columns to receive this encounter data.\\n\\nIf you need to perform any data transformations or add error handling, you can do so within the `fn` function before calling `appendValues`.', 'v2', 'Regex Match: False\\nSuccess: True'], [\"What's the syntax for filtering Google Sheets data before sending it to SatuSehat?\", \"I apologize, but there seems to be a misunderstanding in your question. The Google Sheets adaptor for OpenFn does not have built-in functionality to directly send data to SatuSehat. However, I can explain how you might filter Google Sheets data before using it in a subsequent operation.\\n\\nTo filter data from Google Sheets, you would typically:\\n\\n1. Fetch the data using `getValues()`\\n2. Use `fn()` to filter the data\\n3. Use the filtered data in your next operation\\n\\nHere's an example of how you might structure this:\\n\\n```javascript\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:E100')\\n.fn(state => {\\n const allData = state.data.values;\\n \\n // Filter the data\\n const filteredData = allData.filter(row => {\\n // Add your filtering logic here\\n // For example, to filter rows where the first column is not empty:\\n return row[0] !== '';\\n });\\n\\n return { ...state, filteredData };\\n})\\n// Use the filtered data in your next operation\\n// For example, if you were sending to another system:\\n// .then(postToAnotherSystem(state => state.filteredData))\\n```\\n\\nRemember to replace 'your-spreadsheet-id' with your actual Google Sheets ID, and adjust the range ('Sheet1!A1:E100') as needed.\\n\\nThe filtering logic in this example is very simple - you would replace it with whatever criteria you need to filter your data.\\n\\nIf you need to send this data to SatuSehat, you would need to use an appropriate adaptor or HTTP requests to interact with SatuSehat's API after filtering the Google Sheets data.\", 'v2', \"Regex Match: False\\nSuccess: True\\n\\nThe answer provides a detailed explanation of how to filter Google Sheets data, which addresses the core question about filtering Google Sheets data, even though it clarifies that direct integration with SatuSehat isn't built-in. The pattern `)(state)` is not present in the answer, and the response provides useful, relevant information about data filtering in Google Sheets.\"], ['How can I use combine() to merge data from multiple sources before creating a FHIR resource?', \"To use `combine()` to merge data from multiple sources before creating a FHIR resource, you can follow this approach:\\n\\n1. Use `getValues()` to fetch data from different Google Sheets.\\n2. Use `combine()` to merge the results of those operations.\\n3. Use `fn()` to process the combined data and create your FHIR resource.\\n\\nHere's an example:\\n\\n```javascript\\ncombine(\\n getValues('spreadsheetId1', 'Sheet1!A1:C10'),\\n getValues('spreadsheetId2', 'Sheet2!A1:D5')\\n)\\n\\nfn(state => {\\n const [data1, data2] = state.references;\\n \\n // Process and merge the data\\n const mergedData = {\\n // Combine data from both sources\\n ...data1[0],\\n ...data2[0]\\n };\\n\\n // Create your FHIR resource\\n const fhirResource = {\\n resourceType: 'Patient',\\n name: [{\\n given: [mergedData.firstName],\\n family: mergedData.lastName\\n }],\\n // Add more FHIR fields as needed\\n };\\n\\n console.log('Created FHIR resource:', fhirResource);\\n\\n return { ...state, fhirResource };\\n})\\n```\\n\\nThis approach allows you to fetch data from multiple Google Sheets, combine the operations, and then process the results to create your FHIR resource.\", 'v2', \"Let me evaluate this:\\n\\n1. First, checking if the answer addresses the question about using combine() to merge data:\\nThe answer provides a detailed explanation of using combine() with multiple data sources and includes a code example, so it does answer the question.\\n\\n2. Checking for the regex pattern `)(state)`:\\nLooking through the answer, while it contains `state` in various places, it doesn't contain the specific pattern `)(state)`.\\n\\nTherefore:\\nRegex Match: False\\nSuccess: True\\n\\nThe final answer is: True\"], [\"What's the proper way to map CommCare case properties to custom OpenMRS patient attributes?\", 'To map CommCare case properties to custom OpenMRS patient attributes, you\\'ll typically use the `each()` function to iterate over CommCare cases, then use `createPatient()` or `updatePatient()` operations from the OpenMRS adaptor to create or update patients with the mapped attributes. \\n\\nHere\\'s an example of how you might structure this:\\n\\n```javascript\\neach(\\n \"$.forms[*]\", // Assuming CommCare data is in state.data.forms\\n fn(state => {\\n const commCareCase = state.data;\\n \\n return createPatient({\\n person: {\\n names: [\\n {\\n givenName: dataValue(\"first_name\")(state),\\n familyName: dataValue(\"last_name\")(state)\\n }\\n ],\\n gender: dataValue(\"gender\")(state),\\n birthdate: dataValue(\"dob\")(state),\\n attributes: [\\n {\\n attributeType: \"00000000-0000-0000-0000-000000000000\", // Replace with actual UUID\\n value: dataValue(\"custom_field_1\")(state)\\n },\\n {\\n attributeType: \"11111111-1111-1111-1111-111111111111\", // Replace with actual UUID\\n value: dataValue(\"custom_field_2\")(state)\\n }\\n // Add more attribute mappings as needed\\n ]\\n }\\n })(state);\\n })\\n);\\n```\\n\\nKey points:\\n1. Use `each()` to iterate over CommCare cases.\\n2. Inside `each()`, use `fn()` to access the current case data.\\n3. Use `createPatient()` (or `updatePatient()` if updating existing patients) from the OpenMRS adaptor.\\n4. Map CommCare properties to OpenMRS fields using `dataValue()`.\\n5. For custom attributes, you\\'ll need to know the UUID of the attribute type in OpenMRS.\\n\\nRemember to replace the attribute type UUIDs with the actual UUIDs from your OpenMRS instance. Also, adjust the data paths (like \"$.forms[*]\") to match your actual CommCare data structure.', 'v2', 'Regex Match: True (the pattern `)(state)` appears in the code)\\nSuccess: False']]\n" - ] - } - ], - "source": [ - "print(results)" - ] - }, - { - "cell_type": "code", - "execution_count": 340, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "V1 success rate: 0.8888888888888888, V2 success rate: 0.9\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
questionanswersystem_prompt_versionresultcode
0How can I map CommCare case data to create mul...To map CommCare case data to create multiple F...v1NaNeach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...
1What's the syntax to create a new ODK form sub...To create a new ODK form submission using data...v1Falseeach(\\n \"$.cases[*]\",\\n appendValues({\\n ...
2How do I transform a Google Sheets row into a ...To transform a Google Sheets row into a CommCa...v1True// Fetch data from Google Sheets\\ngetValues('y...
3What's the best way to map nested CommCare for...I apologize, but I can't provide specific advi...v1TrueNone
4How can I merge data from multiple OpenMRS obs...To merge data from multiple OpenMRS observatio...v1Truefn(state => {\\n const patients = state.data.m...
5What's the correct way to use each() to proces...To use `each()` to process multiple FHIR resou...v1Trueeach(\\n \"$.bundle.entry[*].resource\",\\n fn(s...
6How do I map an array of ODK repeat groups to ...To map an array of ODK repeat groups to create...v1Trueeach(\\n \"$.form.repeat_group[*]\",\\n fn(state...
7What's the syntax for filtering Google Sheets ...I apologize, but there's a misunderstanding in...v1TruegetValues('your-spreadsheet-id', 'Sheet1!A1:E1...
8How can I use combine() to merge data from mul...To use `combine()` to merge data from multiple...v1Truecombine(\\n getValues('spreadsheetId1', 'Sheet...
9What's the proper way to map CommCare case pro...To map CommCare case properties to custom Open...v1Trueeach(\\n \"$.forms[*]\", // or wherever your Com...
10How can I map CommCare case data to create mul...To map CommCare case data to create multiple F...v2Trueeach(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...
11What's the syntax to create a new ODK form sub...To create a new ODK form submission using data...v2True// Assuming you have already fetched CommCare ...
12How do I transform a Google Sheets row into a ...To transform a Google Sheets row into a CommCa...v2True// Fetch data from Google Sheets\\ngetValues('y...
13What's the best way to map nested CommCare for...I apologize, but I don't have specific informa...v2TruegetValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B1...
14How can I merge data from multiple OpenMRS obs...To merge data from multiple OpenMRS observatio...v2Truefn(state => {\\n // Assume observations are in...
15What's the correct way to use each() to proces...To use `each()` to process multiple FHIR resou...v2Trueeach(\\n \"$.bundle.entry[*].resource\",\\n fn(s...
16How do I map an array of ODK repeat groups to ...To map an array of ODK repeat groups to create...v2Trueeach(\\n \"$.form.repeat_group[*]\",\\n fn(state...
17What's the syntax for filtering Google Sheets ...I apologize, but there seems to be a misunders...v2TruegetValues('your-spreadsheet-id', 'Sheet1!A1:E1...
18How can I use combine() to merge data from mul...To use `combine()` to merge data from multiple...v2Truecombine(\\n getValues('spreadsheetId1', 'Sheet...
19What's the proper way to map CommCare case pro...To map CommCare case properties to custom Open...v2Falseeach(\\n \"$.forms[*]\", // Assuming CommCare da...
\n", - "
" - ], - "text/plain": [ - " question \\\n", - "0 How can I map CommCare case data to create mul... \n", - "1 What's the syntax to create a new ODK form sub... \n", - "2 How do I transform a Google Sheets row into a ... \n", - "3 What's the best way to map nested CommCare for... \n", - "4 How can I merge data from multiple OpenMRS obs... \n", - "5 What's the correct way to use each() to proces... \n", - "6 How do I map an array of ODK repeat groups to ... \n", - "7 What's the syntax for filtering Google Sheets ... \n", - "8 How can I use combine() to merge data from mul... \n", - "9 What's the proper way to map CommCare case pro... \n", - "10 How can I map CommCare case data to create mul... \n", - "11 What's the syntax to create a new ODK form sub... \n", - "12 How do I transform a Google Sheets row into a ... \n", - "13 What's the best way to map nested CommCare for... \n", - "14 How can I merge data from multiple OpenMRS obs... \n", - "15 What's the correct way to use each() to proces... \n", - "16 How do I map an array of ODK repeat groups to ... \n", - "17 What's the syntax for filtering Google Sheets ... \n", - "18 How can I use combine() to merge data from mul... \n", - "19 What's the proper way to map CommCare case pro... \n", - "\n", - " answer system_prompt_version \\\n", - "0 To map CommCare case data to create multiple F... v1 \n", - "1 To create a new ODK form submission using data... v1 \n", - "2 To transform a Google Sheets row into a CommCa... v1 \n", - "3 I apologize, but I can't provide specific advi... v1 \n", - "4 To merge data from multiple OpenMRS observatio... v1 \n", - "5 To use `each()` to process multiple FHIR resou... v1 \n", - "6 To map an array of ODK repeat groups to create... v1 \n", - "7 I apologize, but there's a misunderstanding in... v1 \n", - "8 To use `combine()` to merge data from multiple... v1 \n", - "9 To map CommCare case properties to custom Open... v1 \n", - "10 To map CommCare case data to create multiple F... v2 \n", - "11 To create a new ODK form submission using data... v2 \n", - "12 To transform a Google Sheets row into a CommCa... v2 \n", - "13 I apologize, but I don't have specific informa... v2 \n", - "14 To merge data from multiple OpenMRS observatio... v2 \n", - "15 To use `each()` to process multiple FHIR resou... v2 \n", - "16 To map an array of ODK repeat groups to create... v2 \n", - "17 I apologize, but there seems to be a misunders... v2 \n", - "18 To use `combine()` to merge data from multiple... v2 \n", - "19 To map CommCare case properties to custom Open... v2 \n", - "\n", - " result code \n", - "0 NaN each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... \n", - "1 False each(\\n \"$.cases[*]\",\\n appendValues({\\n ... \n", - "2 True // Fetch data from Google Sheets\\ngetValues('y... \n", - "3 True None \n", - "4 True fn(state => {\\n const patients = state.data.m... \n", - "5 True each(\\n \"$.bundle.entry[*].resource\",\\n fn(s... \n", - "6 True each(\\n \"$.form.repeat_group[*]\",\\n fn(state... \n", - "7 True getValues('your-spreadsheet-id', 'Sheet1!A1:E1... \n", - "8 True combine(\\n getValues('spreadsheetId1', 'Sheet... \n", - "9 True each(\\n \"$.forms[*]\", // or wherever your Com... \n", - "10 True each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... \n", - "11 True // Assuming you have already fetched CommCare ... \n", - "12 True // Fetch data from Google Sheets\\ngetValues('y... \n", - "13 True getValues('YOUR_SPREADSHEET_ID', 'Sheet1!A1:B1... \n", - "14 True fn(state => {\\n // Assume observations are in... \n", - "15 True each(\\n \"$.bundle.entry[*].resource\",\\n fn(s... \n", - "16 True each(\\n \"$.form.repeat_group[*]\",\\n fn(state... \n", - "17 True getValues('your-spreadsheet-id', 'Sheet1!A1:E1... \n", - "18 True combine(\\n getValues('spreadsheetId1', 'Sheet... \n", - "19 False each(\\n \"$.forms[*]\", // Assuming CommCare da... " - ] - }, - "execution_count": 340, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "import os\n", - "import re\n", - "\n", - "def extract_success(result_text):\n", - " # Use regex to extract \"Success: True\" or \"Success: False\"\n", - " import re\n", - " match = re.search(r'Success:\\s*(True|False)', result_text)\n", - " return match.group(1) if match else None\n", - "\n", - "def extract_code(answer_text):\n", - " # Regex to match code inside ```javascript ... ```\n", - " match = re.search(r\"```javascript\\s*([\\s\\S]*?)\\s*```\", answer_text)\n", - " return match.group(1).strip() if match else None\n", - "\n", - "\n", - "results_df = pd.DataFrame(results, columns=[\"question\", \"answer\", \"system_prompt_version\", \"result\"])\n", - "results_df['code'] = results_df['answer'].apply(extract_code)\n", - "results_df['result'] = results_df['result'].apply(extract_success).map({'True': True, 'False': False})\n", - "v1_mean = results_df[results_df['system_prompt_version'] == 'v1']['result'].mean()\n", - "v2_mean = results_df[results_df['system_prompt_version'] == 'v2']['result'].mean()\n", - "\n", - "print(f\"V1 success rate: {v1_mean}, V2 success rate: {v2_mean}\")\n", - "downloads_path = os.path.join(os.path.expanduser(\"~\"), \"Downloads\", \"202411_claude_system_prompt_issue_97.csv\")\n", - "results_df.to_csv(downloads_path, index=False)\n", - "\n", - "results_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "apollo", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/services/job_chat/evaluation/test_prompts_issue_97.ipynb b/services/job_chat/evaluation/test_related_questions.ipynb similarity index 100% rename from services/job_chat/evaluation/test_prompts_issue_97.ipynb rename to services/job_chat/evaluation/test_related_questions.ipynb From 6619bc29ad7f9e1806a270934c4091463f09f42f Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Sat, 7 Dec 2024 20:56:47 +0300 Subject: [PATCH 4/6] remove comments Signed-off-by: Hunter Achieng --- .../evaluation/test_nested_state.ipynb | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/services/job_chat/evaluation/test_nested_state.ipynb b/services/job_chat/evaluation/test_nested_state.ipynb index 22a6ba4..c5361dc 100644 --- a/services/job_chat/evaluation/test_nested_state.ipynb +++ b/services/job_chat/evaluation/test_nested_state.ipynb @@ -400,29 +400,13 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", - "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", @@ -434,7 +418,7 @@ "import pandas as pd\n", "import re\n", "\n", - "# Function to extract JavaScript code\n", + "# Extract JavaScript code\n", "def extract_code(answer_text):\n", " \"\"\"\n", " Extracts JavaScript code from an answer text enclosed in ```javascript ... ```.\n", @@ -442,20 +426,20 @@ " match = re.search(r\"```javascript\\s*([\\s\\S]*?)\\s*```\", answer_text)\n", " return match.group(1).strip() if match else None\n", "\n", - "# Function to check for `)(state)` pattern\n", + "# Check for `)(state)` pattern\n", "def check_nested_state(code):\n", " \"\"\"\n", " Checks for the presence of the `)(state)` pattern in the code.\n", " \"\"\"\n", " return bool(re.search(r\"\\)\\(state\\)\", code)) if code else False\n", "\n", - "# Evaluate QA pairs\n", + "\n", "results = []\n", "\n", "for q, a, version in qa_pairs:\n", " evaluate_answer_template_filled = evaluate_answer_template.format(question=q, answer=a)\n", "\n", - " # Send to AI for evaluation\n", + "\n", " message = client.messages.create(\n", " model=\"claude-3-5-sonnet-20241022\",\n", " max_tokens=1000,\n", @@ -473,13 +457,12 @@ " }\n", " ]\n", " )\n", - " \n", - " # AI response and programmatic check\n", + "\n", " ai_result = message.content[0].text.strip()\n", " code = extract_code(a)\n", " contains_nested_state = check_nested_state(code)\n", " \n", - " # Append results\n", + "\n", " results.append([q, code, version, contains_nested_state])\n" ] }, From c1703ca0a0386497d5bcdeffdb83dbecef5e3c6c Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Sat, 7 Dec 2024 20:58:33 +0300 Subject: [PATCH 5/6] Signed-off-by: Hunter Achieng (achienghunter@gmail.com) Signed-off-by: Hunter Achieng --- .../evaluation/test_nested_state.ipynb | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/services/job_chat/evaluation/test_nested_state.ipynb b/services/job_chat/evaluation/test_nested_state.ipynb index c5361dc..d09752e 100644 --- a/services/job_chat/evaluation/test_nested_state.ipynb +++ b/services/job_chat/evaluation/test_nested_state.ipynb @@ -407,6 +407,22 @@ "name": "stdout", "output_type": "stream", "text": [ + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", "INFO:httpx:HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", @@ -468,7 +484,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -739,7 +755,7 @@ "19 True False " ] }, - "execution_count": 26, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } From b16db195df64e3ff5a5a5fd1fd138219b8e13716 Mon Sep 17 00:00:00 2001 From: Hunter Achieng Date: Mon, 9 Dec 2024 19:57:15 +0300 Subject: [PATCH 6/6] allow code reading Signed-off-by: Hunter Achieng --- .../evaluation/test_nested_state.ipynb | 295 +++++++++++------- 1 file changed, 189 insertions(+), 106 deletions(-) diff --git a/services/job_chat/evaluation/test_nested_state.ipynb b/services/job_chat/evaluation/test_nested_state.ipynb index d09752e..ca8925b 100644 --- a/services/job_chat/evaluation/test_nested_state.ipynb +++ b/services/job_chat/evaluation/test_nested_state.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -52,7 +52,28 @@ "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2.9.0.post0)\n", "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2024.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n" + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\n", + "Collecting jsbeautifier\n", + " Downloading jsbeautifier-1.15.1.tar.gz (75 kB)\n", + " Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", + "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25hRequirement already satisfied: six>=1.13.0 in /opt/anaconda3/envs/apollo/lib/python3.11/site-packages (from jsbeautifier) (1.16.0)\n", + "Collecting editorconfig>=0.12.2 (from jsbeautifier)\n", + " Downloading EditorConfig-0.12.4.tar.gz (13 kB)\n", + " Installing build dependencies ... \u001b[?25ldone\n", + "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", + "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25hBuilding wheels for collected packages: jsbeautifier, editorconfig\n", + " Building wheel for jsbeautifier (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for jsbeautifier: filename=jsbeautifier-1.15.1-py3-none-any.whl size=94709 sha256=4fba67b8f360e679edfc3cdf554e0d34aa0f7099f366ec0ad12456038456133d\n", + " Stored in directory: /Users/hunter/Library/Caches/pip/wheels/32/eb/d8/07a66b0cf535ed05084d0b9a920b89e35f20512a5e28e615ca\n", + " Building wheel for editorconfig (pyproject.toml) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for editorconfig: filename=EditorConfig-0.12.4-py3-none-any.whl size=16361 sha256=eae2e58b99f91dbf9cc8bb62f395df18aa2019754ea1d0896c1146d296613d14\n", + " Stored in directory: /Users/hunter/Library/Caches/pip/wheels/d8/d7/36/17a1b838b1a805065c05529b1607ea4df55d5923dcdaf9cf7b\n", + "Successfully built jsbeautifier editorconfig\n", + "Installing collected packages: editorconfig, jsbeautifier\n", + "Successfully installed editorconfig-0.12.4 jsbeautifier-1.15.1\n" ] } ], @@ -62,6 +83,8 @@ "!pip install tenacity\n", "!pip install apollo\n", "!pip install pandas\n", + "!pip install jsbeautifier\n", + "\n", "\n" ] }, @@ -484,7 +507,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -520,250 +543,310 @@ " system_prompt_version\n", " contains_nested_state\n", " success\n", + " formatted_code\n", " \n", " \n", " \n", " \n", " 0\n", - " How can I map CommCare case data to create mul...\n", - " each(\\n \"$.forms[*].cases[*]\",\\n fn(state =>...\n", + " How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + " each(\\n \"$.forms[*].cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcarehq.org/case\",\\n value: caseData.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }\\n ],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n { body: patientData },\\n {\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n);\n", " v1\n", " True\n", " False\n", + " each(\\n \"$.forms[*].cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [{\\n system: \"https://www.commcarehq.org/case\",\\n value: caseData.case_id\\n }],\\n name: [{\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\", {\\n body: patientData\\n }, {\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n);\n", " \n", " \n", " 1\n", - " What's the syntax to create a new ODK form sub...\n", - " fn(state => {\\n const commcareCases = state.d...\n", + " What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + " fn(state => {\\n const commcareCases = state.data.cases; // Assume CommCare cases are in state.data.cases\\n \\n // Transform CommCare data into the format needed for ODK\\n const odkSubmissionData = commcareCases.map(caseData => [\\n caseData.case_id,\\n caseData.name,\\n caseData.date_of_birth,\\n // Add more fields as needed\\n ]);\\n\\n return { ...state, odkSubmissionData };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\\n range: 'Sheet1!A1:D1', // Adjust the range based on your data\\n values: state.odkSubmissionData,\\n});\n", " v1\n", " False\n", " True\n", + " fn(state => {\\n const commcareCases = state.data.cases; // Assume CommCare cases are in state.data.cases\\n\\n // Transform CommCare data into the format needed for ODK\\n const odkSubmissionData = commcareCases.map(caseData => [\\n caseData.case_id,\\n caseData.name,\\n caseData.date_of_birth,\\n // Add more fields as needed\\n ]);\\n\\n return {\\n ...state,\\n odkSubmissionData\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\\n range: 'Sheet1!A1:D1', // Adjust the range based on your data\\n values: state.odkSubmissionData,\\n});\n", " \n", " \n", " 2\n", - " How do I merge data from multiple Google Sheet...\n", - " // Get data from Sheet 1\\ngetValues(\\n '1O-a4...\n", + " How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record?\n", + " // Get data from Sheet 1\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet1!A2:E',\\n state => {\\n const patients = state.data.values.map(row => ({\\n id: row[0],\\n name: row[1],\\n age: row[2]\\n }));\\n return { ...state, patients };\\n }\\n);\\n\\n// Get data from Sheet 2\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet2!A2:C',\\n state => {\\n const addresses = state.data.values.map(row => ({\\n id: row[0],\\n address: row[1],\\n city: row[2]\\n }));\\n return { ...state, addresses };\\n }\\n);\\n\\n// Combine and transform data\\nfn(state => {\\n const mergedData = state.patients.map(patient => {\\n const address = state.addresses.find(a => a.id === patient.id);\\n return {\\n ...patient,\\n ...address\\n };\\n });\\n return { ...state, mergedData };\\n});\\n\\n// Create or update OpenMRS patient records\\neach(\\n 'mergedData[*]',\\n post(\\n 'https://openmrs-instance.org/ws/rest/v1/patient',\\n state => ({\\n person: {\\n names: [{ givenName: state.data.name }],\\n age: state.data.age,\\n addresses: [\\n {\\n address1: state.data.address,\\n cityVillage: state.data.city\\n }\\n ]\\n }\\n })\\n )\\n);\n", " v1\n", " False\n", " True\n", + " // Get data from Sheet 1\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet1!A2:E',\\n state => {\\n const patients = state.data.values.map(row => ({\\n id: row[0],\\n name: row[1],\\n age: row[2]\\n }));\\n return {\\n ...state,\\n patients\\n };\\n }\\n);\\n\\n// Get data from Sheet 2\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet2!A2:C',\\n state => {\\n const addresses = state.data.values.map(row => ({\\n id: row[0],\\n address: row[1],\\n city: row[2]\\n }));\\n return {\\n ...state,\\n addresses\\n };\\n }\\n);\\n\\n// Combine and transform data\\nfn(state => {\\n const mergedData = state.patients.map(patient => {\\n const address = state.addresses.find(a => a.id === patient.id);\\n return {\\n ...patient,\\n ...address\\n };\\n });\\n return {\\n ...state,\\n mergedData\\n };\\n});\\n\\n// Create or update OpenMRS patient records\\neach(\\n 'mergedData[*]',\\n post(\\n 'https://openmrs-instance.org/ws/rest/v1/patient',\\n state => ({\\n person: {\\n names: [{\\n givenName: state.data.name\\n }],\\n age: state.data.age,\\n addresses: [{\\n address1: state.data.address,\\n cityVillage: state.data.city\\n }]\\n }\\n })\\n )\\n);\n", " \n", " \n", " 3\n", - " What's the best way to transform an array of C...\n", - " each(\\n \"$.data.referrals[*]\",\\n fn(state =>...\n", + " What's the best way to transform an array of CommCare referrals into Satu Sehat encounters?\n", + " each(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n const encounter = {\\n // Map CommCare referral fields to Satu Sehat encounter fields\\n encounterId: referral.id,\\n patientId: referral.patient_id,\\n date: referral.referral_date,\\n // Add more fields as needed\\n };\\n return { ...state, encounter };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A1:Z1\",\\n values: state => [\\n [\\n state.encounter.encounterId,\\n state.encounter.patientId,\\n state.encounter.date,\\n // Add more fields as needed\\n ]\\n ]\\n});\n", " v1\n", " False\n", " True\n", + " each(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n const encounter = {\\n // Map CommCare referral fields to Satu Sehat encounter fields\\n encounterId: referral.id,\\n patientId: referral.patient_id,\\n date: referral.referral_date,\\n // Add more fields as needed\\n };\\n return {\\n ...state,\\n encounter\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A1:Z1\",\\n values: state => [\\n [\\n state.encounter.encounterId,\\n state.encounter.patientId,\\n state.encounter.date,\\n // Add more fields as needed\\n ]\\n ]\\n});\n", " \n", " \n", " 4\n", - " How can I split a single FHIR Bundle resource ...\n", - " fn(state => {\\n const bundle = state.data.bun...\n", + " How can I split a single FHIR Bundle resource into multiple Google Sheets rows?\n", + " fn(state => {\\n const bundle = state.data.bundle; // Assume the FHIR Bundle is in state.data.bundle\\n \\n // Extract patient data from the bundle\\n const patients = bundle.entry\\n .filter(entry => entry.resource.resourceType === 'Patient')\\n .map(entry => {\\n const patient = entry.resource;\\n return [\\n patient.id,\\n patient.name[0]?.family,\\n patient.name[0]?.given?.join(' '),\\n patient.gender,\\n patient.birthDate\\n ];\\n });\\n\\n return { ...state, patients };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.patients\\n});\n", " v1\n", " False\n", " True\n", + " fn(state => {\\n const bundle = state.data.bundle; // Assume the FHIR Bundle is in state.data.bundle\\n\\n // Extract patient data from the bundle\\n const patients = bundle.entry\\n .filter(entry => entry.resource.resourceType === 'Patient')\\n .map(entry => {\\n const patient = entry.resource;\\n return [\\n patient.id,\\n patient.name[0]?.family,\\n patient.name[0]?.given?.join(' '),\\n patient.gender,\\n patient.birthDate\\n ];\\n });\\n\\n return {\\n ...state,\\n patients\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.patients\\n});\n", " \n", " \n", " 5\n", - " What's the correct way to map nested CommCare ...\n", - " each(\\n \"$.data.form.observations[*]\",\\n fn(...\n", + " What's the correct way to map nested CommCare form data to FHIR Observation resources?\n", + " each(\\n \"$.data.form.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const fhirObservation = {\\n resourceType: \"Observation\",\\n status: \"final\",\\n code: {\\n coding: [\\n {\\n system: \"http://loinc.org\",\\n code: obs.code,\\n display: obs.name\\n }\\n ]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date\\n };\\n\\n return { ...state, fhirObservation };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\",\\n values: state => [\\n [\\n JSON.stringify(state.fhirObservation)\\n ]\\n ]\\n});\n", " v1\n", " False\n", " True\n", + " each(\\n \"$.data.form.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const fhirObservation = {\\n resourceType: \"Observation\",\\n status: \"final\",\\n code: {\\n coding: [{\\n system: \"http://loinc.org\",\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date\\n };\\n\\n return {\\n ...state,\\n fhirObservation\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\",\\n values: state => [\\n [\\n JSON.stringify(state.fhirObservation)\\n ]\\n ]\\n});\n", " \n", " \n", " 6\n", - " How do I create multiple Google Sheets workshe...\n", - " fn(state => {\\n // Assume state.data contains...\n", + " How do I create multiple Google Sheets worksheets from a single OpenMRS patient query?\n", + " fn(state => {\\n // Assume state.data contains the result of your OpenMRS query\\n const patients = state.data.patients;\\n\\n // Organize data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.birthDate]);\\n const contactInfo = patients.map(p => [p.id, p.phoneNumber, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return { ...state, basicInfo, contactInfo, medicalInfo };\\n});\\n\\nfn(state => {\\n // Create a new spreadsheet\\n const spreadsheetId = 'YOUR_SPREADSHEET_ID'; // Replace with actual ID or method to create new spreadsheet\\n\\n return { ...state, spreadsheetId };\\n});\\n\\nfn(state => {\\n // Create multiple worksheets (you'd need to implement this function)\\n const sheetNames = ['Basic Info', 'Contact Info', 'Medical Info'];\\n // Assuming createSheets is a function you've implemented to create sheets\\n return createSheets(state.spreadsheetId, sheetNames);\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Basic Info!A1',\\n values: state => [['ID', 'Name', 'Birth Date'], ...state.basicInfo]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Contact Info!A1',\\n values: state => [['ID', 'Phone', 'Email'], ...state.contactInfo]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Medical Info!A1',\\n values: state => [['ID', 'Diagnosis', 'Medication'], ...state.medicalInfo]\\n});\n", " v1\n", " False\n", " True\n", + " fn(state => {\\n // Assume state.data contains the result of your OpenMRS query\\n const patients = state.data.patients;\\n\\n // Organize data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.birthDate]);\\n const contactInfo = patients.map(p => [p.id, p.phoneNumber, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {\\n ...state,\\n basicInfo,\\n contactInfo,\\n medicalInfo\\n };\\n});\\n\\nfn(state => {\\n // Create a new spreadsheet\\n const spreadsheetId = 'YOUR_SPREADSHEET_ID'; // Replace with actual ID or method to create new spreadsheet\\n\\n return {\\n ...state,\\n spreadsheetId\\n };\\n});\\n\\nfn(state => {\\n // Create multiple worksheets (you'd need to implement this function)\\n const sheetNames = ['Basic Info', 'Contact Info', 'Medical Info'];\\n // Assuming createSheets is a function you've implemented to create sheets\\n return createSheets(state.spreadsheetId, sheetNames);\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Basic Info!A1',\\n values: state => [\\n ['ID', 'Name', 'Birth Date'], ...state.basicInfo\\n ]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Contact Info!A1',\\n values: state => [\\n ['ID', 'Phone', 'Email'], ...state.contactInfo\\n ]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Medical Info!A1',\\n values: state => [\\n ['ID', 'Diagnosis', 'Medication'], ...state.medicalInfo\\n ]\\n});\n", " \n", " \n", " 7\n", - " What's the syntax for transforming ODK repeat ...\n", + " What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources?\n", " None\n", " v1\n", " False\n", " True\n", + " None\n", " \n", " \n", " 8\n", - " How can I combine data from multiple Satu Seha...\n", - " getValues('your_spreadsheet_id', 'Sheet1!A1:Z1...\n", + " How can I combine data from multiple Satu Sehat resources to create a single CommCare case?\n", + " getValues('your_spreadsheet_id', 'Sheet1!A1:Z100', state => {\\n const satuSehatData = state.data.values;\\n // Process the data here\\n return { ...state, satuSehatData };\\n});\n", " v1\n", " False\n", " True\n", + " getValues('your_spreadsheet_id', 'Sheet1!A1:Z100', state => {\\n const satuSehatData = state.data.values;\\n // Process the data here\\n return {\\n ...state,\\n satuSehatData\\n };\\n});\n", " \n", " \n", " 9\n", - " What's the proper way to map OpenMRS observati...\n", - " each(\\n \"$.observations[*]\",\\n fn(state => {...\n", + " What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy?\n", + " each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.name,\\n obs.value,\\n obs.obsDatetime,\\n obs.encounter.encounterType.name,\\n obs.encounter.patient.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:E\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n);\n", " v1\n", " True\n", " False\n", + " each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.name,\\n obs.value,\\n obs.obsDatetime,\\n obs.encounter.encounterType.name,\\n obs.encounter.patient.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:E\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n);\n", " \n", " \n", " 10\n", - " How can I map CommCare case data to create mul...\n", - " each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n...\n", + " How can I map CommCare case data to create multiple FHIR patients in a single job?\n", + " each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const commCareCase = state.data;\\n \\n const fhirPatient = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcarehq.org/case\",\\n value: commCareCase.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: commCareCase.last_name,\\n given: [commCareCase.first_name]\\n }\\n ],\\n gender: commCareCase.gender,\\n birthDate: commCareCase.dob\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n {\\n body: fhirPatient,\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n);\n", " v2\n", " True\n", " False\n", + " each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const commCareCase = state.data;\\n\\n const fhirPatient = {\\n resourceType: \"Patient\",\\n identifier: [{\\n system: \"https://www.commcarehq.org/case\",\\n value: commCareCase.case_id\\n }],\\n name: [{\\n use: \"official\",\\n family: commCareCase.last_name,\\n given: [commCareCase.first_name]\\n }],\\n gender: commCareCase.gender,\\n birthDate: commCareCase.dob\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\", {\\n body: fhirPatient,\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n);\n", " \n", " \n", " 11\n", - " What's the syntax to create a new ODK form sub...\n", - " // Assuming you have data from CommCare cases ...\n", + " What's the syntax to create a new ODK form submission using data from multiple CommCare cases?\n", + " // Assuming you have data from CommCare cases in state.data.cases\\n\\n// First, transform the CommCare data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_field_1': caseData.commcare_field_1,\\n 'ODK_field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n \\n return { ...state, odkData };\\n});\\n\\n// Now append the transformed data to a Google Sheet (acting as an ODK form)\\nappendValues({\\n spreadsheetId: 'your_spreadsheet_id_here',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData\\n});\n", " v2\n", " False\n", " True\n", + " // Assuming you have data from CommCare cases in state.data.cases\\n\\n// First, transform the CommCare data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_field_1': caseData.commcare_field_1,\\n 'ODK_field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n\\n return {\\n ...state,\\n odkData\\n };\\n});\\n\\n// Now append the transformed data to a Google Sheet (acting as an ODK form)\\nappendValues({\\n spreadsheetId: 'your_spreadsheet_id_here',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData\\n});\n", " \n", " \n", " 12\n", - " How do I merge data from multiple Google Sheet...\n", - " // Fetch data from Google Sheets tabs\\ngetValu...\n", + " How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record?\n", + " // Fetch data from Google Sheets tabs\\ngetValues('your-spreadsheet-id', 'Tab1!A1:Z')\\nfn(state => {\\n const tab1Data = state.data.values;\\n return { ...state, tab1Data };\\n})\\n\\ngetValues('your-spreadsheet-id', 'Tab2!A1:Z')\\nfn(state => {\\n const tab2Data = state.data.values;\\n return { ...state, tab2Data };\\n})\\n\\n// Combine data into patient object\\nfn(state => {\\n const patientData = {\\n identifier: state.tab1Data[0][0], // Assuming patient ID is in first cell of Tab1\\n name: state.tab1Data[0][1],\\n age: state.tab2Data[0][0],\\n // Add more fields as needed\\n };\\n return { ...state, patientData };\\n})\\n\\n// Send data to OpenMRS (you'll need to use the OpenMRS adaptor for this part)\\n// This is just a placeholder to show where you'd send the data\\n// createPatient(state.patientData)\n", " v2\n", " False\n", " True\n", + " // Fetch data from Google Sheets tabs\\ngetValues('your-spreadsheet-id', 'Tab1!A1:Z')\\nfn(state => {\\n const tab1Data = state.data.values;\\n return {\\n ...state,\\n tab1Data\\n };\\n})\\n\\ngetValues('your-spreadsheet-id', 'Tab2!A1:Z')\\nfn(state => {\\n const tab2Data = state.data.values;\\n return {\\n ...state,\\n tab2Data\\n };\\n})\\n\\n// Combine data into patient object\\nfn(state => {\\n const patientData = {\\n identifier: state.tab1Data[0][0], // Assuming patient ID is in first cell of Tab1\\n name: state.tab1Data[0][1],\\n age: state.tab2Data[0][0],\\n // Add more fields as needed\\n };\\n return {\\n ...state,\\n patientData\\n };\\n})\\n\\n// Send data to OpenMRS (you'll need to use the OpenMRS adaptor for this part)\\n// This is just a placeholder to show where you'd send the data\\n// createPatient(state.patientData)\n", " \n", " \n", " 13\n", - " What's the best way to transform an array of C...\n", - " // Assuming referrals are in state.data.referr...\n", + " What's the best way to transform an array of CommCare referrals into Satu Sehat encounters?\n", + " // Assuming referrals are in state.data.referrals\\neach(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n \\n // Transform referral to Satu Sehat encounter format\\n const encounter = {\\n resourceType: \"Encounter\",\\n status: \"finished\",\\n class: {\\n system: \"http://terminology.hl7.org/CodeSystem/v3-ActCode\",\\n code: \"AMB\",\\n display: \"ambulatory\"\\n },\\n subject: {\\n reference: `Patient/${referral.patient_id}`\\n },\\n period: {\\n start: referral.referral_date,\\n end: referral.referral_date\\n },\\n // Add more fields as needed\\n };\\n\\n return {...state, encounter};\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: state => [\\n [\\n state.encounter.resourceType,\\n state.encounter.status,\\n state.encounter.class.code,\\n state.encounter.subject.reference,\\n state.encounter.period.start,\\n state.encounter.period.end\\n // Add more fields as needed\\n ]\\n ]\\n});\n", " v2\n", " False\n", " True\n", + " // Assuming referrals are in state.data.referrals\\neach(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n\\n // Transform referral to Satu Sehat encounter format\\n const encounter = {\\n resourceType: \"Encounter\",\\n status: \"finished\",\\n class: {\\n system: \"http://terminology.hl7.org/CodeSystem/v3-ActCode\",\\n code: \"AMB\",\\n display: \"ambulatory\"\\n },\\n subject: {\\n reference: `Patient/${referral.patient_id}`\\n },\\n period: {\\n start: referral.referral_date,\\n end: referral.referral_date\\n },\\n // Add more fields as needed\\n };\\n\\n return {\\n ...state,\\n encounter\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: state => [\\n [\\n state.encounter.resourceType,\\n state.encounter.status,\\n state.encounter.class.code,\\n state.encounter.subject.reference,\\n state.encounter.period.start,\\n state.encounter.period.end\\n // Add more fields as needed\\n ]\\n ]\\n});\n", " \n", " \n", " 14\n", - " How can I split a single FHIR Bundle resource ...\n", - " fn(state => {\\n const bundle = state.data.bun...\n", + " How can I split a single FHIR Bundle resource into multiple Google Sheets rows?\n", + " fn(state => {\\n const bundle = state.data.bundle; // Assuming the FHIR Bundle is in state.data.bundle\\n const rows = [];\\n\\n // Process each entry in the bundle\\n bundle.entry.forEach(entry => {\\n const resource = entry.resource;\\n \\n // Extract relevant data based on resource type\\n if (resource.resourceType === 'Patient') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.name[0]?.family,\\n resource.name[0]?.given.join(' '),\\n resource.gender,\\n resource.birthDate\\n ]);\\n } else if (resource.resourceType === 'Observation') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.code.coding[0]?.display,\\n resource.valueQuantity?.value,\\n resource.valueQuantity?.unit,\\n resource.effectiveDateTime\\n ]);\\n }\\n // Add more resource types as needed\\n });\\n\\n return { ...state, rows };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:F1',\\n values: state.rows\\n});\n", " v2\n", " False\n", " True\n", + " fn(state => {\\n const bundle = state.data.bundle; // Assuming the FHIR Bundle is in state.data.bundle\\n const rows = [];\\n\\n // Process each entry in the bundle\\n bundle.entry.forEach(entry => {\\n const resource = entry.resource;\\n\\n // Extract relevant data based on resource type\\n if (resource.resourceType === 'Patient') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.name[0]?.family,\\n resource.name[0]?.given.join(' '),\\n resource.gender,\\n resource.birthDate\\n ]);\\n } else if (resource.resourceType === 'Observation') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.code.coding[0]?.display,\\n resource.valueQuantity?.value,\\n resource.valueQuantity?.unit,\\n resource.effectiveDateTime\\n ]);\\n }\\n // Add more resource types as needed\\n });\\n\\n return {\\n ...state,\\n rows\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:F1',\\n values: state.rows\\n});\n", " \n", " \n", " 15\n", - " What's the correct way to map nested CommCare ...\n", - " each(\\n '$.data.form.observations[*]',\\n fn(...\n", + " What's the correct way to map nested CommCare form data to FHIR Observation resources?\n", + " each(\\n '$.data.form.observations[*]',\\n fn(state => {\\n const obs = state.data;\\n const fhirObs = {\\n resourceType: 'Observation',\\n status: 'final',\\n code: {\\n coding: [{\\n system: 'http://loinc.org',\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date,\\n subject: {\\n reference: `Patient/${obs.patient_id}`\\n }\\n };\\n \\n return appendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'FHIRObservations',\\n values: [[\\n JSON.stringify(fhirObs)\\n ]]\\n })(state);\\n })\\n);\n", " v2\n", " True\n", " False\n", + " each(\\n '$.data.form.observations[*]',\\n fn(state => {\\n const obs = state.data;\\n const fhirObs = {\\n resourceType: 'Observation',\\n status: 'final',\\n code: {\\n coding: [{\\n system: 'http://loinc.org',\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date,\\n subject: {\\n reference: `Patient/${obs.patient_id}`\\n }\\n };\\n\\n return appendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'FHIRObservations',\\n values: [\\n [\\n JSON.stringify(fhirObs)\\n ]\\n ]\\n })(state);\\n })\\n);\n", " \n", " \n", " 16\n", - " How do I create multiple Google Sheets workshe...\n", - " fn(state => {\\n const patients = state.data.p...\n", + " How do I create multiple Google Sheets worksheets from a single OpenMRS patient query?\n", + " fn(state => {\\n const patients = state.data.patients; // Assuming the OpenMRS data is in state.data.patients\\n \\n // Process data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.age]);\\n const contactInfo = patients.map(p => [p.id, p.phone, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {...state, worksheetData: {basicInfo, contactInfo, medicalInfo}};\\n});\\n\\nfn(state => {\\n const spreadsheetId = '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos'; // Replace with your actual spreadsheet ID\\n \\n // Define worksheet names and their corresponding data\\n const worksheets = [\\n {name: 'Basic Info', data: state.worksheetData.basicInfo},\\n {name: 'Contact Info', data: state.worksheetData.contactInfo},\\n {name: 'Medical Info', data: state.worksheetData.medicalInfo}\\n ];\\n\\n // Create operations for each worksheet\\n const operations = worksheets.map(sheet => \\n appendValues({\\n spreadsheetId,\\n range: `${sheet.name}!A1`,\\n values: sheet.data,\\n })\\n );\\n\\n return each(operations, state => state);\\n});\n", " v2\n", " False\n", " True\n", + " fn(state => {\\n const patients = state.data.patients; // Assuming the OpenMRS data is in state.data.patients\\n\\n // Process data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.age]);\\n const contactInfo = patients.map(p => [p.id, p.phone, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {\\n ...state,\\n worksheetData: {\\n basicInfo,\\n contactInfo,\\n medicalInfo\\n }\\n };\\n});\\n\\nfn(state => {\\n const spreadsheetId = '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos'; // Replace with your actual spreadsheet ID\\n\\n // Define worksheet names and their corresponding data\\n const worksheets = [{\\n name: 'Basic Info',\\n data: state.worksheetData.basicInfo\\n }, {\\n name: 'Contact Info',\\n data: state.worksheetData.contactInfo\\n }, {\\n name: 'Medical Info',\\n data: state.worksheetData.medicalInfo\\n }];\\n\\n // Create operations for each worksheet\\n const operations = worksheets.map(sheet =>\\n appendValues({\\n spreadsheetId,\\n range: `${sheet.name}!A1`,\\n values: sheet.data,\\n })\\n );\\n\\n return each(operations, state => state);\\n});\n", " \n", " \n", " 17\n", - " What's the syntax for transforming ODK repeat ...\n", + " What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources?\n", " None\n", " v2\n", " False\n", " True\n", + " None\n", " \n", " \n", " 18\n", - " How can I combine data from multiple Satu Seha...\n", - " // Fetch data from Satu Sehat resources (assum...\n", + " How can I combine data from multiple Satu Sehat resources to create a single CommCare case?\n", + " // Fetch data from Satu Sehat resources (assuming it's stored in Google Sheets)\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:Z100')\\n .then(state => {\\n const satuSehatData = state.data.values;\\n \\n // Process and combine the data\\n const combinedData = processDataForCommCare(satuSehatData);\\n \\n // Create CommCare case (you'd need to use the CommCare adaptor for this)\\n // This is just a placeholder to show where you'd create the case\\n return createCommCareCase(combinedData);\\n });\\n\\n// Helper function to process Satu Sehat data\\nfunction processDataForCommCare(data) {\\n // Your logic to combine and format data goes here\\n // This is just a simple example\\n return data.reduce((acc, row) => {\\n // Combine data as needed\\n return {\\n ...acc,\\n [row[0]]: row[1]\\n };\\n }, {});\\n}\\n\\n// Placeholder for CommCare case creation\\nfunction createCommCareCase(data) {\\n // You'd use the CommCare adaptor here to actually create the case\\n console.log('Creating CommCare case with data:', data);\\n return state => {\\n // Update state or perform additional operations\\n return { ...state, createdCase: data };\\n };\\n}\n", " v2\n", " False\n", " True\n", + " // Fetch data from Satu Sehat resources (assuming it's stored in Google Sheets)\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:Z100')\\n .then(state => {\\n const satuSehatData = state.data.values;\\n\\n // Process and combine the data\\n const combinedData = processDataForCommCare(satuSehatData);\\n\\n // Create CommCare case (you'd need to use the CommCare adaptor for this)\\n // This is just a placeholder to show where you'd create the case\\n return createCommCareCase(combinedData);\\n });\\n\\n// Helper function to process Satu Sehat data\\nfunction processDataForCommCare(data) {\\n // Your logic to combine and format data goes here\\n // This is just a simple example\\n return data.reduce((acc, row) => {\\n // Combine data as needed\\n return {\\n ...acc,\\n [row[0]]: row[1]\\n };\\n }, {});\\n}\\n\\n// Placeholder for CommCare case creation\\nfunction createCommCareCase(data) {\\n // You'd use the CommCare adaptor here to actually create the case\\n console.log('Creating CommCare case with data:', data);\\n return state => {\\n // Update state or perform additional operations\\n return {\\n ...state,\\n createdCase: data\\n };\\n };\\n}\n", " \n", " \n", " 19\n", - " What's the proper way to map OpenMRS observati...\n", - " each(\\n \"$.observations[*]\",\\n fn(state => {...\n", + " What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy?\n", + " each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.display,\\n obs.value.display || obs.value,\\n obs.obsDatetime,\\n obs.person.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:D\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n);\n", " v2\n", " True\n", " False\n", + " each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.display,\\n obs.value.display || obs.value,\\n obs.obsDatetime,\\n obs.person.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:D\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n);\n", " \n", " \n", "\n", "" ], "text/plain": [ - " question \\\n", - "0 How can I map CommCare case data to create mul... \n", - "1 What's the syntax to create a new ODK form sub... \n", - "2 How do I merge data from multiple Google Sheet... \n", - "3 What's the best way to transform an array of C... \n", - "4 How can I split a single FHIR Bundle resource ... \n", - "5 What's the correct way to map nested CommCare ... \n", - "6 How do I create multiple Google Sheets workshe... \n", - "7 What's the syntax for transforming ODK repeat ... \n", - "8 How can I combine data from multiple Satu Seha... \n", - "9 What's the proper way to map OpenMRS observati... \n", - "10 How can I map CommCare case data to create mul... \n", - "11 What's the syntax to create a new ODK form sub... \n", - "12 How do I merge data from multiple Google Sheet... \n", - "13 What's the best way to transform an array of C... \n", - "14 How can I split a single FHIR Bundle resource ... \n", - "15 What's the correct way to map nested CommCare ... \n", - "16 How do I create multiple Google Sheets workshe... \n", - "17 What's the syntax for transforming ODK repeat ... \n", - "18 How can I combine data from multiple Satu Seha... \n", - "19 What's the proper way to map OpenMRS observati... \n", + " question \\\n", + "0 How can I map CommCare case data to create multiple FHIR patients in a single job? \n", + "1 What's the syntax to create a new ODK form submission using data from multiple CommCare cases? \n", + "2 How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record? \n", + "3 What's the best way to transform an array of CommCare referrals into Satu Sehat encounters? \n", + "4 How can I split a single FHIR Bundle resource into multiple Google Sheets rows? \n", + "5 What's the correct way to map nested CommCare form data to FHIR Observation resources? \n", + "6 How do I create multiple Google Sheets worksheets from a single OpenMRS patient query? \n", + "7 What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources? \n", + "8 How can I combine data from multiple Satu Sehat resources to create a single CommCare case? \n", + "9 What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy? \n", + "10 How can I map CommCare case data to create multiple FHIR patients in a single job? \n", + "11 What's the syntax to create a new ODK form submission using data from multiple CommCare cases? \n", + "12 How do I merge data from multiple Google Sheets tabs into a single OpenMRS patient record? \n", + "13 What's the best way to transform an array of CommCare referrals into Satu Sehat encounters? \n", + "14 How can I split a single FHIR Bundle resource into multiple Google Sheets rows? \n", + "15 What's the correct way to map nested CommCare form data to FHIR Observation resources? \n", + "16 How do I create multiple Google Sheets worksheets from a single OpenMRS patient query? \n", + "17 What's the syntax for transforming ODK repeat groups into separate FHIR Encounter resources? \n", + "18 How can I combine data from multiple Satu Sehat resources to create a single CommCare case? \n", + "19 What's the proper way to map OpenMRS observations to multiple Google Sheets cells while preserving the hierarchy? \n", "\n", - " code system_prompt_version \\\n", - "0 each(\\n \"$.forms[*].cases[*]\",\\n fn(state =>... v1 \n", - "1 fn(state => {\\n const commcareCases = state.d... v1 \n", - "2 // Get data from Sheet 1\\ngetValues(\\n '1O-a4... v1 \n", - "3 each(\\n \"$.data.referrals[*]\",\\n fn(state =>... v1 \n", - "4 fn(state => {\\n const bundle = state.data.bun... v1 \n", - "5 each(\\n \"$.data.form.observations[*]\",\\n fn(... v1 \n", - "6 fn(state => {\\n // Assume state.data contains... v1 \n", - "7 None v1 \n", - "8 getValues('your_spreadsheet_id', 'Sheet1!A1:Z1... v1 \n", - "9 each(\\n \"$.observations[*]\",\\n fn(state => {... v1 \n", - "10 each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n... v2 \n", - "11 // Assuming you have data from CommCare cases ... v2 \n", - "12 // Fetch data from Google Sheets tabs\\ngetValu... v2 \n", - "13 // Assuming referrals are in state.data.referr... v2 \n", - "14 fn(state => {\\n const bundle = state.data.bun... v2 \n", - "15 each(\\n '$.data.form.observations[*]',\\n fn(... v2 \n", - "16 fn(state => {\\n const patients = state.data.p... v2 \n", - "17 None v2 \n", - "18 // Fetch data from Satu Sehat resources (assum... v2 \n", - "19 each(\\n \"$.observations[*]\",\\n fn(state => {... v2 \n", + " code \\\n", + "0 each(\\n \"$.forms[*].cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcarehq.org/case\",\\n value: caseData.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }\\n ],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n { body: patientData },\\n {\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n); \n", + "1 fn(state => {\\n const commcareCases = state.data.cases; // Assume CommCare cases are in state.data.cases\\n \\n // Transform CommCare data into the format needed for ODK\\n const odkSubmissionData = commcareCases.map(caseData => [\\n caseData.case_id,\\n caseData.name,\\n caseData.date_of_birth,\\n // Add more fields as needed\\n ]);\\n\\n return { ...state, odkSubmissionData };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\\n range: 'Sheet1!A1:D1', // Adjust the range based on your data\\n values: state.odkSubmissionData,\\n}); \n", + "2 // Get data from Sheet 1\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet1!A2:E',\\n state => {\\n const patients = state.data.values.map(row => ({\\n id: row[0],\\n name: row[1],\\n age: row[2]\\n }));\\n return { ...state, patients };\\n }\\n);\\n\\n// Get data from Sheet 2\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet2!A2:C',\\n state => {\\n const addresses = state.data.values.map(row => ({\\n id: row[0],\\n address: row[1],\\n city: row[2]\\n }));\\n return { ...state, addresses };\\n }\\n);\\n\\n// Combine and transform data\\nfn(state => {\\n const mergedData = state.patients.map(patient => {\\n const address = state.addresses.find(a => a.id === patient.id);\\n return {\\n ...patient,\\n ...address\\n };\\n });\\n return { ...state, mergedData };\\n});\\n\\n// Create or update OpenMRS patient records\\neach(\\n 'mergedData[*]',\\n post(\\n 'https://openmrs-instance.org/ws/rest/v1/patient',\\n state => ({\\n person: {\\n names: [{ givenName: state.data.name }],\\n age: state.data.age,\\n addresses: [\\n {\\n address1: state.data.address,\\n cityVillage: state.data.city\\n }\\n ]\\n }\\n })\\n )\\n); \n", + "3 each(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n const encounter = {\\n // Map CommCare referral fields to Satu Sehat encounter fields\\n encounterId: referral.id,\\n patientId: referral.patient_id,\\n date: referral.referral_date,\\n // Add more fields as needed\\n };\\n return { ...state, encounter };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A1:Z1\",\\n values: state => [\\n [\\n state.encounter.encounterId,\\n state.encounter.patientId,\\n state.encounter.date,\\n // Add more fields as needed\\n ]\\n ]\\n}); \n", + "4 fn(state => {\\n const bundle = state.data.bundle; // Assume the FHIR Bundle is in state.data.bundle\\n \\n // Extract patient data from the bundle\\n const patients = bundle.entry\\n .filter(entry => entry.resource.resourceType === 'Patient')\\n .map(entry => {\\n const patient = entry.resource;\\n return [\\n patient.id,\\n patient.name[0]?.family,\\n patient.name[0]?.given?.join(' '),\\n patient.gender,\\n patient.birthDate\\n ];\\n });\\n\\n return { ...state, patients };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.patients\\n}); \n", + "5 each(\\n \"$.data.form.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const fhirObservation = {\\n resourceType: \"Observation\",\\n status: \"final\",\\n code: {\\n coding: [\\n {\\n system: \"http://loinc.org\",\\n code: obs.code,\\n display: obs.name\\n }\\n ]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date\\n };\\n\\n return { ...state, fhirObservation };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\",\\n values: state => [\\n [\\n JSON.stringify(state.fhirObservation)\\n ]\\n ]\\n}); \n", + "6 fn(state => {\\n // Assume state.data contains the result of your OpenMRS query\\n const patients = state.data.patients;\\n\\n // Organize data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.birthDate]);\\n const contactInfo = patients.map(p => [p.id, p.phoneNumber, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return { ...state, basicInfo, contactInfo, medicalInfo };\\n});\\n\\nfn(state => {\\n // Create a new spreadsheet\\n const spreadsheetId = 'YOUR_SPREADSHEET_ID'; // Replace with actual ID or method to create new spreadsheet\\n\\n return { ...state, spreadsheetId };\\n});\\n\\nfn(state => {\\n // Create multiple worksheets (you'd need to implement this function)\\n const sheetNames = ['Basic Info', 'Contact Info', 'Medical Info'];\\n // Assuming createSheets is a function you've implemented to create sheets\\n return createSheets(state.spreadsheetId, sheetNames);\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Basic Info!A1',\\n values: state => [['ID', 'Name', 'Birth Date'], ...state.basicInfo]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Contact Info!A1',\\n values: state => [['ID', 'Phone', 'Email'], ...state.contactInfo]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Medical Info!A1',\\n values: state => [['ID', 'Diagnosis', 'Medication'], ...state.medicalInfo]\\n}); \n", + "7 None \n", + "8 getValues('your_spreadsheet_id', 'Sheet1!A1:Z100', state => {\\n const satuSehatData = state.data.values;\\n // Process the data here\\n return { ...state, satuSehatData };\\n}); \n", + "9 each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.name,\\n obs.value,\\n obs.obsDatetime,\\n obs.encounter.encounterType.name,\\n obs.encounter.patient.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:E\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n); \n", + "10 each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const commCareCase = state.data;\\n \\n const fhirPatient = {\\n resourceType: \"Patient\",\\n identifier: [\\n {\\n system: \"https://www.commcarehq.org/case\",\\n value: commCareCase.case_id\\n }\\n ],\\n name: [\\n {\\n use: \"official\",\\n family: commCareCase.last_name,\\n given: [commCareCase.first_name]\\n }\\n ],\\n gender: commCareCase.gender,\\n birthDate: commCareCase.dob\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\",\\n {\\n body: fhirPatient,\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n); \n", + "11 // Assuming you have data from CommCare cases in state.data.cases\\n\\n// First, transform the CommCare data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_field_1': caseData.commcare_field_1,\\n 'ODK_field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n \\n return { ...state, odkData };\\n});\\n\\n// Now append the transformed data to a Google Sheet (acting as an ODK form)\\nappendValues({\\n spreadsheetId: 'your_spreadsheet_id_here',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData\\n}); \n", + "12 // Fetch data from Google Sheets tabs\\ngetValues('your-spreadsheet-id', 'Tab1!A1:Z')\\nfn(state => {\\n const tab1Data = state.data.values;\\n return { ...state, tab1Data };\\n})\\n\\ngetValues('your-spreadsheet-id', 'Tab2!A1:Z')\\nfn(state => {\\n const tab2Data = state.data.values;\\n return { ...state, tab2Data };\\n})\\n\\n// Combine data into patient object\\nfn(state => {\\n const patientData = {\\n identifier: state.tab1Data[0][0], // Assuming patient ID is in first cell of Tab1\\n name: state.tab1Data[0][1],\\n age: state.tab2Data[0][0],\\n // Add more fields as needed\\n };\\n return { ...state, patientData };\\n})\\n\\n// Send data to OpenMRS (you'll need to use the OpenMRS adaptor for this part)\\n// This is just a placeholder to show where you'd send the data\\n// createPatient(state.patientData) \n", + "13 // Assuming referrals are in state.data.referrals\\neach(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n \\n // Transform referral to Satu Sehat encounter format\\n const encounter = {\\n resourceType: \"Encounter\",\\n status: \"finished\",\\n class: {\\n system: \"http://terminology.hl7.org/CodeSystem/v3-ActCode\",\\n code: \"AMB\",\\n display: \"ambulatory\"\\n },\\n subject: {\\n reference: `Patient/${referral.patient_id}`\\n },\\n period: {\\n start: referral.referral_date,\\n end: referral.referral_date\\n },\\n // Add more fields as needed\\n };\\n\\n return {...state, encounter};\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: state => [\\n [\\n state.encounter.resourceType,\\n state.encounter.status,\\n state.encounter.class.code,\\n state.encounter.subject.reference,\\n state.encounter.period.start,\\n state.encounter.period.end\\n // Add more fields as needed\\n ]\\n ]\\n}); \n", + "14 fn(state => {\\n const bundle = state.data.bundle; // Assuming the FHIR Bundle is in state.data.bundle\\n const rows = [];\\n\\n // Process each entry in the bundle\\n bundle.entry.forEach(entry => {\\n const resource = entry.resource;\\n \\n // Extract relevant data based on resource type\\n if (resource.resourceType === 'Patient') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.name[0]?.family,\\n resource.name[0]?.given.join(' '),\\n resource.gender,\\n resource.birthDate\\n ]);\\n } else if (resource.resourceType === 'Observation') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.code.coding[0]?.display,\\n resource.valueQuantity?.value,\\n resource.valueQuantity?.unit,\\n resource.effectiveDateTime\\n ]);\\n }\\n // Add more resource types as needed\\n });\\n\\n return { ...state, rows };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:F1',\\n values: state.rows\\n}); \n", + "15 each(\\n '$.data.form.observations[*]',\\n fn(state => {\\n const obs = state.data;\\n const fhirObs = {\\n resourceType: 'Observation',\\n status: 'final',\\n code: {\\n coding: [{\\n system: 'http://loinc.org',\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date,\\n subject: {\\n reference: `Patient/${obs.patient_id}`\\n }\\n };\\n \\n return appendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'FHIRObservations',\\n values: [[\\n JSON.stringify(fhirObs)\\n ]]\\n })(state);\\n })\\n); \n", + "16 fn(state => {\\n const patients = state.data.patients; // Assuming the OpenMRS data is in state.data.patients\\n \\n // Process data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.age]);\\n const contactInfo = patients.map(p => [p.id, p.phone, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {...state, worksheetData: {basicInfo, contactInfo, medicalInfo}};\\n});\\n\\nfn(state => {\\n const spreadsheetId = '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos'; // Replace with your actual spreadsheet ID\\n \\n // Define worksheet names and their corresponding data\\n const worksheets = [\\n {name: 'Basic Info', data: state.worksheetData.basicInfo},\\n {name: 'Contact Info', data: state.worksheetData.contactInfo},\\n {name: 'Medical Info', data: state.worksheetData.medicalInfo}\\n ];\\n\\n // Create operations for each worksheet\\n const operations = worksheets.map(sheet => \\n appendValues({\\n spreadsheetId,\\n range: `${sheet.name}!A1`,\\n values: sheet.data,\\n })\\n );\\n\\n return each(operations, state => state);\\n}); \n", + "17 None \n", + "18 // Fetch data from Satu Sehat resources (assuming it's stored in Google Sheets)\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:Z100')\\n .then(state => {\\n const satuSehatData = state.data.values;\\n \\n // Process and combine the data\\n const combinedData = processDataForCommCare(satuSehatData);\\n \\n // Create CommCare case (you'd need to use the CommCare adaptor for this)\\n // This is just a placeholder to show where you'd create the case\\n return createCommCareCase(combinedData);\\n });\\n\\n// Helper function to process Satu Sehat data\\nfunction processDataForCommCare(data) {\\n // Your logic to combine and format data goes here\\n // This is just a simple example\\n return data.reduce((acc, row) => {\\n // Combine data as needed\\n return {\\n ...acc,\\n [row[0]]: row[1]\\n };\\n }, {});\\n}\\n\\n// Placeholder for CommCare case creation\\nfunction createCommCareCase(data) {\\n // You'd use the CommCare adaptor here to actually create the case\\n console.log('Creating CommCare case with data:', data);\\n return state => {\\n // Update state or perform additional operations\\n return { ...state, createdCase: data };\\n };\\n} \n", + "19 each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.display,\\n obs.value.display || obs.value,\\n obs.obsDatetime,\\n obs.person.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:D\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n); \n", "\n", - " contains_nested_state success \n", - "0 True False \n", - "1 False True \n", - "2 False True \n", - "3 False True \n", - "4 False True \n", - "5 False True \n", - "6 False True \n", - "7 False True \n", - "8 False True \n", - "9 True False \n", - "10 True False \n", - "11 False True \n", - "12 False True \n", - "13 False True \n", - "14 False True \n", - "15 True False \n", - "16 False True \n", - "17 False True \n", - "18 False True \n", - "19 True False " + " system_prompt_version contains_nested_state success \\\n", + "0 v1 True False \n", + "1 v1 False True \n", + "2 v1 False True \n", + "3 v1 False True \n", + "4 v1 False True \n", + "5 v1 False True \n", + "6 v1 False True \n", + "7 v1 False True \n", + "8 v1 False True \n", + "9 v1 True False \n", + "10 v2 True False \n", + "11 v2 False True \n", + "12 v2 False True \n", + "13 v2 False True \n", + "14 v2 False True \n", + "15 v2 True False \n", + "16 v2 False True \n", + "17 v2 False True \n", + "18 v2 False True \n", + "19 v2 True False \n", + "\n", + " formatted_code \n", + "0 each(\\n \"$.forms[*].cases[*]\",\\n fn(state => {\\n const caseData = state.data;\\n const patientData = {\\n resourceType: \"Patient\",\\n identifier: [{\\n system: \"https://www.commcarehq.org/case\",\\n value: caseData.case_id\\n }],\\n name: [{\\n use: \"official\",\\n family: caseData.last_name,\\n given: [caseData.first_name]\\n }],\\n gender: caseData.gender,\\n birthDate: caseData.date_of_birth\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\", {\\n body: patientData\\n }, {\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n); \n", + "1 fn(state => {\\n const commcareCases = state.data.cases; // Assume CommCare cases are in state.data.cases\\n\\n // Transform CommCare data into the format needed for ODK\\n const odkSubmissionData = commcareCases.map(caseData => [\\n caseData.case_id,\\n caseData.name,\\n caseData.date_of_birth,\\n // Add more fields as needed\\n ]);\\n\\n return {\\n ...state,\\n odkSubmissionData\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos', // Replace with your actual spreadsheet ID\\n range: 'Sheet1!A1:D1', // Adjust the range based on your data\\n values: state.odkSubmissionData,\\n}); \n", + "2 // Get data from Sheet 1\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet1!A2:E',\\n state => {\\n const patients = state.data.values.map(row => ({\\n id: row[0],\\n name: row[1],\\n age: row[2]\\n }));\\n return {\\n ...state,\\n patients\\n };\\n }\\n);\\n\\n// Get data from Sheet 2\\ngetValues(\\n '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n 'Sheet2!A2:C',\\n state => {\\n const addresses = state.data.values.map(row => ({\\n id: row[0],\\n address: row[1],\\n city: row[2]\\n }));\\n return {\\n ...state,\\n addresses\\n };\\n }\\n);\\n\\n// Combine and transform data\\nfn(state => {\\n const mergedData = state.patients.map(patient => {\\n const address = state.addresses.find(a => a.id === patient.id);\\n return {\\n ...patient,\\n ...address\\n };\\n });\\n return {\\n ...state,\\n mergedData\\n };\\n});\\n\\n// Create or update OpenMRS patient records\\neach(\\n 'mergedData[*]',\\n post(\\n 'https://openmrs-instance.org/ws/rest/v1/patient',\\n state => ({\\n person: {\\n names: [{\\n givenName: state.data.name\\n }],\\n age: state.data.age,\\n addresses: [{\\n address1: state.data.address,\\n cityVillage: state.data.city\\n }]\\n }\\n })\\n )\\n); \n", + "3 each(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n const encounter = {\\n // Map CommCare referral fields to Satu Sehat encounter fields\\n encounterId: referral.id,\\n patientId: referral.patient_id,\\n date: referral.referral_date,\\n // Add more fields as needed\\n };\\n return {\\n ...state,\\n encounter\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A1:Z1\",\\n values: state => [\\n [\\n state.encounter.encounterId,\\n state.encounter.patientId,\\n state.encounter.date,\\n // Add more fields as needed\\n ]\\n ]\\n}); \n", + "4 fn(state => {\\n const bundle = state.data.bundle; // Assume the FHIR Bundle is in state.data.bundle\\n\\n // Extract patient data from the bundle\\n const patients = bundle.entry\\n .filter(entry => entry.resource.resourceType === 'Patient')\\n .map(entry => {\\n const patient = entry.resource;\\n return [\\n patient.id,\\n patient.name[0]?.family,\\n patient.name[0]?.given?.join(' '),\\n patient.gender,\\n patient.birthDate\\n ];\\n });\\n\\n return {\\n ...state,\\n patients\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:E1',\\n values: state.patients\\n}); \n", + "5 each(\\n \"$.data.form.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const fhirObservation = {\\n resourceType: \"Observation\",\\n status: \"final\",\\n code: {\\n coding: [{\\n system: \"http://loinc.org\",\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date\\n };\\n\\n return {\\n ...state,\\n fhirObservation\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\",\\n values: state => [\\n [\\n JSON.stringify(state.fhirObservation)\\n ]\\n ]\\n}); \n", + "6 fn(state => {\\n // Assume state.data contains the result of your OpenMRS query\\n const patients = state.data.patients;\\n\\n // Organize data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.birthDate]);\\n const contactInfo = patients.map(p => [p.id, p.phoneNumber, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {\\n ...state,\\n basicInfo,\\n contactInfo,\\n medicalInfo\\n };\\n});\\n\\nfn(state => {\\n // Create a new spreadsheet\\n const spreadsheetId = 'YOUR_SPREADSHEET_ID'; // Replace with actual ID or method to create new spreadsheet\\n\\n return {\\n ...state,\\n spreadsheetId\\n };\\n});\\n\\nfn(state => {\\n // Create multiple worksheets (you'd need to implement this function)\\n const sheetNames = ['Basic Info', 'Contact Info', 'Medical Info'];\\n // Assuming createSheets is a function you've implemented to create sheets\\n return createSheets(state.spreadsheetId, sheetNames);\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Basic Info!A1',\\n values: state => [\\n ['ID', 'Name', 'Birth Date'], ...state.basicInfo\\n ]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Contact Info!A1',\\n values: state => [\\n ['ID', 'Phone', 'Email'], ...state.contactInfo\\n ]\\n});\\n\\nappendValues({\\n spreadsheetId: state => state.spreadsheetId,\\n range: 'Medical Info!A1',\\n values: state => [\\n ['ID', 'Diagnosis', 'Medication'], ...state.medicalInfo\\n ]\\n}); \n", + "7 None \n", + "8 getValues('your_spreadsheet_id', 'Sheet1!A1:Z100', state => {\\n const satuSehatData = state.data.values;\\n // Process the data here\\n return {\\n ...state,\\n satuSehatData\\n };\\n}); \n", + "9 each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.name,\\n obs.value,\\n obs.obsDatetime,\\n obs.encounter.encounterType.name,\\n obs.encounter.patient.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:E\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n); \n", + "10 each(\\n \"$.data.cases[*]\",\\n fn(state => {\\n const commCareCase = state.data;\\n\\n const fhirPatient = {\\n resourceType: \"Patient\",\\n identifier: [{\\n system: \"https://www.commcarehq.org/case\",\\n value: commCareCase.case_id\\n }],\\n name: [{\\n use: \"official\",\\n family: commCareCase.last_name,\\n given: [commCareCase.first_name]\\n }],\\n gender: commCareCase.gender,\\n birthDate: commCareCase.dob\\n };\\n\\n return post(\\n \"https://your-fhir-server.com/Patient\", {\\n body: fhirPatient,\\n headers: {\\n 'Content-Type': 'application/fhir+json'\\n }\\n }\\n )(state);\\n })\\n); \n", + "11 // Assuming you have data from CommCare cases in state.data.cases\\n\\n// First, transform the CommCare data into the format needed for ODK\\nfn(state => {\\n const odkData = state.data.cases.map(caseData => ({\\n // Map CommCare fields to ODK fields\\n 'ODK_field_1': caseData.commcare_field_1,\\n 'ODK_field_2': caseData.commcare_field_2,\\n // Add more field mappings as needed\\n }));\\n\\n return {\\n ...state,\\n odkData\\n };\\n});\\n\\n// Now append the transformed data to a Google Sheet (acting as an ODK form)\\nappendValues({\\n spreadsheetId: 'your_spreadsheet_id_here',\\n range: 'Sheet1!A1:Z1', // Adjust the range as needed\\n values: state.odkData\\n}); \n", + "12 // Fetch data from Google Sheets tabs\\ngetValues('your-spreadsheet-id', 'Tab1!A1:Z')\\nfn(state => {\\n const tab1Data = state.data.values;\\n return {\\n ...state,\\n tab1Data\\n };\\n})\\n\\ngetValues('your-spreadsheet-id', 'Tab2!A1:Z')\\nfn(state => {\\n const tab2Data = state.data.values;\\n return {\\n ...state,\\n tab2Data\\n };\\n})\\n\\n// Combine data into patient object\\nfn(state => {\\n const patientData = {\\n identifier: state.tab1Data[0][0], // Assuming patient ID is in first cell of Tab1\\n name: state.tab1Data[0][1],\\n age: state.tab2Data[0][0],\\n // Add more fields as needed\\n };\\n return {\\n ...state,\\n patientData\\n };\\n})\\n\\n// Send data to OpenMRS (you'll need to use the OpenMRS adaptor for this part)\\n// This is just a placeholder to show where you'd send the data\\n// createPatient(state.patientData) \n", + "13 // Assuming referrals are in state.data.referrals\\neach(\\n \"$.data.referrals[*]\",\\n fn(state => {\\n const referral = state.data;\\n\\n // Transform referral to Satu Sehat encounter format\\n const encounter = {\\n resourceType: \"Encounter\",\\n status: \"finished\",\\n class: {\\n system: \"http://terminology.hl7.org/CodeSystem/v3-ActCode\",\\n code: \"AMB\",\\n display: \"ambulatory\"\\n },\\n subject: {\\n reference: `Patient/${referral.patient_id}`\\n },\\n period: {\\n start: referral.referral_date,\\n end: referral.referral_date\\n },\\n // Add more fields as needed\\n };\\n\\n return {\\n ...state,\\n encounter\\n };\\n })\\n);\\n\\nappendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID\",\\n range: \"Sheet1!A:Z\", // Adjust range as needed\\n values: state => [\\n [\\n state.encounter.resourceType,\\n state.encounter.status,\\n state.encounter.class.code,\\n state.encounter.subject.reference,\\n state.encounter.period.start,\\n state.encounter.period.end\\n // Add more fields as needed\\n ]\\n ]\\n}); \n", + "14 fn(state => {\\n const bundle = state.data.bundle; // Assuming the FHIR Bundle is in state.data.bundle\\n const rows = [];\\n\\n // Process each entry in the bundle\\n bundle.entry.forEach(entry => {\\n const resource = entry.resource;\\n\\n // Extract relevant data based on resource type\\n if (resource.resourceType === 'Patient') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.name[0]?.family,\\n resource.name[0]?.given.join(' '),\\n resource.gender,\\n resource.birthDate\\n ]);\\n } else if (resource.resourceType === 'Observation') {\\n rows.push([\\n resource.id,\\n resource.resourceType,\\n resource.code.coding[0]?.display,\\n resource.valueQuantity?.value,\\n resource.valueQuantity?.unit,\\n resource.effectiveDateTime\\n ]);\\n }\\n // Add more resource types as needed\\n });\\n\\n return {\\n ...state,\\n rows\\n };\\n});\\n\\nappendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'Sheet1!A1:F1',\\n values: state.rows\\n}); \n", + "15 each(\\n '$.data.form.observations[*]',\\n fn(state => {\\n const obs = state.data;\\n const fhirObs = {\\n resourceType: 'Observation',\\n status: 'final',\\n code: {\\n coding: [{\\n system: 'http://loinc.org',\\n code: obs.code,\\n display: obs.name\\n }]\\n },\\n valueQuantity: {\\n value: obs.value,\\n unit: obs.unit\\n },\\n effectiveDateTime: obs.date,\\n subject: {\\n reference: `Patient/${obs.patient_id}`\\n }\\n };\\n\\n return appendValues({\\n spreadsheetId: '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos',\\n range: 'FHIRObservations',\\n values: [\\n [\\n JSON.stringify(fhirObs)\\n ]\\n ]\\n })(state);\\n })\\n); \n", + "16 fn(state => {\\n const patients = state.data.patients; // Assuming the OpenMRS data is in state.data.patients\\n\\n // Process data into separate arrays\\n const basicInfo = patients.map(p => [p.id, p.name, p.age]);\\n const contactInfo = patients.map(p => [p.id, p.phone, p.email]);\\n const medicalInfo = patients.map(p => [p.id, p.diagnosis, p.medication]);\\n\\n return {\\n ...state,\\n worksheetData: {\\n basicInfo,\\n contactInfo,\\n medicalInfo\\n }\\n };\\n});\\n\\nfn(state => {\\n const spreadsheetId = '1O-a4_RgPF_p8W3I6b5M9wobA3-CBW8hLClZfUik5sos'; // Replace with your actual spreadsheet ID\\n\\n // Define worksheet names and their corresponding data\\n const worksheets = [{\\n name: 'Basic Info',\\n data: state.worksheetData.basicInfo\\n }, {\\n name: 'Contact Info',\\n data: state.worksheetData.contactInfo\\n }, {\\n name: 'Medical Info',\\n data: state.worksheetData.medicalInfo\\n }];\\n\\n // Create operations for each worksheet\\n const operations = worksheets.map(sheet =>\\n appendValues({\\n spreadsheetId,\\n range: `${sheet.name}!A1`,\\n values: sheet.data,\\n })\\n );\\n\\n return each(operations, state => state);\\n}); \n", + "17 None \n", + "18 // Fetch data from Satu Sehat resources (assuming it's stored in Google Sheets)\\ngetValues('your-spreadsheet-id', 'Sheet1!A1:Z100')\\n .then(state => {\\n const satuSehatData = state.data.values;\\n\\n // Process and combine the data\\n const combinedData = processDataForCommCare(satuSehatData);\\n\\n // Create CommCare case (you'd need to use the CommCare adaptor for this)\\n // This is just a placeholder to show where you'd create the case\\n return createCommCareCase(combinedData);\\n });\\n\\n// Helper function to process Satu Sehat data\\nfunction processDataForCommCare(data) {\\n // Your logic to combine and format data goes here\\n // This is just a simple example\\n return data.reduce((acc, row) => {\\n // Combine data as needed\\n return {\\n ...acc,\\n [row[0]]: row[1]\\n };\\n }, {});\\n}\\n\\n// Placeholder for CommCare case creation\\nfunction createCommCareCase(data) {\\n // You'd use the CommCare adaptor here to actually create the case\\n console.log('Creating CommCare case with data:', data);\\n return state => {\\n // Update state or perform additional operations\\n return {\\n ...state,\\n createdCase: data\\n };\\n };\\n} \n", + "19 each(\\n \"$.observations[*]\",\\n fn(state => {\\n const obs = state.data;\\n const row = [\\n obs.concept.display,\\n obs.value.display || obs.value,\\n obs.obsDatetime,\\n obs.person.display\\n ];\\n\\n return appendValues({\\n spreadsheetId: \"YOUR_SPREADSHEET_ID_HERE\",\\n range: \"Sheet1!A:D\", // Adjust this range as needed\\n values: [row]\\n })(state);\\n })\\n); " ] }, - "execution_count": 28, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "import jsbeautifier\n", + "\n", + "def format_code(js_code):\n", + " \"\"\"\n", + " Formats JavaScript code using jsbeautifier for better readability.\n", + " \"\"\"\n", + " if js_code:\n", + " \n", + " clean_code = js_code.replace(\"\\\\n\", \"\\n\").replace(\"\\\\t\", \"\\t\").replace(\"\\\\'\", \"'\")\n", + " \n", + " \n", + " opts = jsbeautifier.default_options()\n", + " opts.indent_size = 2\n", + " return jsbeautifier.beautify(clean_code, opts)\n", + " return None\n", + "\n", "results_df = pd.DataFrame(results, columns=[\"question\", \"code\", \"system_prompt_version\", \"contains_nested_state\"])\n", "\n", "results_df['success'] = ~results_df['contains_nested_state']\n", + "results_df['formatted_code'] = results_df['code'].apply(lambda x: format_code(x) if x else None)\n", "\n", "v1_mean = results_df[results_df['system_prompt_version'] == 'v1']['success'].mean()\n", "v2_mean = results_df[results_df['system_prompt_version'] == 'v2']['success'].mean()\n", @@ -772,7 +855,7 @@ "\n", "downloads_path = os.path.join(os.path.expanduser(\"~\"), \"Downloads\", \"code_check_results.csv\")\n", "results_df.to_csv(downloads_path, index=False)\n", - "\n", + "pd.set_option('display.max_colwidth', None)\n", "results_df" ] },