From 4bb184e08b13308771361913460dcae2742d176b Mon Sep 17 00:00:00 2001 From: calle <93376500+callebtc@users.noreply.github.com> Date: Mon, 25 Jul 2022 09:13:41 +0200 Subject: [PATCH] CI: DB migration with real data (#776) * fix migration conv.py * migrate with mock data --- .github/workflows/migrations.yml | 4 +- .gitignore | 2 +- tests/data/mock_data.zip | Bin 0 -> 24900 bytes tools/conv.py | 103 +++++++++++++++++++++---------- 4 files changed, 74 insertions(+), 35 deletions(-) create mode 100644 tests/data/mock_data.zip diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 45de97277..08557bc1e 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -38,12 +38,14 @@ jobs: ./venv/bin/python -m pip install --upgrade pip ./venv/bin/pip install -r requirements.txt ./venv/bin/pip install pytest pytest-asyncio pytest-cov requests mock + sudo apt install unzip - name: Run migrations run: | rm -rf ./data mkdir -p ./data export LNBITS_DATA_FOLDER="./data" + unzip tests/data/mock_data.zip -d ./data timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres" timeout 5s ./venv/bin/uvicorn lnbits.__main__:app --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi - ./venv/bin/python tools/conv.py --dont-ignore-missing + ./venv/bin/python tools/conv.py diff --git a/.gitignore b/.gitignore index 07ee15b86..5b6199125 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,7 @@ __pycache__ .webassets-cache htmlcov test-reports -tests/data +tests/data/*.sqlite3 *.swo *.swp diff --git a/tests/data/mock_data.zip b/tests/data/mock_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..0480adc27373df73d4d3a352ee4bbe5b40a7fc41 GIT binary patch literal 24900 zcmcG$1ymeM*Y6Dp0TKcP3GN|CaCZpq?(WXu?g_!&9fAjU9W1!J4-(wnbv}r^=bYy` z?|0X_Yu$$GWTt1js=9vvy?5=ZL`M8M6e)^79FP!iAP67~bnSKZbnOhO z?QASe?F|_e72qLY^fDIoo(g;Rf}Vpj90VlPIV1$c<6R&i_EP~N*xfGPz3&dg8s9ld z*JMX$6l74%E@sI8|rza^^};=8Wg4i~wZ` z7bq?c6>n;54Hqv&c-59_4=)U8_bv^TYW6K*r6SrI_gSp(%{ZhgIhz zJ3b5?tyE?l^)1fE0vWkl+)oVdYSCc^2|{gsEGFxg5W1MrNw8=g_4sZzhA3=Gxn0J+TX9wcx3H5+YK zT|FK4x#gI&3K7K;Ip?XToh|Rx2))cbs*moZ&Fx4`Y6ZybH_oEJZ&%ijdp0?_8m?QE z2z(wZYt+c<>}Qf&%hFUUt&y~=YQNn_!vtKiQcdp0_BMrnm7GicL8X8_$6RYXPX`x* zd2ME7d41K(N`4!1xjiNHZ93sC!@E;0bzq7Y8w-0nu$ zN?LB(l9g zVW(`lNH))~cSugqXvx|EgAQFkUvF1maX4u{Mn_7G6T`$k9Fp$7T~IY}pm9vW_O7IsXX?NPSziEel7%~BLIuoJMZu0bK*jO7{hlL;N zNi;PqGCY?8l7%XAa>($ey?zJ>xVKoWmT_4D=_u}fZv$F=>(|Xm4>vXST>%OD9Kjq; z8=LXr_~A>Eh(b#y79pN;DDNz-<|RxT++ds^Hmcip207|CPn<}MsnaL$Xmf4rN!e|` zaUQpVV51Qfm(>xYX3=L33+?jj*C$smccWzOmFvz6aJ@F?25LXd-02SwQt_N# z-8bmB&|z;pT--@9)3vy-2)O4lyIUMJj`;d4EUm3QZ{OMiIUs4hTi;QiKf@x1Jt*Jy zU7J72LAi7|jeIuNEv#M%I{qZ<bl@YEDFA&q-|^HK&I-%H+8+@x=%kAp?P2@kqW`EECo zT&2(NuswNCeV>rI9HKFvVHA zaR;SKjT6=gEu18*d%c$yL?6u-#Y#9_-LK_a+s-l_E`b*BqL6kN3wl3A)U?+2F0gy0 zk%LT;BZf5*AsUBiC7Q^}Fb?@pw7-@6+k|;`@opgLI4`hMG*rQ*5SP-dVro#zcih!W zXqZH*PmQDa!>X>OBS~~5`lSenV)A}xsx7m~$hskNfSMRjy3+Q?q7VTS82;(KSn> zxE3W}U`3>H;TufgK!}AoSzESPuYv|%eD%rfi&@0}Ic5DDm;1vr{O!`M@SPX;msO=^ z;F6_Lw;R%GuL-ZO;G7HXo7;1m?euA|B3Q7gXqfBuD#c$vG1{jIz&7jICpGE4^3FSRb|LrJtMJ7@!egYc<3} z^-%7;heWIHSUtYSY2~W}rau?)?+jUokX!a`$7yvZt51KP07rqp4q=3TX$R3eq4i3K z@;NO-I%^aH3WVw`z;ypS;Q{dm{VaU<{e9*G+#F(cX)pi(jPc`8b3#ewMEj~p=wQ#x zGtoJ|CV2C4#ExL4gHdAKZ-guMRZf{cT{+Ov>mgh}To^(UsX>3a*3Zn= z03l9)NF!deIy@f|+K7*BHb3&~W%^cg$#Kl)TJ>7bDc5X=9dUiIMwqyyqz3)usI$Fk zd-6B=bRgo1)`|o=nl{vm_8&Se`aEM=i6!OZfNo1X4co#epXWgtl)~}g<0~YB_c^E zww&o)dQe)KmBL}GzTVVtWU>3Xv2mWMVgCGjb;WR+m{eBJQS#^0K+}0gFr4=(!+6i3 z*GFq=Zb|-2Zavr~FuwYog#W!e5^JID8VT}Ed zb>-C?b?jC3VfDC$Tk$&UN;_HXeBp|RLvb|giiaV#G^7(8EWcfW4W^M;r)j_9;fnpB zf9z$}*VcaJ90d)N(T*j!Y)r5)+F>)Z5w`5LZr?d1ao!Hn8h1wz1`NHze#OK{YEq#;V2FH*s8)Uv>H6#5CESarkd; zyrPgJ?D7@HK_SfNrUeLn+Ltxx-+=G|h8zfzV*@bZ1&hCs;q z#hRcCs&9_AR>EwC8YY54B2Z!YOU5(mHG|EgJsB!Q{@LR%U&IR_JBc;;s5n6+0T5vl zPIATH6qoQE%O=9uC2GRuLkzX&1~%q>drmQ)LBAjA{5ZD=#U~>$P{yXg8fTqPI4OoO zNzbK=q}%UV5h8=t4@U{AkeIce_SSp6k;4Yx4dt@|qxonderBc|LPEYbDq;Hi%SUC^ z-ugtv-zCHK`t4slCvn16W}g~b?vSCfzt~;IaKtZTJSdtZUFlguQrnMpy=D_^)uo`R%V-H?#6)1a!f-5dbyZt& zl#4XO)mmuC9A2=%8)1)0bP(dOYSe|{)Y~FN%EP_&^B(iIy?GuZ=ConkgG{y{l$3nv z0WQ5)DcxQgYx(BE3;uE;d0*gpqAR`gH^*`XmbyB2Q^>VCy7y9e06d+QES}D}W>4ou zlczJ9@zYt#@agQQ|8%Y)Oy5D!ETwMzg?Yy`}h72L;kxhb&kT zF&1@iY0DLh0DTSS7C`9~AvkqMyI_n|OGG$T$I=~r4M5c=MH`W5rWz=dOW+U**~!-` zLk=11FU6AGXqgkQ%(xGbIa;&V(FNa?`D&g=%EuUj1mOT#c)M8D1T!baYNc|kRMVEs zIDQF;fcd@Jq&jjPL_Tb`RvG~GBo$1(w*`l~haZ)m!%ebM;RbU`mzvsO4H9 z88zM6>fmLmfSTY#hyP)-6E7A`lxm{{2Rcl&=Dy}t7jvTjyo)g zp=yF{8ihN`;$Ky&0eeb#SS1xL_PP1-RRw3JHQ3iaxrzV;?bmewzjT_mf6%0REN z$_y;fT}sD|Jxa~0@67U)Y=cJV=RQ`wp84R2k2cBMnF&)hF zC!54d_MCPN!#6_^wd|+Uz3`Um-*wNK!xX#Il47;9$qsc)nMienHFCK^GGbi#4DNe7 zgh&GC0LleL1_Ui%1YZy(AFHLm4KAVC6d3rJKj!2dy^OpwWtQ2`%#yqS$Wtl$L^O}^ z=HT<7To%RVT(0c)<=ns_rjRJ3d6DjI&pQ*4IodK>#k^W4HJir@j9}@wd7Z$SF1#aE z9-pgXZ5st>AI~DfF&Jr{;=*hJ+O7v}FR=LQ5@XVc6?4&r5}ZN>y8gZY^83=Q{Ly)A z*T6Frm)Eoy&86N6b0_7`?1FnZzNYZN8@(D@GuV1Fz7ml%e^%FsNdEy}p-wc?T;-!> zH_u7$w|ZrJccqJ9A9NmU<>@^R7Rp;R4fwn@pDD{{yyRyR5|VW8Fz*iAPjvf^B;uV* z2w>m&)$7VkFXI`Ju+&*2d20~p$yu{-0ng?S$^zm2BxkKoqtV*2TXy~jTQ80db$VZU8kdJv6iBAGwkn41f}s6I!xQqO2YP~&3YG-V=aLZNMs zxDZgf;=}Nf>GSFMQS%*V8x_OfYG*Av6V)&5i1Qhzfj7_?Im;CH;ChqSW9w{8p%l-& zQE935ieAeD(;Ja!7P}>@@e*at^1toaxYlfYOOuPVPxm! zSA-*|GPVk&mp(GylED};gvxbsR95+M=6>Rm2Fn!&(ID#elO=+jj|0AYEgf=+)A|j? z;EW4ptl^7ljg9j?LKl@kfs+gw9j_-N>1tnTn)>-_F;-c$ZnL zLYSTN{@ymbNHZY=DdRlzJzhJ@UR@^H?^nFHjC4gLJH^MmOYk%&Wl24kmoS6)lv}AP zA8yqoz)=@Xd**Zg6;?awRnq4Db0-`OW^ey8|Htxc(Yj}@BX^x&W1hpL>*wGQ+|rV* zx8_GV3B=FxlEX;RHNI;g&Mq&-nb)Qk*BpZh1-HL{Q=a2Pk@a)-`DYTk8GqCQ&Rg;N zGb??1e{td4CHSwpGGrQWWe>rBtNk%zJZ+H?hk>P@K8Q|7dCK!aLdg7U@~`ai8unxI zPg`4(mtR_5g@#)D|2GfF|KE8)eVw=Z#yWaBN0JgsM~F%RGTzT3FrJgX!G^TKedBY5 zjS=uHjF6C5hD46lj4+QJ))ESqLkF+R8o?nqLiYF2(ZxP771P~ zF9o`x)XG1bJT@8zR)y5wQsc3iPfh)+6PHDQ%V1hVK)ikIgrT#&ww{Hdow*f2*Zv=A z%;#Sh^q$JUr7@Gmt%#N1b{8%Hq;c}{B1oc?MVO$oLy$=??>~xs?`2FxvgF56C&X5} zG_BREP4zS|f}%_jDdR;-deIlJGx!;IPxt#Xv?A4GNEO>YhO~0^XiFIJL=iTgdW-3% z%|-Teo;>m9>AXcZr%E0vMQ$v-2Aj@ctGA&-$L$3lg7jX5nRm*g{V+-8utGgmCD-x- zO9Ah!%Bw0AtH4SRId>L@h4-pJx4?4P&je4uCrq2;+BGG`C8aE$r#my(K2X|Oh~YmF zcNf3w&~su)rqeVxEzHhrD{x?20UQgq7k6UO3+>MItH-ZrrOwCqfeWGghz0mg+|K)- zzegX3@6~ud>ps}n5X^Er_(3pOHRF_ZuX?Xv%Tjnd1HZ6r66ectaLFx!=xWSzW>NU? zmFEEHvuF?_G`UcmI>eE>zvH+mj15fYc-v&cA>og(2jezeLxNu1PFrsi)MNvuO`X-M zO2Fx6Lj$TiEiLRf%9%Utigp*X#K$Qusav7m^GaL_O5MW{saN+Pk}1G_21h z2Wy)>V_&E^+eXBiRpkp1&W5;;H7em; zr3DNEA4?X`C2tiiCW3ASOy%qG`?5o5w|tf_FqzTyb3Nz_>+oJCsS(p3PgXxeiWeH_zX(+|eL&)P^(yl;N~mb3+` zX>2=AOV6=1JFS9p&c0&YU40#@#8Ab;&MP8rsYGqK(xZeI71jYMqL2O-bx3Vd{>+;&xCCJf=rGqucxvrvA(K*<6W3j~BlZ{G07N;y2#wUQC@KdWyFm=&ZpIbz1lY>a~L zBFpk2_N!3!WEr}qs#1J|Pm$p8Y^=SWeG%nW!~$KMkdH`p|A zXQ{GCq4QAAo0QNa%65%SL9&t*duIOYiB~cp5laE$mje1x`OcH*uS2HQrc{lH z3?Xf2+&XdXNL1HjYQO9;1k_@=+;wDt)KSyrxosD@f~M*xnU8VIt3mEp|BT&(`zA5Ozy!#vQm9K-x2?! zfHx>}oZXbAM9pA~Gj+Y-Jvv;iZrpky8o?dnlXGhbc7Ejw{pi^qUD#jFE#~jbb21PR zD8HSXzLmA9h1EY@8~Nkv+*A2C*EXwZo-Bg{Z4DS>rlOkF+!Ew{_3mw7oPea9Yh|{B zy2@Dmgyn85&iRa(NACos{z&iwD)c?_E#oaJ5ceJynDUasRE)7rsjKUhU-ImZD-Z`i zk!#N$({`LHghqX~v~A{F$>8InQ?r8>TBQXkL>{R{%BZGkQ|SOlk}*D?&jU z8wVAyr6QL;%s=olrdNUHb!#V0?WOCLPqwEFwuN}UK6HJ%(-JrFWlo*a0eUPc@gMF{ znpy2roWRw)l^T4sNcJ0dBq@1&pIymWn=+#&YVkJ84Pn>=E5)q_>2$v4C=AK6Xr^T`a}k3%bdCJKCK!Py&WR@?Hoq+!!e{F8)28ka^( zHP09FCKMf3I#5+}rEv;`{ORwZLc$1$rNZUaaIaNN?Gd_-&(iK1ZN`-91_{mvb=QrK zZDi(Gf=XD(PVVYRJxPHwtUpw{Y~gPRosETA&^~PYFPdnkzK&8$(@1Zb(?y5d-Q9CA zM;!s}*OuC$WP}9j-!OLh(OG^q7N%nk4CIv<(A9e_pGWHeD{o9I2{lekD+v*O%EIB< zyA4EX-WF!3@H8EZ$7tq>;(FMg=hJoKqIO*!S!l*y1a10FcTO&^jA}PAK7SmM$D{SR zIv%w(81dJuBW-MMu<>bng2}UAZ+QFL_M+8rnAZ$t2~|#lAyA5yDH~ZOC&`jsp;D0SP^lIxq9Y)mpBtz*i5e25gidiTmMWHM zG)0lAK&AMiqS%*x|8}mH=I-d4CmB2>@H0|w~kD3+OrE5Y3vy}rIf=D0@UoSRx<9vQ?Hcuc<%1IF@*U3%WH z?M)m#Q(nBd!R3f>kVOt6s>wm+%WjK8(w9|FHKEXJj+tqkp+H^oRl^9(RC1>ZQCjjm z#(#RoI=b!l?0K>Mef^3$+O-QXzy-(WJTBjWl1;ihV-|ZD4HE3@YFbi9IHo)Cxtyj4 z_{>REE?;Y>@}61btvc}NhlRDZnQA3EL-R)i11d1Jd)m?O14g4?)W85 zVi^iLYj%5_9fFoKrYm50X)M)y+ltjrqFhQS*d)heGVn;LlG5>&^Yg!SY6j<;YcX%+ zjvwD8Yb}*WMRo-jcB8h@9A9I{DP|WYXJFp%(NtIqm+fvIIqs%dH=h1uRdy;YFY=yuS0{XZQ6WJb z$e9)MChExTL@(r;%X5@QlvwREr%QQDf@OsBP70-``b`qM^!N{l(8Hh7iMUiEs*maw z!59C0?x&G(k3!wHe&?vJ$QWzZr=*#)%-j%{*+K_jkWa%jXEu!`b{oL09R4_{#ui(L zjIWsx?-q-v^HA=XYK@^SjE)mI$`aU8-dbOmZ@)aS^|8donNZAe5wz|Y_!dCxUN~RWV~oU#n7(4m)kacV^cZJh*)n9+H185>joW; zo#XA$RTJJr&y5f(NxY}np{3jByKe5r*skj1e1?+HV~O)NLyH3DHUPMpIWxD}6fs3L z{n9_oc&@wQd&eH~t1#_LD7#Lk%6Vp@=mA@)dlS}pE;VzrowT{;f(wnx;d3M>niC|O zVN5Kf$QGK(nIbv(;reEzFuMnPRoIPz%%z;1;SQ@OS{@JU=Njai00a8M!osMkH^zfa zQDamH2u{VG^|G`0FU=?BdgI+XS2n)t^fZadmh`;U`fvgLwq9j!zzR(z)}<@E*G1Jc zh2|DeL84s(KpiVzyrcne?kK@2XjL`E9ZYyf$$g>Je7Z` zVt>}kYV6~qUMR|VqG&Hd-lQ?QW%I&Q0LTML=;gXmySixmvNq%U^ouJPz#;juTifZT$&%2Op^+SBP2VvI`^MTnY70Dk}R{HeA#?S*jrW- zAs$$ki#CT-uNuYaKHFo^5~<)95o&*7=V#D-*X8ikMFA`Z6;5p*4^J-M zQ{<0^GJ-gV8?cTSmTlY(1lkW$9f8+bmy4?tW(Q?GNNy$-f#9x}!wy)uFb6K`>Pf{l zO%vrO%Sg3x6sUFK`< zxJo0@Lg|vbJvoYKk_OY|+>M-1L0aho@IKegXm!t-L-2d`575$ zM9Wddyj(l%f=K}kZ%Q%k$r9>maRqQjaiG*$emQ3Dr%GzZZ~X+D*H785Uo5^ z=|ziyU~$5m2We*T#oh1se=rL!fo=ZKf7Rjv^F)7ah0Pt?e>vovzh`6~$^7GxEgtE& zv$r+W1^fffmyd@0sr(z9bHCvPLL%p3y9uG;@6jVORCE&1b{J3X=XQ(@mUJ8k#9WAA ziO=42b2;wp9>rd9yJI#(g%fE8_g}PPES%hDN0_@^0(V)~zrV+6QB0L78|}bPCNz&= z;v;wYjzLIF!}G$H{Bm~==RHe*H0hyMsRTCSrD%TD4Pd>{;%E0;iydsDXyT@)x8q?l z*z5(jlBaHUE;MJx>-DUS$Y>OYjHYL%=*4YP5&16AHWC~deOgx7#}VLj&lO^~@4R<4 zK&##PmKEOm%1-oxI2@Sn*-i~Xi^f*Y>Ev9iWm&yU_Vq>v!ZXo!!>I`u1SgtCN725Z zyk`Bt@2nBYnM+F5!-z3yW@T!Q0mfmSv9TnM1}}N?qg}sUH_g6#&p^Cdh#qd+dC~=a zuMv+=w;xaDGYvA6jxloFRFnI35eUTV1y3!mFCMekkj= zpETxH-L5ZzP<$6lv(0E#6j|KUXPC1De!N;TVz6|phLLxX{Wo2&>C;8i@S0Wo!9*Ko z$we*0U=={=v?VRV&&P7`iG~!e-&d~pswV&1l%Q@?kT2iD@W3HqopwhB$eg80T3j^J za)swG6Uh%sb#33l8L!=Qb|a!)<0Y`iGOJlxZ>=FkN!zpINr`=9LxAyuN>7ls%kuk2~SkwfjJbIdV009U!%rcnIum%Cp}cM zsB;@OX`=1BC4SMzUWThC%f3UBA$A@3!D`%=%zxeN+zfjX*DhZt_VtQ?+Eq3;^f4%_ zA4VbQWmD?!MKacF{2~ zYO}%T*M^I~6BDE0Ds}u&Pp@8|({{R2)t5c}y*Ovr_e#klX1>&1w31ik;Wm|<`8Pp7 z)-ITvq8NK`AE)7<-%c9hjx@aV5YOvZ82UKt^f$k|{~2jNehIH+=?SO16oRz9C&C^L z)&K6m|HPsemIfvcdjBwA|CjDN`|rS7{}m!wSlXNFn;ZUfYA^8UyPwLxnMlz7FaN#U zRSD8K6Kscz?ni#HQc-qtrzczE5VUGqgyJF?CacGv5xv-ktWAC6b=7v+eAeXEy?Eo~ zOojA5@vCYKZ43UvwBJSdD6`XASZ=qlpwDiVKD=wlIFu@Blt7xC80^cSqu`?;6=bA0 z%Y-~41~mk;1w7jDo+N3l(DkP#>+(AK`)Q{)p~H{LY@OFwM7%LnjazhMGf!+_kdvA3 z9Mh5O&0Ed7@g@?GEol_P5b11H4_; zO%Lv&hUN-cx$x?9%)ohTsY02@zq;qMOJRArOl1)EEFpQnKj?l&fV*$OM4>61&mtlqF-RkOK2tSrKlg=Zfu>f)C!w}=fYx= zoW@mI%Oj*0BfT7a7t*gcYJh5JcC9q0+!}R}ib!J%;Q~*)sh^I7JU`UFSiCRoyyQPz z5;<4SeIzyoYduvYjopHh$}f?RB`6XloR9%Iv63HD-xuaZ6-z0J=8hfPe_l^maO}UV zmS3zX;okEPU;u-~1b*93Z(lU6p%(nnju2UgPuz@*et)c$ukGIJ z*ClVzu*(Df>Hoai$9_`G2TnPE%O?Z;NbO&Wq5eD1k6J>9__YjUVd-FNVPNQJs{haZ z$1!=q?Wo~MW&$3O^hi0FWDoOBzB&Wp_*UQ`*MJl2pve9rNMOQ+oBn zRiO`kJcoTjc}E>3lEo=8EIq^~ZFJPx7?s|xbI7+U*dc<=|`3l`zU1y^(x=) zoWdY_Y()|^-U5QIbZ-vDi&($9t=;w>8QNAKbw-o(A)n==X>8xhG(j&WBBXY+O1Ou4 z`zcR&0D)`PY1iI76Hz4qt~4&^y%zyN6S?$eKsY||pbrkb9FkJr?{Z51TDQUlax=-4 zo{lDyY3nj^GU+xUS$06uq9qUe^TsRxZQ#D z>_iE|vf?mw%Omf9$u$+m9Bs>pDOpiBKM^g9XR})=hw$tTlV38(%BfJ|KB0$-{~xS`~uM< zoqt>Dd|EI3KY{xHe!)}Ybk7z!z{sUx`kBD5nIGE#Awcu7xlc|0OKMDim)gg_5+y%- zVEqs0pQHb$1XuY{P4Q8316HoQraes zv|&L-;dW>v;Ew7P-QI3xw1}SVLg_aUWB@gEZ$NSHUI0#g*>qrhEQc{dX42U7x<(cc zFD7wajh3d+T~m%|Q86nAotSqI3)!oRI$vvx(7=H>f~7SeZcKm4NthEIIBt=@?G_cEmmkoyO(~Hn(vi4|d!m5W z&c%?9&EIdtlqz^BK3*#2NDE_;KDxC>5!=C^m9 zHYHt}2+C+v4%;hA1Me~lP1N^A)LHJs9vr3=bFk9`Km2?E5D-T&E`t5p${B@)v0}b5 zV&*{0%2_;?y9SWy6vug(3Dbg2-l9dd`98E)M^0&wqe}t}J|ox`>?Or~q>1Oc3IQIkR{M7K%33K&f~|0V}sGGX5Q|GFf6j{9=p}THv9F1 z_Xi%`aw9nn^e!S~sg3-V$w|>uS;{tJJlU$*Z&PN~8k}!exn3R#l}_8tE%f7Gx6PX2 z+iMg>$85w0plpouIKIb^ewBL~&X#H|i`2vlueC@qproX<*G|c@6OhcIFwF)tKFe$SUiF*cGYjXuB#VH z)57`o@-jdwTdqf6?A01#B2VVC7x_GVF<(+LAxV)?G6XUtFi1^Ah;mFnt0W-vj=1TS zyu=dKyQvF&Pe#Nonytf{OC+KCMF1<&FygE_MBZ0V>a6!_gDmGJ#yaNBus#-f+q=`( z>`ddipRehWw)p!Tl4R^DuOU3L`MO@EMj)~0Xz_Jzg88n2iVwPf^!J$AbGGx<%ZX|F zL~oo7+*|n43kGXb{qa7JVNNvFOiomFnNIAUo4||l6H%mllqXN4if{`X`4k6z{JXM5 z|4#j59P}qGXRYhvq-$aE59^%tNc&UyH{xf1Tj#v}kA*em&&MYC-)qaLo2rQ?#m1oM z)@4P6jT-53DT=NJh`4Mc$HVUjTp=e%T^h&RZSG+4%9&QRW%^CZF*aAy0d-Lm9REN-8;ALmtyHP@@o?M zKy!GoVPzHDKXz-UtA>}JU?nX%99*fdTRP0>Bg^kr#E`U*Gf8y+$z0wC7yGPiY2Wmo z&|}CC1ozVP9d+}2+(wZVk!f^>+(;+I>f!sN0@vj3)t*#Xme)q6JUI!Ttng6#co-Jx z=i4Y#qhaI+jt|9_RSF4>- zwWqOpP$DtC@Q)iSlxXJX<|xEF;4M7S(w%$HYz(9Xug+a#n*2xa_b3zj#PH+pslHnk z9G9R5SOlgLGbg%qs9+KrolNJ=)`}(wqCW!pE+O>f1h6_HgRDy^Kyx^ z(!bq6_m>$$kBn$&n4m13l#>N+rJ{aZx^$gC>K=(hSz^p$SY!HYPGl~kLp z>A{(_c)Q*4?qDL(O>nxX+@Jc=mHKLxz3vg0e}*n%7(?hsPp0}2o4>rh(%*sl`j@G; z)3vuV_|K@i`jMum@^3_ehAmfwa9V9R2tRI(?ZGRuO~|2|QUKz3Ao<4l`PGcEeI*;9 z<}9nD40Ys+nEX0dBXFE*3tV^KJ?^C%j=e$Vs`ke#;O3pw$4%nV!FkVSQJ8rMqh4X` zhdDfq2ghK2RHrAJf$MmSD{ofVrzxlyD=8_DYB;2g(wEaS>nf#K!ji{%wKBferrqbR zCoB3;bF`Xz8O=kSPU7jgIo+YRat?3y*!k9$^-c-SI5jmn?LDzd-Qb$-nouKk#&mcg z=1rcGo%N)W$v*YBn%KP~2k)zpl*9|SwWq3v3pm32CQ0W?9@ZXIAVJ4BH8x6nM4LWs z?u^xy*&pJ@k-HDby@#3OyrQA4pKy$HLLnhx*25I)T z$nW3Ioy{&!uL-0+-sxN1^>p@`TxZ)DXMFa86i7IpUjiofJUlqG@%DdsW#erQ6@uM`u zg7W8Q)uTM&p^?cVL2>VD3yC-)883GznBvs5lv;2lUi*&jCsO#g0)}T>s{n-jZRAv@ z!4YR+>lpB0+zj4>gl=Wyq|}nlo?g2*bpZlkk*biehz-Hfru(ECcCkZ=EFPr!;PEd^ky&D!{M43 zE~j^?L!21R%&!t9YL5GNSss_icG?buT7C-48(e*9G`-p0BFmZzRz7%`?*5ezTFhO> zzjOJO!`3M{zWX`9Ydu@N1poY+1kc8=RAYkYz^_Fg+WUmxE0{O;@J}Y6$y{0O)#b>{C9=Y{$=u?6#8iP|6%gSAI14p{!O4WlHv1=7!W@U)OhnT&}{OR+yXyjEX0bj z=yRlH7eMhR$K^1H15>xVP1idvh${n6mj{I7P(H|AW8UEorPQBbv#tQ*<9t>7-kxz+ z#Ihcmb5z*b;PGJPhFTlQZWh^;otGPerqILmRh7CY#HZA2^5el{p7CK0r`yU4p-V+l zmpPLj^~d8aTfb*m5MbjaFUuA$fO-Fe?pPTaicQ;2{)ln>P-?q1Ekf$?kAB^)6fS$m z5_YdBTVotRDb{5tnbTqo4i+dUG)RIc2qmNr4_?4b%|wC{UlTqNhTQs z9rTer-V0oc7`wEgqR9QLHZgV1=$ao{U z$ebO`Wuc-e$%X8C>kGuL<@=)}_b=<68ce(TNc{;xOZN5$ewWbE%QH4$?m9~qE%w!W z*(+yG3@&$IsMp!-s3es%gHsL9B#0#iph#KkQ;5L$=rtuB=#1aLep?V!K-6K z^0>gm{$dp8e8^I2_*_|vr?DjeLilxtVqT8j}=s!+8{nad_fo(`M8lQ*H)KHXcf(CmcanOvU@n!8`3JvdrdTHkQLfmoIe!3Dp6&T!^A1&!~M` zDZ~1q^pQeU#u#&&t7|WBwEG9^dM>9pU7LoMJTdN1R)>`b(dMvQ)N+4n?&(*zUBFo$keC46+=A{_$Z~n+@Sm4MfT_*F@NWm?5rREkAg>)0sd=~z}=(Fp31*T zZFcl2Y1(>B@jOq!z^cEkpNsEvZq*C%6|_jE#PW~1Z_WcXGAM~zN%k{`Z}6`hT6Dmp zcuuLfk?6;QgvYnr_a*xahjZd{@#agZH{+}^(bp@{K{-?xUvvZFY z48Pf3@2I>a^(Dm@%pF1T)jDRMebB*q>b3>Y3djMx&51hZ3Q?pTpIo5~$+C|Jw4AGX zNMERzx=ZJVR;0wYTxA-$ChWQ4-&BCc>5lGq#Do}IG(y^z3^l&6FLTD-SU%7T)<`#A zV5>GCa-Xa2OS(J4wl-#ZYY(8Nuva(NHV25879h;FJho4 zdt!m3^JZagi=zm0H&>6$&s=hq&-ooNhYyGRxBHoka$tHJ9(3u8A{5-PQ`w#x4ROU1 z<#nAXDg7&gy2I@Sdu3hmZ~b~%<7c(Tj=QR(;H0?VNLgzNBPH+KXBjdm3NWaT!%oQ6 z=SIZIqbu)L`oS)$uj2GM(46huZ?;VFv^=P*VNI~&p#{0&XD#0Qb}9Szc*W`)rMD8+ zKZ7GF%W|&L{|b)&PDR<{SBW2~_>a8K{~3_~_nDodq^GUyaE2h!-&@&-M`zKWW|mKF z{)_R5|6m;B592=P$J+Myl2J^VWNTbGUn0CgT9LwbmT=tL6b3aAE|=H*WM|P(%2T}sHDXFr z{7YcHU)hN}M5p6cK2~JegVm^M&`M8D9V4J0GZhMvDXy4Strrg!0~daL3)!QM(Zp9% zVJkn5#;E!Om+s_#xi|KrB^(>9oVXtt|%Bx(am$%ijL1xFJ`L(6$XJa1h zEclCaas~GWJQ5Baiqa}K1}wC3SAoH@H0mHr8pqlk$}cP|v9~H-gUh&xEt_3Rm|+iE zTjdFd32hRj^l}Ky}>dz!A8|iWucj^%*Yjx&w%M1W2P0hSvBfy8ZoWL%C zxRuTLj{jZxH=`yf%Ov%K!36enO^vaK570d4D(r-{&@n7a0d7Ba@)#}KU1w58--m{V z61f0Hq9#FFJ`NXBB{Zdb8>0_-6368Bv0wsvvIm(|4kwnf( z9%e_ow@?BA?`wbMkgaiPO52*WUu~}(o8y8)iKGkVIhCP^n(_g^%Q72 zdcAK}XSVPE8rQ6kRn?MAJvauLCQh|lEg0ShBiN*oK@-xqBlDwMwMk}Mrdoa+N6Px` zQP#8C%P?4%!Eaf(8@C!li(h&p@M0)e6J_y`Y(P{bTu32-Oo^^!9RB{f8Sz2_YIhP< z8tQ2*E(+<{(=y$NL0y44=;1enHxo1C(vrc{z7>l5esT1O?E@bMp)#XYwGOoPUAVYo z8ON}WpT)PpFI|~!7**DwHUWTOjs5e7oi#zpWT{-axplEp>XJ(>FR9mx9fz^)SD!F= zCU6fXzI?kRY8M_Vs6lnWbE)ZTM1iceYGBVq=j1eZ8Co&7p)9ov38%J?iY+(M{>cqF5I`Yun!-ILx+*@FzjHJ4=^u(X}x0m0@Mr|iAO2I@P?0ezb}g&53pez801YEpbaaAWC~- zn$v~~F(+}?QEP6mnWXm%*I?yLwBMj!<>lysEp!ao z;C+rKg6Z2@@3o#<^$&sz{)6B@tHAcA)@Hi4|A^~l9?5+w|3>fJuqE;ny_g>hh)1+w zVGxESMUuy!2~Gr?GZd3iIsij$i7Lx_YRbeD0cShu{zrzK&S#IybH)%1nD1|e^d74MZZsEt{N+w_Kg8%yvfOLi)9+0 z2ag*DHfSsy4JNVtCiSFU%*_dZ#A3Xjx5hGHR^-W_Mt|o{XRln0s7c=0A4@3rC2DdY zoyvc4OCdxV3Wt$Oi-Yle+iv^^XXO=SZC(9v__-4gl78Z8!}THP#7XaXz=!dD+LD8I z|LLcO?3zwN42cauYQaSb-ua93T&ZJO;|)W+#@N|zp366>>!$XE`{KcDsby45 z{Vg_IY9z@yhZRdfx#4VX0&)LeCszUu^%}){q}#63`O>Rd&Pv2b?os_wwI;s=FLAQ-}E}2Gp_SL$2iXY`MdYK-}l}7 zyH_pAY}pWo=0v_B%x#qHuXJv?m{n4CX<_D}hezI~l1A=WIEzvH$f+eh*{IaZgC|q8 zm0RJvnpa(<*UX9}o7crxpx~M0G?bn9^Y_@3b5|Jyie_3ciPWiL42)QTt zpCtwJUOCglIdWyL-V}<0F3p`B4#__wBQni)PcOweE{<_PM--}*#{Eg!G){|?t?dSo zmuWjVonropNJ^jktutqOx z+R$FmW^WogIH%u1XReay{Zh8W{D{d?!RIfo;v~YCTE}{2wXpoF=_jTls&^xs$MX`M zPByynvm0KyVdNiyNeqcbzK$A2)R3~4MlckAf)hmwJHRbuCN%szhcBP$iWxSPnn_|^ zvtJb(8>dX7L-oWXmFK3EN=Usi9d2Ds+H%kNtk_KZb+vjG)tegc*x!$ST!)h^4AbLO zo{Bdm){hkCK6+feNKq_YRriOl)3FGX`FXSQuZL3K+3w!!kj1HHTif{PW~xyG3a#+r zaoyU@$=uo8#+p8M{A{!undsbjmXo>nT9~xvk$W**)|TjP=);rNqF**X_=8`I{Tz$@ zD^|`RnB@_BQ7Lsj!|Xy<4ISy!n>#VG>VGNTkxkW1pfqA1!On!jl)E1dea6>aNhDZF z?yJTQgcekazp*6p-_y;=KA(ohSi*n98Ty^RwUDY$Q5Vh<(UuuDvp2X=?EGmpSQ_?c zTuzIuO`Q1Hby4ZuZP9>pFtu=#0lS>8;Vgpbh)R;vvCV`9xfU1Zn&GSv?-uhJKI(0R zFf`V`S%gJ9h`IlKU0}G8Pvgf_!AAoZ(hG`xFvB0m^B*nl7MkzRkA@#w%rDL{BTFML z-DUe*)%Su%0C_T7tp4hg#ln(kuO6fQjwZZ&^ctmth>eA|Q$uP~aqm>iCZGCJUGBtI zVv4ex_*)MB*F8ijSlmVKnUdIfQ@f5^R?AMqbDZ<)-RR8LftJOX6zke#UqK_9d-!@- zIR*s1Xt=O3#M@X_zSJplw#i1qrWg}ECcEEKAyJrhSTba*MsCz?OeQJz@Jr20j>S3VR*eIIdIg$zO)6kH2>l^`ifcj`KyQvb&|9 zGhUO}(UP9`N|$L{SFLmpMQncS#MetR*C=e^_y2{KLNK&kS?i6xVeW2e>*DN4pPTZa z#un)LQGDra(D+Kpz;8LlQ7K~~^>V^aO`7ZQeV@GwL%daXM}&(XV@wm;-zgyEd&C#c zeJO-nRWFAwZS(2aDs9{4Uma%cJco1(Hw_V49B(b=y@Y`6bPvynY(8NfUiP77}mJoV(Vu?4e- zfj*nPc4PKYfwszr{AQ26^o1Q+PxH8EXbiQ<>B9m%4U8VZF>OURkOY?UDbkj9ed_VL z;Jw7C*V{}KCsrjbRlK!Gi!I~1XG*`OO=raRDv9x8?gwez3x2n+mJ$_Z*F5;VE6P2R-krCjRQMOGva-LHZr^MkL%$vYd4w`H$wc*p^7do)=PoHZeWu+HxTNjen=}JC@&WOK_>}#vsuRd;&qbq&;Z^HUX_(o#_`X!e?pR;Qf-l z5oAJ5@2yZQ>=r^-mf(wlT))*&Oe@s)UX@iPmW}vbM}jxvJBtWJ+|% zyPWOfqnc*e!S0~Hfb%X@QBKp9NDpovzgAuJDx}6cuj5pXthfCLnxm>^VAH|dp51uo zr0J_k-KsKKVV-RU3(?OLCJ`7-XYNentX6WXy4cL|W5i(}o6q<=;SH(*gXqy@ebPWYa)iyzmF%jZ67S!;tfpqE-0ydvmf`+HAI24m(Z5!GmH&C zx;DU6oaAQGaWk)Ye<9wj_T_-!(spQHJ!k7B9|^ez)y3Dsuq^2Ou4{#p8+PutR&M4u z=+1Aqsfi-!`H}CpyJ$2*bR@YeJV!+ZCd%R;R8MSC*_-L<8#P;c@Yl_LFJ7{ow$D!^ zMta1gj-5K#@Hk>t`j*(4=@YX}LWRE^Xji|oJLwOL`}0e<*3Qq!F_UzYCkW>xexBeu z<07PihoS#l4%KRozqcIJf>$j{^$QIry|G(n`#n4`Ou3+b7fXm@pU@#>m{;^))dT-5 zRO;1zHT>*8m+`tv?ZCik!g%sFof)>Iu2Fof#a?|O-TS8rstJ*m_6gQT7Egu0h}RXH zOBr2i$G;r2znQ7qWyACC4ENSgx{XaSTAI1#61+Nbu7wVUHAaE*Vn#Rdf&>-4Y<>b} z*ez&+yRZ3uR-XwXM=Wqcw?d2enGf=%YnHi4nl*23PY$}6)N)wW4)3`zBHo-+gU;vN zKKzM8>FJC@4)Wb5ieO~8^mtG5)g4OQ@bun@B$@UiQ{`(5gUR+PkCb$x0{`5juXCV% zb0VBaTetONP>t$`_aX|(dpHCP;OHG(&#LREE1W3%dzA`33i#X(_m#xk!VlU=FeG?5Z2h zR`iu^|3+t{oPQyBWjI@I^~AP8M$C|+@nr9W4DGi)UeDNRciuXCBqoZZ3QOJNuU?{xr095P?l39lq)pui12+tb{*ob$!nL%(I7gWARy^i*LKT z+vxEa-n4kYKnec!yiBhsOc7LHH1p|G{K?bcvc`7Y{3NqAodN|0|#YB zXo5pkQ1fQcJ22kA2x36pbYOE5ba$l*4i!lK0AVoJTLHD6@kHaClCPYQvtgP=*^%jxbg~&cddXJP+tN_F{gLeH-}d-80}I3 zA-=Z+y`{NDXb_Ar*FpUdQyt{|LR>ML;7~We>1T|$4kAG8Oksex30;_IfT0M-A&q9y$+D8kkd7+?c{08o_+tbakZPnzIRDQ8y!)>8@n9tJE- zLB%wh;7}zB|63vJDy@NtP{9i<9zX>zn&41D=T={A9px_&1u8^<310d*k5F1gVYn;- zVnC^4Fug~=+V`ofU@(?A2K7TpLNMDze|o#CwyK|jlp&}c%2ra+ON!zSMH;JDY0(Y@Iepy F^dF>r`m_K5 literal 0 HcmV?d00001 diff --git a/tools/conv.py b/tools/conv.py index 24414da15..5821f7af8 100644 --- a/tools/conv.py +++ b/tools/conv.py @@ -38,8 +38,6 @@ else: pgport = LNBITS_DATABASE_URL.split("@")[1].split(":")[1].split("/")[0] pgschema = "" -print(pgdb, pguser, pgpswd, pghost, pgport, pgschema) - def get_sqlite_cursor(sqdb) -> sqlite3: consq = sqlite3.connect(sqdb) @@ -99,8 +97,12 @@ def insert_to_pg(query, data): for d in data: try: cursor.execute(query, d) - except: - raise ValueError(f"Failed to insert {d}") + except Exception as e: + if args.ignore_errors: + print(e) + print(f"Failed to insert {d}") + else: + raise ValueError(f"Failed to insert {d}") connection.commit() cursor.close() @@ -256,9 +258,10 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True): k1, open_time, used, - usescsv + usescsv, + webhook_url ) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s); """ insert_to_pg(q, res.fetchall()) # WITHDRAW HASH CHECK @@ -316,8 +319,8 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True): # TPOSS res = sq.execute("SELECT * FROM tposs;") q = f""" - INSERT INTO tpos.tposs (id, wallet, name, currency) - VALUES (%s, %s, %s, %s); + INSERT INTO tpos.tposs (id, wallet, name, currency, tip_wallet, tip_options) + VALUES (%s, %s, %s, %s, %s, %s); """ insert_to_pg(q, res.fetchall()) elif schema == "tipjar": @@ -512,12 +515,13 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True): wallet, url, memo, + description, amount, time, remembers, - extra + extras ) - VALUES (%s, %s, %s, %s, %s, to_timestamp(%s), %s, %s); + VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s), %s, %s); """ insert_to_pg(q, res.fetchall()) elif schema == "offlineshop": @@ -543,15 +547,15 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True): # lnurldevice res = sq.execute("SELECT * FROM lnurldevices;") q = f""" - INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit) - VALUES (%s, %s, %s, %s, %s, %s, %s); + INSERT INTO lnurldevice.lnurldevices (id, key, title, wallet, currency, device, profit, timestamp) + VALUES (%s, %s, %s, %s, %s, %s, %s, to_timestamp(%s)); """ insert_to_pg(q, res.fetchall()) # lnurldevice PAYMENT res = sq.execute("SELECT * FROM lnurldevicepayment;") q = f""" - INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats) - VALUES (%s, %s, %s, %s, %s, %s); + INSERT INTO lnurldevice.lnurldevicepayment (id, deviceid, payhash, payload, pin, sats, timestamp) + VALUES (%s, %s, %s, %s, %s, %s, to_timestamp(%s)); """ insert_to_pg(q, res.fetchall()) elif schema == "lnurlp": @@ -710,36 +714,69 @@ def migrate_ext(sqlite_db_file, schema, ignore_missing=True): sq.close() -parser = argparse.ArgumentParser(description="Migrate data from SQLite to PostgreSQL") +parser = argparse.ArgumentParser( + description="LNbits migration tool for migrating data from SQLite to PostgreSQL" +) parser.add_argument( - dest="sqlite_file", + dest="sqlite_path", const=True, nargs="?", - help="SQLite DB to migrate from", - default="data/database.sqlite3", + help=f"SQLite DB folder *or* single extension db file to migrate. Default: {sqfolder}", + default=sqfolder, type=str, ) parser.add_argument( - "-i", - "--dont-ignore-missing", - help="Error if migration is missing for an extension.", + "-e", + "--extensions-only", + help="Migrate only extensions", required=False, default=False, - const=True, - nargs="?", - type=bool, + action="store_true", ) + +parser.add_argument( + "-s", + "--skip-missing", + help="Error if migration is missing for an extension", + required=False, + default=False, + action="store_true", +) + +parser.add_argument( + "-i", + "--ignore-errors", + help="Don't error if migration fails", + required=False, + default=False, + action="store_true", +) + args = parser.parse_args() -print(args) +print("Selected path: ", args.sqlite_path) -check_db_versions(args.sqlite_file) -migrate_core(args.sqlite_file) +if os.path.isdir(args.sqlite_path): + file = os.path.join(args.sqlite_path, "database.sqlite3") + check_db_versions(file) + if not args.extensions_only: + print(f"Migrating: {file}") + migrate_core(file) + +if os.path.isdir(args.sqlite_path): + files = [ + os.path.join(args.sqlite_path, file) for file in os.listdir(args.sqlite_path) + ] +else: + files = [args.sqlite_path] -files = os.listdir(sqfolder) for file in files: - path = f"data/{file}" - if file.startswith("ext_"): - schema = file.replace("ext_", "").split(".")[0] - print(f"Migrating: {schema}") - migrate_ext(path, schema, ignore_missing=not args.dont_ignore_missing) + filename = os.path.basename(file) + if filename.startswith("ext_"): + schema = filename.replace("ext_", "").split(".")[0] + print(f"Migrating: {file}") + migrate_ext( + file, + schema, + ignore_missing=args.skip_missing, + )