From 4c88d5da663d7dfb924b71853bf8677842d92172 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Wed, 8 May 2024 17:35:16 -0700 Subject: [PATCH] Nick: v8 python --- .../build/lib/firecrawl/firecrawl.py | 53 ++++++++++++++---- .../python-sdk/dist/firecrawl-py-0.0.6.tar.gz | Bin 3476 -> 0 bytes .../python-sdk/dist/firecrawl-py-0.0.8.tar.gz | Bin 0 -> 4068 bytes .../dist/firecrawl_py-0.0.6-py3-none-any.whl | Bin 2573 -> 0 bytes .../dist/firecrawl_py-0.0.8-py3-none-any.whl | Bin 0 -> 3119 bytes apps/python-sdk/firecrawl/firecrawl.py | 8 +-- .../python-sdk/firecrawl_py.egg-info/PKG-INFO | 2 +- apps/python-sdk/setup.py | 4 +- 8 files changed, 47 insertions(+), 20 deletions(-) delete mode 100644 apps/python-sdk/dist/firecrawl-py-0.0.6.tar.gz create mode 100644 apps/python-sdk/dist/firecrawl-py-0.0.8.tar.gz delete mode 100644 apps/python-sdk/dist/firecrawl_py-0.0.6-py3-none-any.whl create mode 100644 apps/python-sdk/dist/firecrawl_py-0.0.8-py3-none-any.whl diff --git a/apps/python-sdk/build/lib/firecrawl/firecrawl.py b/apps/python-sdk/build/lib/firecrawl/firecrawl.py index ef3eb53..701810c 100644 --- a/apps/python-sdk/build/lib/firecrawl/firecrawl.py +++ b/apps/python-sdk/build/lib/firecrawl/firecrawl.py @@ -1,5 +1,7 @@ import os +from typing import Any, Dict, Optional import requests +import time class FirecrawlApp: def __init__(self, api_key=None): @@ -7,26 +9,45 @@ class FirecrawlApp: if self.api_key is None: raise ValueError('No API key provided') - def scrape_url(self, url, params=None): + + + def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: headers = { 'Content-Type': 'application/json', 'Authorization': f'Bearer {self.api_key}' } - json_data = {'url': url} + # Prepare the base scrape parameters with the URL + scrape_params = {'url': url} + + # If there are additional params, process them if params: - json_data.update(params) + # Initialize extractorOptions if present + extractor_options = params.get('extractorOptions', {}) + # Check and convert the extractionSchema if it's a Pydantic model + if 'extractionSchema' in extractor_options: + if hasattr(extractor_options['extractionSchema'], 'schema'): + extractor_options['extractionSchema'] = extractor_options['extractionSchema'].schema() + # Ensure 'mode' is set, defaulting to 'llm-extraction' if not explicitly provided + extractor_options['mode'] = extractor_options.get('mode', 'llm-extraction') + # Update the scrape_params with the processed extractorOptions + scrape_params['extractorOptions'] = extractor_options + + # Include any other params directly at the top level of scrape_params + for key, value in params.items(): + if key != 'extractorOptions': + scrape_params[key] = value + # Make the POST request with the prepared headers and JSON data response = requests.post( 'https://api.firecrawl.dev/v0/scrape', headers=headers, - json=json_data + json=scrape_params ) if response.status_code == 200: response = response.json() - if response['success'] == True: + if response['success']: return response['data'] else: raise Exception(f'Failed to scrape URL. Error: {response["error"]}') - elif response.status_code in [402, 409, 500]: error_message = response.json().get('error', 'Unknown error occurred') raise Exception(f'Failed to scrape URL. Status code: {response.status_code}. Error: {error_message}') @@ -88,11 +109,23 @@ class FirecrawlApp: 'Authorization': f'Bearer {self.api_key}' } - def _post_request(self, url, data, headers): - return requests.post(url, headers=headers, json=data) + def _post_request(self, url, data, headers, retries=3, backoff_factor=0.5): + for attempt in range(retries): + response = requests.post(url, headers=headers, json=data) + if response.status_code == 502: + time.sleep(backoff_factor * (2 ** attempt)) + else: + return response + return response - def _get_request(self, url, headers): - return requests.get(url, headers=headers) + def _get_request(self, url, headers, retries=3, backoff_factor=0.5): + for attempt in range(retries): + response = requests.get(url, headers=headers) + if response.status_code == 502: + time.sleep(backoff_factor * (2 ** attempt)) + else: + return response + return response def _monitor_job_status(self, job_id, headers, timeout): import time diff --git a/apps/python-sdk/dist/firecrawl-py-0.0.6.tar.gz b/apps/python-sdk/dist/firecrawl-py-0.0.6.tar.gz deleted file mode 100644 index c1b4206e6db72385a8c4bb6b0d84642b749911bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3476 zcmV;F4QuiriwFq8oGNAl|7K}&Wn*$-cWf-HXy4-XDda*@##mVa ze`TrQadb+0PS<)#SL~G3b;By#rIJ{0@)ZbbeM(-fvt=BSH$VPN=5b1%7GubxS7+9< zc*Twrx`4@+S(b>?PG`ZhWj=HKc-2|4D4?^DQQj$+vL59yUwTSj=thTfU3bh@l!vDT zcIy2Pt6H~obT{~afby`0*cFsp*4ri9*+RG=l!Sbi0{W9|Ni#yjFuoFG9p@yA z38R9CYa(D^!U&B5qNccFGr=<^+OjUrvn0=8JR=x=5FWv-5I?M7z7(KnErW?%FmeU) z0tsof$RUg%t2khxAg$-mUx*`zoMj|Q<4Ybeq)PBr5~?+Ml;u3*mkd9#bWVLHXO!Ql zpYdp+X3iRXLd@b=*29P7EQ0+GLz+Rz1Hy=5psd*FU&*B{`S)ogiB5QOYFX3ibVkL} zf^X&cMoa}ADlWik6%Cf?YReIyq}-^XG%ay2^E6yOpb>y zDf9)g;qKz>(~|#3NiroqSd9RwSKFV@FSavyX7@H(F6a zd!hx<{Olc(qk7Ll~D1*kgY0hN(wRNpmN}0&ROw(5`-YP$} z)wYU>`g2sMqLv4m2bH2g-51*;YL`1Yt+<~O(352xNW0AFdoW7u0o^w>2~9L*S)N9! zix#T#1+s6zedHaSj8oT_4GC&R!>fr~?M%s~+#ki|Z+eT6Q@^=wH7-lZ-M}QGi1aFw zTl8>p?`T+YEhJ~@B1cOU4nY??Rq~&f^&8$)B+Tb?u;d`%dOkI=knRjuRy6+}&)P)f z{t_3#>J|OKSNV!W`Dz9d2UA3i5;&4Uu;QuQg!R%K{5Drs>DLE0bjPo!cIDVUENluJ=m@zVA(9X1Cpe(Vg(qxeK zoCtJpF`1&iPuul1t~j{Xgrz(Ny#c<~#9uQ1eX%oHE>Hl?l_f9eqfNG88u94(;3MTEZymk*att%0f3?Q+@6K;qVrGtqFuG~ zsA|7xlZON@_yL#MD#;YeBP>eYtj2_V#nH1tNa(7muG`I-U*f~z-DWt1b_p?0+HYAVNgv(7a=q*C)wA%UN#DZ0TMw7sFn@ZIA9 z>}Gv)&uyqa3+xL^PoXz^i_Df9UV;L7lBOWRXFw4mgK~Qq@h-6jm|)m0G_~LvX6xIQ#PgX}EJ;BCzCO^PbpGP(Z82FDO5;E^&WY1SCsggC1D%f? zbN}zFpa1o@@Bekp{lA0U{~PZ*{~M3TCjH~S-v2ABI(KOQpO^nJ8g<7e|HGXB82^9J z^PgUKa55a2^B?2?mH+oV9>Gd$M1DXHfj{Du-$OT`}b4~uoq4pndz&rcz zz#A(u*MA28|Ni)o*MI%-_XhtB{ww^iPME(a0N9WJ-JUxtum6T4ycg8#c84ba;~+)j zeJm{fN#BkYmL(7N8CdK*s%Q#|3r{$pxHafaZ-iKG8@V>Wj z7A#}YWy{`5L9ma?3HZD@2NwjFs8+xt?~0ROX_zy43%_N*jLQ@VogNbCNlON5C&P+` z`svRG8%v3e)p1yP=A_S>j^C~mW}gz9CP~PBnKaV*UBr>yZuVDsto#qz0|w0Pe>0l0 zlzgmnd2N@YuFFq2U2(Gg(FS5+IQ)NI7nP#EDV%bglPB)0Jn3Az9VN^SZne}V`fq!StVu~*+0@wif-*;)*LZ#$0F@KcbKMOE zsI#EQ;M}dFMz`9O=eEdwp9x`K;EZ@a*PK#KE05%=F-WR?(dfrQv1_E%$S1}5O>1sH zr95PTJkeLl6z^U?chrf~Dfw6~@tr)!`R?M{Zjx?Q>)j+69+C5b+iR17`+b`XUH76% zmz>$tCq&zlLUU}{vXk8=_PgkP6kjz?snXJjK9@~(u1Ju~8nbokuPa$J=&Z|OThw>B zer=t~MSyMkOYI)OGSc;5$3aMPVFLO0|HMg1aT+IaEA5rC`g=PF&Ehw44!2($lLQNg zP$i7)4jDd`F4EiSGNfgDNy=a*=IQ#HRvrTRH4R|Vp0@ASeq)wjKo81A+66~{&N|Cf zj-6VkGCH;lbwk=W8mHoJ6-Ss);Nen@43Luaa;J`NW4>zckD9ns&BFB_l~i=7bP`Xh za;sn$EJ|h&R+%mh#oJC^Q5vXISpJM!ZS6ity4?KvEq1@7DCvCEjN4zR0vn!m(>Z^x zI8_Fd5~o_aF3~N);AAy97Q1_JIImGyx5k>oXm8Ys-Cl zAvDol>(nRTVSIHB-eH2(diuf?8_K$?CHV3kkgS#LAj(}_pf;EMq@LHv)Lwmah8A@;A!8o8c&FOWuUW_Tw^twaUGn~l8t4uq4TNX0R*)0u>b8LLX)?QRcJLK56N%!Oz0}nxR_06LHk8+CS6b-S+^JWcHmRVdagnsSR3W5 zGewIy2tuH`M88vCwqwoROYiW;8+!d|r`#MZ{X;dF$_E1Ke&MeX=zdg$Umr5meHXD0 zpjc>BO}e$?%+j??8vjh{fC_WR+?@a2`T3uD|I4>K|J(ikm%+(N&)E7sKmYTRwZj&R zV;;@p&KKJMXXStP&;gkIPm}*){QsfzKhXc1{14;*xBUMXycO*8-v8AF|KICk{Acn% zzJ4+9|1#Hq2LBEI8~iu;zZd`C{POPg<0o&N>_c|v_&@4x;{Vtk8vH**DPQ(x36|xF z14zH~`5$X{Rfg7@pR!$X|OQkKT(tUGZ2C8uoR`;=~KtB@6$I_8R;* z_;2vvlzWcyEGg-R>s- z_Xl0q;Qt{?`M9Ss1%v+v{|){d{5SZ2$M;Y_jsKhX|NFy%dH>U4_utsO!GDwgegE+P zhTnfa>6-lSdy4<+AlRQT?$ZAEzyEQx`TpO*$mIVWq?|*5=V5?%VP$mT&2k>%WBBq5 zzBK?oyO!|;2LH|V|NXfB@0sg=ga0xS>`v`}AN~*f)%Aa`D}Vp9KQj0K53~RH{%5)4 zSsaI=c-e-mY0KaJ@e=C4mp{*?QUujIzwq}XCw5(X^vh2!)pG(9TPl;T%+qjU-|`6n zJ%q}f|A}3D4xrKE$)^D(_U6+7x>Nb%NJz@QXB2e*g2_pP<I{-n!`Xju<FtwM!dGnE-R};2-NC5ecSil;pg*)8nex{!a~?A{rsw`%wBGAD9p`ELo7n&UaH#%{ z)Zb2TP_cQ?yzYLdI~WXmVE=>BVE+*r?$Z7P?uI@UE?v=>*0h6W^Z8$I{hdFA{~v7o z|DN&xJMsTdYyE$}`?PNW;9>gz4?q5}_xjz-_xEc5qru=x|KINnHvIo^e*p1cXP5T> zK>hz)mQar-bnhn?3mygsq~~<4cXY)LNLe?myk07ab(F3^Q0oKoVVx|)fE>N}k<7!G zyv+KLXCGc$ufi4Ei|7IdTP8^)4%+PnPnPM-al=)6#R8Acd`5XYpUQfc!g%okd8;cO z%JsE9wxZlWAh1%W|5}x*jTQgN@xM3P#{a&-|DE7}ufYF7cQhJxhfc>>)I-MqkB84* zyghVQ-j48pv_I@h{O<$s4}1Gy|L}R=;Qubl;}T*QP;ObDmaMjha6u^X`78$XC&`i~ zg!+DXF337eNfHu91^3rPK*xv?8hAtvan5FfCrq^DwJ=ShG=ct%VDv$F0HZ?uu!8Yo zfTFbwCQ`x3Im8RZr@&5=Hx+=@`Rr; z{KVopb(tJdexq)}gM}J7sqhIgi(^@jACuPstheve1VSDVMhG3{g^l)wTw0QUmj;sP zh(`yOHJwgpR4gs{miMp3RM0@q_E`XLmeqC04-DiPARXkCfe_+wMUL}6j>pbU8d)S& zx!kmR4mn~ z0#b^o5QHbPKYWRyEr<)j+tOl$l_4%)?Du1X}s|d1>Gp;QPSI3n`4I1$YRE=$H zIMLtiCXRii6&186SOCqRKfbZj*oP79OwgrR)3%gidlRX)V*-;o>e2*yHQ=8d^|@q+ zW?FAaN+#EhE5$6J`obbmY>b-UN2q_oKF}9sMHu+&x0DCZL(t_Qfyz&D%4Geeb*X2H znMnOa)0a=)$Un8zvWl_#)67&+%LC1WN>QNhi)|6L@)eyH+)oMU$ujh$T_*Gtj1pTw z^G!`c6OCDt#(`?0g{pjkZ0m6sc?T!sl=Wppf?Cn=ZsJxeQ*g;>>@KBCpyYoki`SO* zSpWr~t3uTwU9fyDvd~tw3I#eRpPkVFKcIxcoyj@C?Lik@vl4Hr7mZK7@I0unA!kHJ z&e4H_OU+c_pTLHkzPf_yT0?d#aUr!#t<}$3(Qv)os`8cADqsIAmI|>@`bN2C=%=Ly zQomR`A=f8fr07qCLl7vYN{goX^$koA5p($*{E_EzJ)Rm^$Q8hu70thdAa9X^D5=oW z`@&c0iUjFu26_Vy1!XL7P}!rUWg6hoJqy*P6IX=ALFNlcWL#P@P%wg1#D*C?mlmnm{FYj*cEq1h)h&d z*?6KYjrFIgDeF;47dAkwLjVjN6xZ1(;x1FEq8_8D{J47+CRxG8Q;( z97Zvkd$ABgt{X0moGo;LVL`k!R$vq}Kq`@3yLifT0MIi4P#(!xU5&P40^?y7h@&CI ze3mAXnc^E_hmJ>UZ&PTD=|K0lh z507D;36TMYgi{IA^qbvp+C z4gM?qFHV@hDgd}1|GUFMuay7UgZsZ;w>#<>{NF{Jx4<3unO+7MwNgt*iyRnGe9{)iJmrI7Kl77P$pYexiQLei!Ch6FQ^B(}Sx2 zayCjw%8C*AbU;KBXQwU)xvY*+bjPv^BzbZI{77!e-aqg{LDEu!Z*fwU9|!4UpuJD) zh}j3krcvZ`S7xQOe-U9|x2o+`qAdSawtxF75YfqaT2@C37B$cME@gM_;(hp93eG_p5)+L#lyMQN3L-FHa^O_I2= z)%tjQ)RPvmh5Fds9;{Y6b)$g9Zq?3Fb2XYI<;5<(IZ z+xJ&{#jrNA8-xi=j)uS!zc5$p7L3TLYKd1XFY8-dv({1nIr8vGtjJ!;3?R1=t;fvU zQp~O4DkB_kD_||jl~J~=LLOJk)2>kFGHB{N^vx(XsuVT}BjU3&=Hnq{g^bFGcq|5< zqeafpXQFCqgW(BViAJ+lFVUHPHzwQc)GJ<92+$e1mh+XI$6I=;g#O|E(Wm^>tt4~l z!@S(3qJjSB=>0pHT9l6iP9Xw97)+1>nG=N~X;j^5cCZG1)2U?Jwa+?jMec?N*3*vl z-)bd#>~Xz}EBKXB=cpszi3?gu#$(d!bT(kD%!clRlWdq`t-hN zi!!rMs=m_~tfr^L)5Gkjw=uV0Qts+ueJ|GwQ6yM7u-|tT@OE$**t7HIb zcRc9yT4d1qTZ;@koed`BU=w|Qye$;%-LVlG`RCv?2+u2L*l~{fcjZiF9S|h5#9x{E z%Yyw1omJLFZKQH9^Oxo+q8XJ~rXm@cUX~}1Yy@V1gct>f%hLNUY)LK*Apc(86e;9= zom6`6Mwa<_uRcMF^UgjYr|tLj2RI_8`0UItNLk&|CsZKr&*LC8i@%F=xc%Ii2%R~E zBAqP1yQz4Do07?TwJt+i_C$m8L|-pyW!122ZXfQ!s5K2>(GzXojq8nBegQqmCuwCI z{Wj~&6TND+PUU-T8R~|#uQX1@-6{+)+h~GQRr14f(({#S-o|{@*cY0(1I@zaH7cp- zP|0Q!smhIvUA*(a{Nf@HuA+F`sclLFb%NV(QLC-h2TA9PKflH5w-qJr3(dIwl`62| zNjIJIUB#*L84%98SgBT1aWknHcw5WJcZ7a6X^R z1`Vn+MXkP z#-c`*5%PpIdgRHIjQr*u%rm3Fy6So%@3Q`v#Jy`lqr84j zHM@4c1T3!!$+RqX;@q+w)#{Wt`neLB8r3&Ps0mA}b78+@TYEJboZ5NT_!S!nxQ!-5 zQ`E`QF!im=;8~usCb+#`Q?81}?C1H@4aJigvos+=s2+4Wr$rui%^s5sbL*vnajuP< zQ)hC!Dx#@?A@CCmJRU6YBPc=&7f-H~mDWP?chb9>sjLw!vUOFxHo8o!ahvAn@^do9 zV<_2Z`WYEF%iO06$^+~6;@%E?3t7)q2n=(heD!k#SsV-@a9N_?t1sL48ue4xpz(%Y z-wevt-qJr5oz;G0*)RM#Zgegb;g`F-)%hH-FQ8axR8_jQ<4oeU-bB6=bwGv5|JZT< zN58k7|Isn|A3Mqa7}dW2tGjP8Fs|Lvrh_dh=DV({PKzrlZl|2N|Q(fiLI zpC2AM$(Q8b@qg6Y#Q%M?e}n(uzT_|av;<4i$N{9Ueg4y`e=5YfK3KGF)TJw0cnnW6 zSm3TFJ`YaCHBIrz3o5Q_l7!JjCb|9=?KSvs@ZaFSDGwa~>pb#3;eV&o-N650XMbSu ze;37=y1{>g{|5gJ{u}(i7XQn)S@(SZ-|24Re}B*&8T{Wx$)DplreN^j;J?9tgZ~Eq z?|BdPX8hmW|L+fn=Jy|WyZ*-J4gQ<=zdt5FOXkL_DN0ie53ne#ukOV0sRY9{h&fU&*#G=OfD z|EwI6a@{iux_!pvxWaOyscQAFH&3!du<>#6la1L4iXP~sWi^feG{qECOfkh2Q%o_% W6jMww#S~NiK;{2+#s201pa1|E9WRCe literal 0 HcmV?d00001 diff --git a/apps/python-sdk/dist/firecrawl_py-0.0.6-py3-none-any.whl b/apps/python-sdk/dist/firecrawl_py-0.0.6-py3-none-any.whl deleted file mode 100644 index 5aba56187b34e37ad84764a061ff2b3c769c0bb0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2573 zcmai$3pmsJAIIk&<&s+@Nm0UVtP9I+SjJq3h_u|A9hqU0OSw(%3@5oI|D{|~hs7k3 z6JeNb{%DDmTj^-(2%+&`=RAMuJpKQEzvun@p6~N~f3NTN^ZkBq4oCq(SpWdA4WLw! z<(5Sb9^1zosqrMo(|K%&pHGNa_yrIejm2RJXtWMK0;$!_aA6{Qw5@G*thE`AUA@3A zkftL(K~!H}7S9K8*g(=q1#bD8?!spT0RS*>75N{K8*m=;gR5=O#4)MnRadE*II+^< zlQl*IZYurz_~Lcs;g4p#cPj87$neJbW~*2FI<>7gn`h@7KHb+zMKr;~YWz|l^V6T` z_+D3(hBaCrIaC9yizEk}3oJ({8`N%JCl57Izs=VQGttDKKHMQSNWWFu_yCuu;# z-mx@kiBZj{7uS8(Gkc3%EHu)Ox9|x}d0H^~Kz8;znIajxs}(2 zY^j&q7E$Q|`<~R2jr_wHH zZ-}|aUE3(ksJbGgmt49{p8hFR1~tVDr(r*ITx$mW`Y5eaa)T~~@k0bJqbm+B_D+5GDxDSCZWRR9X_3Uelxn$%6?GwOIJA34 zaYD$qeH+LyB;pg!=9k94u1Dg*fJNHx-sVkA%a@l{li#Q}Mna5IX+<+xy31ZrPeSlu zNv_;aa{Sif`5?}-yzbR4`osm1{)0%1* zec3W=S14B?j47N_l-_b5YLfN`5wq%?6vS8^Hk&h|F3Feo;MDQn@CCMmgxw^W%6xeP z8f~`>9Se}eO*^p-_gf|T(7LN*%kn}7%F^T#^YLHrr_{GVpL&=@7U?fXyt9kbnRcc9 zf?3L*xi~+MnO$pFEvUXc{uyLl`rK-jzI>Z7R2%{Id`Ds_dsjb*?K~fj=ITpIT^&?7 zQjDwrV9F_B)Wg?FVIp@IWb~vtF;bivQ>sPMII&M873BQUR1`O{Y;tE2zB+p)r8)XF zVsU?zTyRv}l9_SkJmAjU`^%C7K32!`>YK=C2N&>WPPI8gYqvV=>U`d1jQV@OjcuiW ze1ER!gsRBZqUqh7oO=&mVQuUSjV;|LCm-pR$=0{lHUwmy!&Ko}ixjW7d%96=bFC7Z zIWD_B*(Wq0XJ4^BWX%#KcSSY+si_u3W381H!XMHS0%lihbpXBt6@{d^E=xlyBGW1N zG(SrkRT6sQ`lX=fv2(2p(L^L$d}0c~xBBjtay0eKTb$rib{;yA_$%pXs8pt@>MapY zC#TwAPh_dm5`jx_;1nYoxj_LSfLkU1E?tBo)7VZqQfH}i# zGkPTA!Lk-h+|*TpKvrVfeKpmSK41&!CD(_h2Lfxe6xPnlBhK^{7Z>F0rz&KdbcuYZ zj&EkyD(yxwvGy6AjIQ#6Ktq`V{TirZ!j)4F^kBUF`*E=w^Y|45lIyOE7q5OHa`le2 z>kFy-{Wd`=AqSt_O#vEPngJWM^2+@fS{kkz4!I+`cmLYDU75`P z#6nCLyeRaemH)b}%Q?fd?wj^+Wu+?u4!7CPK-kh*;z*!oUoYbU3OoRL9b!*M0}AFZf;byP!k3MA)b~Z#FZ(-Kt5{ zlDj4!0Kn$yz*bBY+}s{zahcu}84m`Sz8S@cBBbIalr^sRpK7GtwpkWTAcjE7N{ufc z3Ov*R;g*-B%g%z2Bh#^#1|8#gA)hdk%jfZC>zik<-hF(K^}N%0qFu7?#n|5Jqq840 zJqFdV>}LU9fK&dC#?_YGwufmCt{Eu`_w+`pB(cSdog4&1Ij&~jS?=x%qR$=IApCOV z1(l?vyZcgdlG_g`H<~t4(41<-@dd<_KkR1xt#mP6aDv%m`)k=@$`{76I=Y(IM)KlX z)r+P+iFWBlXO?IB10AP|5H~!Xcj|LhuEz_igyXDfg_)Y-`-7P-MT1R+LoaV0jqRU@`756F z=dby~b1go8S%LrECGgJsucs{T+h?P!Z|)g3BYxHu{0#sA%Ho`UKy1KxJ;G+j&m#Fd zV}jRjZ2bBE;`*P+e}>I>Bu!`wbIliueq9Yoe_kr?~h za8hWHB^^tWFDF6oFkk5e?=CRDpMK?hJ06PQ#;QR;j8=TCXG=FV8cm7lesm34;d?-8Qg-nZ) zZ5uMsadsV#qAkv0>JEsK^noc4WGo3rH66FLq^_rVq^Rbm_A)Qj;g7bl`^1b-wVXz5 z#SL`gnHsj!tNYa=f=lnTH>^7pFtkfyM$ zUtYGc-omj2c$+25FLO*|c_hp@X^YS4BW8EMWR%bJ40VCQWw?YvqJ@T*v{z84e3hBk z&f5=!&Oh_o{1M}ArQJJ?kt(IyqG4ROxUE=Bej5!-!duCF1WQ70@ZV4N+R-_cjPgWD zK93{8{na-LhZv~~CgC2AtGwZj)+P12S{PyL%6Xh zq;useu6}vo;#`k_eK1$K6>MGSnx#?y2wh;v{_H4=)iiyFv~?~BDs+6+xg$d0bBnHCxk_g9cq$zk4~ zT3EJL72yU{V(L!v!%e7x&Mf{vxg(_`w9%oKe67DGsqs*8lnEbq&;-{A6Y7(10Mjjm zaww^SuT`*O*xwg&+$`vHk-+ri4;Zx~RyTevAz{1rBKTG{i%Syj%RRItwvX4)w6_Ci|9jl_; z>~Z&@(DPn$j!JwLHd!JfZ24PiY{_*^eR~QjhQX9%4sVg$574I{%^E!qH;717mp#Ji z^=?K1YnUnFnl*{aNf$%PvN(52r~9rGVM`7;1^AX22iwl{;&ctoI_Jg`Kz!mV4UpOc znm47s`c%J+zWUmemNv##Mafqs=LxfJ!RLq#@0d={ROGiDAE`O62cBNEIdZGcgJJwi z#~4ad2T6`D=OO7buNfOb84P0twm_Z=k?!(@q24_H9Gv4a0X0}sr|h2@M{SlH6!bya z!8vA8AV@8cmIx4EucR?a5SMv=BWunVc1GsQ#})UD<W|HZwR#(MTUphfqcqCs=id1l^=TNk91NaiZ9^|lLI}Td8GZEVzob-q6`{)#>9zmr+R8=2UjNI zuWc)F*vsAc-pV{wNT`~R-nB(1GuPm*rT_L)u{du!!(*{3u7H379gk0n;~rF3@(Vf4 zO6S4&76M}P@gZ?4bpt{Q*g+t#cHoiT?TC9IIdGZjhM*E9C^}G~q2+L4d9H3`STo5p zXg0IxeyZHLV`V5wzJ(o?!ud{BrVvvYeCCZfwO*II#Wgr}su>sjM!e`*@r;ls7e2=b z6Em^(Yg0N(bBCn5d@psoO^j*3Ip01XR9~wVS#S|cD4Y}g)sp@;gm~-q!baaS@IDfl z7{%3fKm=V-ZC8$@77Hkye1`IPW-VEi3BDp5^E@&+j~EmFQvc&Umj_jANZRm*`WpJ_ ze9gO)x;{Kk_7+*XNP5n8)sAcO_LH1pBZG`m-2KmMfw!7(iGewa2J)+VjrGfsy zQf*;|PBo9qLlK6Tu}|jXKzmIrHhR^Aex?`*wZMl&Q^gvo4;+-m;Yr~>?cFa?(1R4J z)QVfLv>*3(D%y-aF-)A6-zxHuf!l5Nxn*ugIM}Y?^W{!EG`w5HoNX&=r2}|E<`#SX zBmL|wuZ|>sf=RCBE_g>xMLu&+kQ`)Xpi{C){}PtB7h{Nui|^$e7h zmXelIkaEQO_)21N&estbl!=;#nu!`Faeyg8nq6c6DDDvGF`BWe!)h-NLJE0>M$^lW zaURCL*>Vc+Ssv`q5f`_QXE}(xIWh#BiRWu6hEnJ6O$2;NxS2cKE0=NySd}KT;zB*i z3w9}%dNiWQTviAS^s{quiPfJ$I3N>@4UG1bm{!GkT<}ohTOXkb?-u*OD|@XG-qDo1 zue@Z-FHxIas3|?@_75t1&fGczg?&KdPXQflzc1ylO%~sm*|Z`z_f-NvYx1%d3Z+Xw ztc4+rGNL76LxaP0M$!{<3*Pz!w4@HaA5m+hrPB|Wx`O~$7>-Df$Tf_}jV{r>97c@p z!lxL;By|Y=y0u1P;xNoIowjbHkyyF>6k%W(J&75tT6#-);sSK#BAFP-HPMj;`&n(i z*FA0Boo+a}OZf))ev2e;PBM|%Gbl+40APkJ!JjdWP)L0vjp(7Kpa@k!^yu#{+$nZ! zaqKSy_(4MaozBWVYM@ySsyC~~SfiXxGtx70tk+mC@;B936Ff53P1AG0{x+BLJ&ran zfpk8f^-#-OclS1NicfDmyz_XfE|##}?s`^_)<2BHd37MDS|!^_*m(lHr;~fJ=}v;1 z&jpJ~t)9JhvsB}`4xd#uVnio34eP)c?{aREJjz(WPoNQ}S6Of4e$Lj+$u?xIoaaaa zhOfI@6gUpC{>40DQ8vO`Drfc7+aPY-?Ol?CEJb(6et@fM?ooOU#RW@qYlYqT?jW`& z(4C{IyovY26E9@JEbU9IG3k9gTF0%MYaouNebke`HoNfeA7aHZcAA>p`dAB|>~a4; z2H9Mrvm(N#i8u4o=Gx-{V_U$22MPsr9qV8NGzBF)@PFq>@-P4E!H0kSd<%X*rpg}? z|CSs71^@u~aN~aT@<+zMdHFj72>zjr|L5$J$dk_bj$HW}@-H8qF*7E$Z8X375%3>z{UXo+ literal 0 HcmV?d00001 diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.py index e955ffe..701810c 100644 --- a/apps/python-sdk/firecrawl/firecrawl.py +++ b/apps/python-sdk/firecrawl/firecrawl.py @@ -9,12 +9,7 @@ class FirecrawlApp: if self.api_key is None: raise ValueError('No API key provided') - from pydantic import BaseModel - from typing import Optional, Dict, Any - - class ScrapeParams(BaseModel): - url: str - extractorOptions: Optional[Dict[str, Any]] = None + def scrape_url(self, url: str, params: Optional[Dict[str, Any]] = None) -> Any: headers = { @@ -41,7 +36,6 @@ class FirecrawlApp: for key, value in params.items(): if key != 'extractorOptions': scrape_params[key] = value - print(scrape_params) # Make the POST request with the prepared headers and JSON data response = requests.post( 'https://api.firecrawl.dev/v0/scrape', diff --git a/apps/python-sdk/firecrawl_py.egg-info/PKG-INFO b/apps/python-sdk/firecrawl_py.egg-info/PKG-INFO index 61589c2..e54fda5 100644 --- a/apps/python-sdk/firecrawl_py.egg-info/PKG-INFO +++ b/apps/python-sdk/firecrawl_py.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: firecrawl-py -Version: 0.0.6 +Version: 0.0.8 Summary: Python SDK for Firecrawl API Home-page: https://github.com/mendableai/firecrawl Author: Mendable.ai diff --git a/apps/python-sdk/setup.py b/apps/python-sdk/setup.py index b870da6..78a4d84 100644 --- a/apps/python-sdk/setup.py +++ b/apps/python-sdk/setup.py @@ -2,12 +2,12 @@ from setuptools import setup, find_packages setup( name='firecrawl-py', - version='0.0.7', + version='0.0.8', url='https://github.com/mendableai/firecrawl', author='Mendable.ai', author_email='nick@mendable.ai', description='Python SDK for Firecrawl API', - packages=find_packages(), + packages=find_packages(), install_requires=[ 'requests', ],