Ug! dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddlm Z ddl mZddlmZddlmZddlZddlZdd lmZdd lmZmZdd lmZdd lmZmZdd lmZddlmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%eddl&m'Z'ddl(m)Z) ddl*Z*n #e+$rdZ*YnwxYw ddl,m-Z-n #e+$rdZ-YnwxYw ddl.Z.n #e+$rdZ.YnwxYwddl/m0Z0ddl/m1Z1ddl/m2Z2ddl/m3Z3ddl/m4Z4ddl5m6Z6m7Z7ddl5m8Z8ddl5m9Z9ddl5m:Z:ddl5m;Z;mZ>e=j?Z@e@AdddgeBe=jCzZDeDd e=Dz ZDegZEndd!gZEeEd"gzZFeGed#r#eDHejId$e=e>ZJd%ZKd&ZLd'ZMGd(d)eNZOdd+ZPd,ZQdd-ZRd.ZSd/ZTd0ZUd1ZVe!d2e8e9gejW!d3d4d5e;d56fd7ggd89ed:ZXe!d;eDe!dgd?d@ZYe!d;eDdAZZdBZ[ee!dCgdDdEZ\ee!d;gdFdGZ]dHZ^e!dIe=e!dJe=dKZ_dLZ`edMZadNZbe!dr2Ms#KKK;x ,..KKKloky threading get_contextspawnc Bt|dt|ddS)N_pool_workers)getattrbackends r1 get_workersr>[s 7GWWj$%G%G H HHr3c ||z SNr.)xys r1divisionrC_s q5Lr3c |dzSNr.rAs r1squarerHcs 6Mr3ceZdZdZdZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cdSr@r.)selfabcds r1__init__z#MyExceptionWithFinickyInit.__init__js r3N__name__ __module__ __qualname____doc__rQr.r3r1rJrJgs-     r3rJFcH|dkr|rtddddnt|S)NrMrNrOrP)rJ ValueError)rAcustom_exceptions r1exception_raiserr[ns4Avv"3)#sC===(2 4 Hr3c8tjdt)Ng?)timerKeyboardInterruptrGs r1interrupt_raiserr_usJsOOO r3c|dz|z|zS)zO A module-level function so that it can be spawn with multiprocessing. rFr.)rArBzs r1frbzs 6A:>r3cNttjdSNr)typer get_active_backendr.r3r1_active_backend_typergs +--a0 1 11r3cdt||dtdDS)Nn_jobsr=c3PK|]!}tt|V"dSr@r$rHr/is r1 z parallel_func..sC:.:. :.:.:.:.:.:.r3r#range) inner_n_jobsr=s r1 parallel_funcrtsE 98< 9 9 9:.:.$)!HH:.:.:. . ..r3c.tdksJdSrd)r)r.r3r1test_cpu_countrvs ;;??????r3c.tdksJdSrdr(r.r3r1test_effective_n_jobsrys    ! ! ! ! ! !r3contextzbackend_n_jobs, expected_n_jobs)rprprjNr)z positive-intz negative-intNone)idsc|d|5td|ksJ dddn #1swxYwYtddksJdS)Nr5r|rrx)rzbackend_n_jobsexpected_n_jobss r1test_effective_n_jobs_Noners ^ 4 4 4@@ t,,,?????@@@@@@@@@@@@@@@ 4 ( ( (A - - - - - -s 266r=rj)rrFr{verbose)rF dcdtdDt|||dtdDksJdS)Nc,g|]}t|Sr.rHr/rAs r1r2z(test_simple_parallel..s ) ) )1VAYY ) ) )r3)rjr=rc3PK|]!}tt|V"dSr@rlrs r1roz'test_simple_parallel..sC&6&6'(""&6&6&6&6&6&6r3rrr#)r=rjrs r1test_simple_parallelrs * )a ) ) ) &HFG$ & & &&6&6,1!HH&6&6&6 6 6 6 6 6 6 6 6r3cl|tjddtjd5}t d|dt d D}|gd ksJ dddn #1swxYwYd |D}t|d ksJdS) Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrFric3PK|]!}tt|V"dSr@rlrs r1roz6test_main_thread_renamed_no_warning..sC6262#$OGFOOA  626262626262r3rp)rrc>g|]}dt|jv|S)zworker timeout)strmessager/ws r1r2z7test_main_thread_renamed_no_warning..s+NNNa'7s19~~'M'M'M'M'Mr3r)setattrr5current_threadwarningscatch_warningsr#rrlen)r= monkeypatchwarninforesultss r1#test_main_thread_renamed_no_warningrs) y799ACCC   - - -$5(!W5556262(-a62626222)))#####$$$$$$$$$$$$$$$ON8NNNH x==A      s.s,,,a ,,,r3c34K|]}d|jdvVdS)zbacked parallel loops cannotrN)args)r/eachs r1roz)_assert_warning_nested..sA''/$)A,>''''''r3nogilFr) rr simplefilterrtallr;sysflagsr)r=rsexpectedrwarnings_are_correctwarnings_have_the_right_lengths r1_assert_warning_nestedrsM   - - -Bh'''gLAAAABBBBBBBBBBBBBBB-,8,,,H  K#&''$'''$$ '.ci%&H&H(H ""]]a' +(J,J Juts&AA A z%parent_backend,child_backend,expected))r4r,T)r4r4F)r,r,T)r,r4T)r5r,T)r5r4Tctd|fdtdDtd|fdtdD}|dkr>tr$t|st jdt|sJdSt |sJdS) NrFric3XK|]$}ttddV%dS)rFr=rsrNr$rr/_ child_backends r1roz0test_nested_parallel_warnings..s[//  (&''!   //////r3rc3XK|]$}ttdV%dS)rFrNr)r/rrrs r1roz0test_nested_parallel_warnings..s[55  (&''!   555555r3r5zThis test often fails in PyPy.reason)r#rrranypytestxfailr)parent_backendrrress `` r1test_nested_parallel_warningsrs/HA~...////q /// 5(!^ 4 4 455555q 555  C$$  B3s88 B L @ A A A A3xx3xxr3)r4r,r5cdg}d}tj||f}|||dsJdS)NFc$tjd5}tddtdDdddn #1swxYwYt t |t |dk|d<dS)NTrrFr|c3PK|]!}ttdV"dS)皙?Nr$rr/rs r1rozPtest_background_thread_parallelism..background_thread..sC66'(r""666666r3rr)rrr#rrprintr)is_run_parallelrs r1background_threadz=test_background_thread_parallelism..background_thread s  $D 1 1 1 6X HA   66,1!HH666 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 c(mm ]]a/s1AAA)rrr)r5Threadstartjoin)r=rrts r1"test_background_thread_parallelismrsfgO000  18JKKKAGGIIIFFHHH 1 r3chtd|dtdDdS)NrFric3PK|]!}ttdV"dS){Gz?Nrlrs r1roznested_loop..sC(0(0!"(0(0(0(0(0(0r3rqr<s r1 nested_looprsO'HAw'''(0(0&+Ahh(0(0(000000r3rrcntd|fdtdDdS)NrFric3RK|]!}ttV"dSr@)r$rrs r1roz#test_nested_loop..!sG/?/?01 ]++/?/?/?/?/?/?r3rq)rrs `r1test_nested_looprs\/HA~.../?/?/?/?5:1XX/?/?/??????r3ctr@)rYr<s r1raise_exceptionr%s r3c 8tt5tdd5}|ttdtt dgdddn #1swxYwYddddS#1swxYwYdS)NrFr4ri)rrYr#r$rrr s r1)test_nested_loop_with_exception_with_lokyr)s   99 Q / / / 98 H*gk**622.go..v668 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 9999999999999999999s5BAA7+ B7A; ;B>A; ?BBBctdtddfdtdDsJdS)z1Input is mutable when using the threading backendr)maxsizerFr5ric3RK|]!}tjdV"dSrN)r$put)r/rqs r1roz1test_mutate_input_with_threads..4sF,-,-q,-,-,-,-,-,-r3N)rr#rrfull)rs@r1test_mutate_input_with_threadsr1su aA+HA{+++,-,-,-,-#(88,-,-,---- 6688OOOOOr3)rrFrpctd}d|Dt|d|DksJdS)z.Check the keyword argument processing of pmap. c0g|]}t|dSrrBrbrs r1r2z(test_parallel_kwargs..=s" $ $ $1QqAYYY $ $ $r3r|c3TK|]#}tt|dV$dSrrNr$rbrs r1roz'test_parallel_kwargs..>s7#D#D1JGAJJqA$6$6$6#D#D#D#D#D#Dr3Nr)rjlsts r1test_parallel_kwargsr9sk ))C $ $ $ $ $ #HF # # ##D#D#D#D#D D D E E E E E Er3ctd}d|D}td|5}|j}||d|DksJ||d|DksJt%t |t |jusJdddn #1swxYwYtt |jJ||d|DksJtt |jJdSdS)Nrc0g|]}t|dSrrrs r1r2z4test_parallel_as_context_manager..Ds"'''a!q '''r3rric3TK|]#}tt|dV$dSrrrs r1roz3test_parallel_as_context_manager..M7==AZWQZZQ///======r3c3TK|]#}tt|dV$dSrrrs r1roz3test_parallel_as_context_manager..Nrr3c3TK|]#}tt|dV$dSrrrs r1roz3test_parallel_as_context_manager..Zs799A+++999999r3)rrr#_backendrr>)r=rrpmanaged_backends r1 test_parallel_as_context_managerrAs ))C''3'''H G , , , K*11===========11=========== >//;qz3J3JJJJJ K K K K K K K K K K K K K K K  ~1:&&... qq99S99999 9 9 9 9 ~1:&&...~..sA*B##B'*B'cGddtttd5tddfdt d Dd d d d S#1swxYwYd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. ceZdZdZdS)1test_parallel_pickling..UnpicklableObjectc td)N123) RuntimeErrorrLs r1 __reduce__z.UnpicklableObject.__reduce__esu%% %r3N)rSrTrUrr.r3r1UnpicklableObjectrds# & & & & &r3rzthe task to sendmatchrFr4ric3bK|])}ttV*dSr@r$id)r/rrs r1roz)test_parallel_pickling..isU+5+5%&,772;;    ,!,!+5+5+5+5+5+5r3rN)objectrrr#rr)rs@r1test_parallel_picklingr _s &&&&&F&&&  %8 9 9 955*6***+5+5+5+5*/))+5+5+5 5 5 5555555555555555555s4A,,A03A0 byteorder)<>=cdtjdd|djj}t ddfdtd D}|D]?\}}||ksJ||jjksJtj |@dS) Nc||jjfSr@)dtyper rGs r1inspect_byteorderz=test_parallel_byteorder_corruption..inspect_byteorderrs!'###r3)rFrpi4rFr4ric3HK|]}tVdSr@r$)r/rrrAs r1roz5test_parallel_byteorder_corruption..ysH00*+"!""1%%000000r3rp) rarangereshapeviewrr r#rrtestingassert_array_equal)r initial_np_byteorderresult x_returnedbyteorder_in_workerrrAs @@r1"test_parallel_byteorder_corruptionr!ms $$$ ! V$$))Y*:*:*:;;A7, /XQ / / /00000/4Qxx000F,255' '"&:::::"j&6&@@@@@ %%a444455r3cttd|ddtdDdksJdS)NrFrjr=timeoutc3PK|]!}ttdV"dS)gMbP?Nrrs r1roz0test_parallel_timeout_success..sC?2?2"#u?2?2?2?2?2?2r3r)rr#rrr<s r1test_parallel_timeout_successr'sr >xq'2>>>?2?2',Ryy?2?2?222 3 368 9 9 9 9 9 9r3ctt5td|ddtdDddddS#1swxYwYdS)NrFrr$c3PK|]!}ttdV"dSrNrrs r1roz-test_parallel_timeout_fail..sC:3:3#$NGENN2  :3:3:3:3:3:3r3r)rr r#rrr<s r1test_parallel_timeout_failr+s   3397D999:3:3(-b :3:3:3 3 3 33333333333333333333AAAc ttt5td|dt ddDdddn #1swxYwYtt 5td|ddDdddn #1swxYwYtd|5}t |jJt |j}tt5|dt ddDdddn #1swxYwYt |jJt |j|usJdtd D|d td DksJt |j}tt 5|d dDdddn #1swxYwYt |jJt |j|usJd td D|d td Dks"J|j |j |j |j f dddn #1swxYwYt |jJnOtt 5tdddDdddn #1swxYwYtt5tddt ddDdddn #1swxYwYtt5tdddtdDddddS#1swxYwYdS)NrFricPg|]#\}}tt||$Sr.r$rCr/rArBs r1r2z&test_error_capture..s@5551#""1a((555r3)rr)rrcHg|]}tt| Sr.r$r_rs r1r2z&test_error_capture..,>>>!*)**1-->>>r3cPg|]#\}}tt||$Sr.r/r0s r1r2z&test_error_capture..s@;;;"a,'(++Aq11;;;r3c0g|]}t|dSrrrs r1r2z&test_error_capture.."2221QqAYYY222r3rc3TK|]#}tt|dV$dSrrrs r1roz%test_error_capture..7CCAZWQZZQ///CCCCCCr3cHg|]}tt| Sr.r2rs r1r2z&test_error_capture..s,GGG13'"233A66GGGr3c0g|]}t|dSrrrs r1r2z&test_error_capture..r6r3c3TK|]#}tt|dV$dSrrrs r1roz%test_error_capture..r8r3r|cHg|]}tt| Sr.r2rs r1r2z&test_error_capture..r3r3cPg|]#\}}tt||$Sr.r/r0s r1r2z&test_error_capture..s1 E E EA WX  q! $ $ E E Er3r)rjrc3TK|]#}tt|dV$dS)T)rZNr$r[rms r1roz%test_error_capture..sN ! !'W% & &q4 @ @ @ ! ! ! ! ! !r3r#)rrZeroDivisionErrorr#zipr^r>rrr _iteratingn_completed_tasksn_dispatched_tasks _abortingrJ)r=r original_workerss r1test_error_capturerGs  ~ % & & 6 6 /HAw / / /55 #FF 3 3555 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 % & & @ @ /HAw / / />>v>>> @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ Q 0 0 0 Hx011===*8+<== )** < <;;&)&&&9&9;;;<<< < < < < < < < < < < < < < < < x011===x0119IIIII22b 222HCCrCCCCCDDDD +8+<== )** I IGGGGGHHH I I I I I I I I I I I I I I Ix011===x0119IIIII22b 222HCCrCCCCCDDD#X%?+X-?GDDDD9               H8,--5555 % & & @ @ HA   >>v>>> @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ ! " "GG E EVV1D1D E E E G G GGGGGGGGGGGGGGGG * + +""%1%%% ! !Bii ! ! ! " " """""""""""""""""""s3AA #A ;%B,,B03B0?J$D7+ J7D; ;J>D; ?BJG9- J9G= =JG= BJJ"%J"$LL L#2M!!M%(M%2N??OOc0d d}td|5}ttd5|d|dDdddn #1swxYwYttd5|d |d Ddddn #1swxYwYttd5|d |d Ddddn #1swxYwYddddS#1swxYwYdS)Nrc3`KtdD]}||krtd|VdS)NIterator Raising Error)rrrY)raise_atrns r1 my_generatorz1test_error_in_task_iterator..my_generatorsHr  AH}} !9:::GGGG  r3rFrirKrc3PK|]!}tt|V"dSr@rlrms r1roz.test_error_in_task_iterator..2 C CQogfooa  C C C C C Cr3)rLc3PK|]!}tt|V"dSr@rlrms r1roz.test_error_in_task_iterator..rOr3rc3PK|]!}tt|V"dSr@rlrms r1roz.test_error_in_task_iterator..s2 D DQogfooa  D D D D D Dr3r)r#rrY)r=rMrs r1test_error_in_task_iteratorrTs G , , , E J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D D J&> ? ? ? D D A C C,,*B*B*B C C C C C C D D D D D D D D D D D D D D DJ&> ? ? ? E E A D D,,*C*C*C D D D D D D E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E Es|D  A D A D  A !D : B& D &B* *D -B* .D  C3' D 3C7 7D :C7 ;D  DDc6|d|zdS)Nz Consumed %s)append)queueitems r1consumerrYs LL%&&&&&r3zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) rZr\r^r`r[r]r_rarbrdrcrectfd}td||fd|D|ksJtdksJdS)zC Test that with only one job, Parallel does act as a iterator. c3hKtdD]}d|z|VdSNrz Produced %irrrVrnrWs r1producerz'test_dispatch_one_job..producerHq  A LL* + + +GGGG  r3rrj batch_sizer=c3TK|]"}tt|V#dSr@r$rY)r/rArWs r1roz(test_dispatch_one_job.. sI?9?9()%##?9?9?9?9?9?9r3 N)listr#r)r=rnexpected_queuerkrWs @r1test_dispatch_one_jobrts FFE ?HA*g>>>?9?9?9?9-5XZZ?9?9?9999 N " " " " u::      r3ctj}|fd}tddd|fd|Dt}|ddksJ|d d d }|d ksJ|d }||ksJt dksJd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3hKtdD]}d|z|VdSrhrirjs r1rkz/test_dispatch_multiprocessing..producerrlr3rFrrp)rjrn pre_dispatchr=c3TK|]"}ttdV#dS)rNrp)r/rrWs r1roz0test_dispatch_multiprocessing..!sQF=F=,-%''F=F=F=F=F=F=r3rrZNrz Consumed anyr{r`rq)rManagerrrr#indexr)r=managerrkqueue_contentsfirst_consumption_indexproduced_3_indexrWs @r1test_dispatch_multiprocessingrs' jllG LLNNE FHA!!WEEEF=F=F=F=19F=F=F====%[[N !  , , , ,-RaR066~FF "R ' ' ' '%++L99 5 5 5 5 5 u::      r3ctddd5}|dtdD|jdksJ ddddS#1swxYwYdS)NrFautor5rmc3PK|]!}tt|V"dSr@rrms r1roz/test_batching_auto_threading..80 . .Q+'"++a.. . . . . . .r3rr#rrrcompute_batch_size)rs r1test_batching_auto_threadingr2s v{ C C C4q  . .%++ . . ....z,,..!33333444444444444444444AA##A'*A'ctdd|5}|dtdD|jdksJ ddddS#1swxYwYdS)NrFrrmc3PK|]!}tt|V"dSr@rrms r1roz2test_batching_auto_subprocesses..@rr3rrr)r=rs r1test_batching_auto_subprocessesr<s vw ? ? ?31  . .%++ . . .... z,,..22222 333333333333333333rctt5tddddtdDddddS#1swxYwYdS)zCMake sure that exception raised during dispatch are indeed capturedrFr)rjrwrc3PK|]!}tt|V"dSr@r?rms r1roz*test_exception_dispatch..KsG7=7=-. %G$ % %a ( (7=7=7=7=7=7=r3r#N)rrYr#rrr.r3r1test_exception_dispatchrHs   ==6A6667=7=27))7=7=7= = = ===================r,cftddtdDdS)NrFr|c3PK|]!}tt|V"dSr@r?r/js r1roz(nested_function_inner..PsG99)*! !!!$$999999r3r#rqrns r1nested_function_innerrOsMHA99.3Bii99999999r3cftddtdDdS)NrFr|c3PK|]!}tt|V"dSr@)r$rrs r1roz(nested_function_outer..UsG>>./&%&&q))>>>>>>r3r#rqrs r1nested_function_outerrTsMHA>>3899>>>>>>>>r3z'https://github.com/joblib/loky/pull/255rctt5}td|dtdDdddn #1swxYwYt |j|j|j}d|}d|vsJd|vsJd |vsJt |jtusJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rFric3PK|]!}tt|V"dSr@)r$rrms r1roz1test_nested_exception_dispatch..csO,B,B23 *G) * *1 - -,B,B,B,B,B,Br3r#Nrrr[) rrYr#rrrrertbr)r=excinfo report_linesreports r1test_nested_exception_dispatchrYs=   Bw+7+++,B,B7tt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJtt5}td|dddn #1swxYwYdt |jvsJdS)Nrriz&n_jobs == 0 in Parallel has no meaning?z2.3z$n_jobs could not be converted to int invalid_str)rrYr#_initialize_backendrr)r=rs r1test_invalid_njobsrs   Bw7+++??AAABBBBBBBBBBBBBBB 3s7=7I7I I I I I   DwW---AACCCDDDDDDDDDDDDDDD 3s7=7I7I I I I I   Fww///CCEEEFFFFFFFFFFFFFFF 1S5G5G G G G G   Nw w777KKMMMNNNNNNNNNNNNNNN 1S5G5G G G G G G GsG$AA  A <$B,,B03B0#$DDD $E::E>E>)2gffffff@rFct||}|dksJ|dtdD}tdt |DsJdS)NrirFc3PK|]!}tt|V"dSr@rlrms r1roz.test_njobs_converted_to_int..s2 2 21OGFOOA   2 2 2 2 2 2r3rc3BK|]\}}|t|kVdSr@r)r/rnrs r1roz.test_njobs_converted_to_int..s099$!QqF1II~999999r3)r#_effective_n_jobsrrr enumerate)r=rjrrs r1test_njobs_converted_to_intrs 000A   A % % % % ! 2 2b 2 2 2 2 2C 99)C..999 9 999999r3c tdtdtvsJtdtksJ td=dS#td=wxYw)N test_backend)r'rr*r.r3r1test_register_parallel_backendrse%!.2EFFF))))'+>>>>>> ^ $ $ $H^ $$$$$s 8A Ac*ttksJ tdtddttksJ t t _n#t t _wxYwttksJdS)Nr5T) make_default)rgDefaultBackendr'r*rr+r r.r3r1test_overwrite_default_backendrs  ! !^ 3 3 3 33!+x /D/3 5 5 5 5#%%)999999$3  ? 2222  ! !^ 3 3 3 3 3 3s 6A$$A7zOnly without multiprocessingcpttd5tddtdDdddn #1swxYwYt d5tdtdDddddS#1swxYwYdS)Nz)joblib backend '.*' is not available on.*rr4r<c3PK|]!}tt|V"dSr@rlrms r1roz2test_backend_no_multiprocessing..s2 F F!3!3 F F F F F Fr3rpc3PK|]!}tt|V"dSr@rlrms r1roz2test_backend_no_multiprocessing..s288!?76??1%%888888r3)r UserWarningr#rrr%r.r3r1test_backend_no_multiprocessingrsU {@ B B BGG    F FU1XX F F FFFFGGGGGGGGGGGGGGG  ( ( (99 88uQxx888888999999999999999999s#1AAA//B++B/2B/c||d5tj\}}|dksJtddksJt}|jdksJ|dkr6t |t usJt |jt usJn|dkr6t |tusJt |jtusJn|dkr6t |tusJt |jtusJnJ| dr5t |tusJt |jtusJddddS#1swxYwYdS)Nrpr|r,r4r5test_) r rfr(r#rjrer rr"r startswithr)rz backend_nameactive_backend active_n_jobsrs r1check_backend_context_managerrs a ( ( (;;(0(C(E(E% !!!!""a'''' JJx1}}}} , , ,''+AAAAA ##'====== V # #'';6666 ##{22222 [ ( (''+;;;;; ##'777777  $ $W - - ;''+>>>>> ##':::::#;;;;;;;;;;;;;;;;;;sE E((E,/E,cg|]}d|zS)ztest_backend_%dr.rms r1r2r2s---q---r3rpch|tvr!|t|tttksJt |ttksJt ddfdtDttksJdS)NrFr5ric3XK|]$}|tt|V%dSr@)r$r)r/rNrzs r1roz/test_backend_context_manager..sQ,<,< ,<.-..w::,<,<,<,<,<,.277MGDMM!$$777777r3rc,g|]}t|Sr.rrms r1r2z>test_parameterized_backend_context_manager..1111tAww111r3) rr*rrgrr rfrerr#rjrrr)rrzrrrrs r1*test_parameterized_backend_context_managerrs/4666  ! !^ 3 3 3 3 1 5 5 588(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !^ 3 3 3 3 3 3s B C!!C%(C%c$ttksJ|tdd5tj\}}t |tusJ|jdksJ|dksJt}|jdksJ|j |usJ|dtdD}dddn #1swxYwY|dtdDksJttksJdS)N+)rrr|c3PK|]!}tt|V"dSr@rrms r1rozFtest_directly_parameterized_backend_context_manager..(rr3c,g|]}t|Sr.rrms r1r2zGtest_directly_parameterized_backend_context_manager..)rr3) rgrrr rfrerr#rjrrr)rzrrrrs r13test_directly_parameterized_backend_context_managerrsy  ! !^ 3 3 3 3 -B777 B B B88(0(C(E(E% N##'CCCCC#r))))!!!! JJx1}}}}z^++++!77eAhh77777888888888888888 11a111 1 1 1 1 ! !^ 3 3 3 3 3 3sB C  CCcFtdtjS)Nr)rosgetpidr.r3r1sleep_and_return_pidr/s "III 9;;r3cttksJtdksJtddt dDS)NrrFr|c3NK|] }ttV!dSr@)r$rrs r1roz"get_nested_pids..;sG11"#DD>>>>>>>>>r3rback_compat_backend)r5r4r,rc||5tddtdD}|D]$}tt|dksJ% ddddS#1swxYwYdS)NrFr|c3NK|] }ttV!dSr@)r$rrs r1roz6test_nested_backend_context_manager..RsF( (  %GO $ $ & &( ( ( ( ( ( r3rr)r#rrrset)rzr= pid_groups pid_groups r1#test_nested_backend_context_managerrIs   ,,'XQ'''( ( 2YY( ( (   $ , ,Is9~~&&!+++++ , ,,,,,,,,,,,,,,,,,,sAA22A69A6)rFr{Nc*dtdfdtdD|5tdfdtdDddddS#1swxYwYdS)Nctt|ksJt|}t|ksJdSr@)rgr*r(r#r)expected_backend_typeexpected_n_jobs r1check_nested_backendz?test_nested_backend_in_sequential..check_nested_backendbsW$%%2G)HHHHH*.99zz++--??????r3rr|c3TK|]"}ttdV#dSr)r$r+)r/rrs r1roz4test_nested_backend_in_sequential..lsL  &$%%oq99r3rc3JK|]}tVdSr@r)r/rr=rrjs r1roz4test_nested_backend_in_sequential..rsL   *G( ) )'6 : :      r3rq)r=rjrzrs`` @r1!test_nested_backend_in_sequentialrZs4@@@HAr  ( ( (        2YY                        s5BB B c||5}|tur|d}|tur|d}|j|ksJ ddddS#1swxYwYdS)Nr=r)r%r& nesting_level)rz inner_backendexpected_levelctxr=s r1check_nesting_levelrxs   73 o % %)nG & & &!fG$66666 777777777777777777s0A  AA outer_backendrcBt|dtd|fdtdDd5tfdtdDddddS#1swxYwYdS)NrrFric3VK|]#}ttdV$dSrr$rr/rrzrs r1roz-test_backend_nesting_level..sN..  %#$$WmQ??......r3rr|c3VK|]#}ttdV$dSrr r s r1roz-test_backend_nesting_level..sN''07.//JJ''''''r3)rr#rr)rzrrs` `r1test_backend_nesting_levelr s. 222-HA}---.....r... q ) ) )'' '''''!"II''' ' ' '''''''''''''''''''s2BBBwith_retrieve_callbackTcddlGfddt}td|d|d5}tdfdt d D|t ur|d jd ksJ|tur|djd ksJddddS#1swxYwYdS) Nrc8eZdZdZZjdZdS))test_retrieval_context..MyBackendrc32K|xjdz c_dVdSr}rrs r1retrieval_contextz;test_retrieval_context..MyBackend.retrieval_contexts" FFaKFF EEEEEr3N)rSrTrUrnsupports_retrieve_callbackcontextmanagerr) contextlibr sr1rrs= %;"  "   # "   r3r retrievalcbtddt|DS)NrFr|c3PK|]!}tt|V"dSr@rrms r1roz>test_retrieval_context..nested_call..s0!C!CQ+'"++a..!C!C!C!C!C!Cr3rq)ns r1 nested_callz+test_retrieval_context..nested_calls2!xq!!!!C!C%((!C!C!CCCCr3rFr|c3HK|]}t|VdSr@r)r/rnrs r1roz)test_retrieval_context..sI   !GK  # #      r3rr=r)rrr'r#rrr%rnr&)rzr rrrrs ` @@r1test_retrieval_contextrss$k9555DDD   !    1XX       o % %y>#q(((( & & &q68q====!!!!!!!!!!!!!!!!!!sA+B44B8;B8rn)rr{gQ?ctt5t|ddddS#1swxYwYdS)Nrn)rrYr#rs r1test_invalid_batch_sizers   ((J''''((((((((((((((((((s 377z)n_tasks, n_jobs, pre_dispatch, batch_size) )rFrFrr)rFrFrjrrrFrjr)irFrjrr )rrrjr)rqrjr)rq 2 * n_jobsr)rqrr)r$rqr#rX)r!rqr#rc|||d}dt|D}tdi|dt|D}||ksJdS)N)rjrwrnc,g|]}t|Sr.rrms r1r2z0test_dispatch_race_condition..s222aq 222r3c3PK|]!}tt|V"dSr@rlrms r1roz/test_dispatch_race_condition..s2 L L!3!3 L L L L L Lr3r.r)n_tasksrjrwrnparamsrrs r1test_dispatch_race_conditionr*sz  &((F225>>222H h    L LU7^^ L L LLLG h      r3ctj}tdd}|jd}|}||ksJdS)NrFr,rirz)rget_start_methodr# _backend_argsget)mp_start_methodrrz start_methods r1test_default_mp_contextr1s`)++O#4555Ao!!),,G++--L ? * * * * * *r3c@|dkrtjd}tjd}|ddtjjtd|fdtdDdS)Nr,r7rirFric3hK|],}ttjjV-dSr@)r$rdotT)r/rnrMs r1rozAtest_no_blas_crash_or_freeze_with_subprocesses..sJ(3(3$%13(3(3(3(3(3(3r3) rr6rrandom RandomStaterandnr4r5r#rr)r=rngrMs @r1.test_no_blas_crash_or_freeze_with_subprocessesr:s###.)) )   # #C $AF1acNNN(HAw'''(3(3(3(3).q(3(3(333333r3afrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) c|dkr+tjdkrtjdt|}t tjd|gdddS)Nr,forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r% stdout_regex) rr,rskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrr executable)r=codes r12test_parallel_with_interactively_defined_functionsrEs ###(;(=(=(G(G 6 7 7 7 7 > >w G GD t$b*,,,,,,r3aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r$rkwargsc |dvr/|tkstjdkrtjdt |||tj tj tj }| d}| |ttj|jgdddS) N)r,r7win32zNot picklable with pickle)rFr=rGjoblib_root_folderzunpicklable_func_script.pyrr=r>) SQUARE_MAINrplatformrr@)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrBrpathdirnamejoblib__file__rwriterrCstrpath)r=rFrGtmpdirrD code_files r10test_parallel_with_unpicklable_functions_in_argsrWCs ... ; & &#,'*A*A /000 4 ; ;+7??27??6?+K+KLL < N ND 899I OOD *+R*,,,,,,r3aimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rKc|d}|tttj|jgdddS)Nz(joblib_interactively_defined_function.pyr=)r?r%)rrS5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrCrTrUscripts r17test_parallel_with_interactively_defined_functions_lokyr\s\ [[C D DF LLFGGG (*r3aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) c|d}|tttj|jgddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r? stderr_regexr%)rrS9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrCrTrZs r1:test_parallel_with_interactively_defined_bound_method_lokyrbs\ [[D E EF LLJKKK3>6>:'>'5"$&&&&&&r3cbtg}td|gksJdS)NrFr|)iterr#)exhausted_iterators r1%test_parallel_with_exhausted_iteratorrfs;b 81   0 1 1R 7 7 7 7 7 7r3cJttjddS)z+Helper function to force gc in each worker.rN)rr]rr.r3r1_cleanup_workerrhsOOOJrNNNNNr3ct|tjstdt ||S)Nz#Expected np.memmap instance, got %r) isinstancermemmap TypeErrorrecopy)rMs r1 check_memmaprnsA a # #!=Q!! ! 6688Or3cd}tdd|d|dD}t||t|D]%\}}tj||&tdd|d|dD}t||t|D]%\}}tj||&dS) Nc3xKt|D]'}tjdtj|zV(dS)Nrr)rrronesfloat32)rrns r1generate_arrayszBtest_auto_memmap_on_arrays_from_generator..generate_arrayssJq 4 4A'"BJ///!3 3 3 3 3 4 4r3rFrrj max_nbytesr=c3PK|]!}tt|V"dSr@r$rnr/rMs r1roz.V@@@@%& a  @@@@@@@@@@@@r3rrc3PK|]!}tt|V"dSr@rxrys r1roz.rzr3)r#rArrrr)r=rtrrrs r1)test_auto_memmap_on_arrays_from_generatorr|sW444 @haAw???@@@@*9/#*>*>@@@@@@@@GW)F)FGG88 %%h7777 @haAw???@@@@*9/#*>*>@@@@@@@@GW)F)FGG88 %%h777788r3c|Sr@r.)args r1identityrs Jr3c|dj}tj}t j|dt j|dg}t||t|dtdfdd D\}td tj sJd j |ksJtj ||dS) Nz test.mmapuint8rqr) mmap_moderFr|c3RK|]!}ttV"dSr@)r$r)r/rrks r1roz.test_memmap_with_big_offset..s6 H Hq!2!2!26!:!: H H H H H Hr3rSr)rrTmmapALLOCATIONGRANULARITYrzerosrrr rr#rjrkoffsetrr)rUfnamesizeobjrrks @r1test_memmap_with_big_offsetrs KK $ $ ,E  %D 8D ( ( ("'$g*F*F*F GCe %3 ' ' 'F ha   H H H HC H H HHHGF fQi + ++++ !9 d " " " "J!!#v.....r3chtjd5}tdddtdDdddn #1swxYwYt |dksJ|d}t |jtsJt|jdksJdS) NTrr)rjr%c3PK|]!}tt|V"dSr@rlrms r1rozFtest_warning_about_timeout_not_supported_by_backend..s2%L%LQogfooa&8&8%L%L%L%L%L%Lr32rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr#rrrrjrrr)rrs r13test_warning_about_timeout_not_supported_by_backendrs   - - -M%1%%%%L%L%))%L%L%LLLLMMMMMMMMMMMMMMM x==A     A ai - ---- qy>>       s2AAAc|||<|Sr@r.) input_listrzrs r1set_list_valuersJu Lr3)rrFrcdgdzt|ddfdtdD}t|}tdt |DsJdS)Nrr generatorr5rj return_asr=c3VK|]#}tt||V$dSr@)r$r)r/rnrs r1rozPtest_parallel_return_order_with_return_as_generator_parameter.. sS+E+E67 Aq11+E+E+E+E+E+Er3c3(K|] \}}||kVdSr@r.r/vrs r1rozPtest_parallel_return_order_with_return_as_generator_parameter..s*::$!QqAv::::::r3)r#rrrrrrA)rjrrs @r1=test_parallel_return_order_with_return_as_generator_parameterrs qJ+XV{)++++E+E+E+E;@88+E+E+EEEF &\\F ::#j&"9"9::: : ::::::r3cB|rtdt|S)Nr#)rr)edelays r1_sqrt_with_delayrs   b 77Nr3cnt|d|dtdDtfdtdD}dgttddz}t d t ||DsJt dS) Ngenerator_unorderedrc3`K|])}tt|dz|dkV*dS)rFrN)r$rrms r1rozK_test_parallel_unordered_generator_returns_fastest_first.. sY'F'F67! !!!Q$a11'F'F'F'F'F'Fr3rc36K|]}tVdSr@)next)r/rrs r1rozK_test_parallel_unordered_generator_returns_fastest_first..#s)==qd6ll======r3 rrFc3(K|] \}}||kVdSr@r.rs r1rozK_test_parallel_unordered_generator_returns_fastest_first..'s;1aQr3)r#rrsortedrrrrAr)r=rjquickly_returnedexpected_quickly_returnedrs @r18_test_parallel_unordered_generator_returns_fastest_firstrs'XV/D%''''F'F;@99'F'F'FFFF====E!HH=====!"d5B<<&8&8 8 8:JKK   OOOOOr3rF sequentialc&t||dSr@)rr=rjs r17test_parallel_unordered_generator_returns_fastest_firstr/s=WfMMMMMr3zThis test requires daskctjdd5|d5td|dddn #1swxYwYddddS#1swxYwYdSNrF n_workersthreads_per_workerdask) distributedClientr)rjrzs r1Atest_parallel_unordered_generator_returns_fastest_first_with_daskr9s   A   OOwvOO AvNNNOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOs3 AA AA AA AAA)rrFrr{cdgdgdzz}tt5tj}t||d|Ddddn #1swxYwYtj|z }|dksJdS)NrMrrric3ZK|]&}ttj|V'dSr@r$r]rrms r1roz%test_abort_backend..KsH1414'( GDJ   " "141414141414r3rJ)rrlr]r#)rjr=delayst_startdts r1test_abort_backendrEsUbTCZ F   44)++00001414,2141414 4 4 4444444444444444 w B 777777s8A##A'*A'c`tjtdt}d|d<|S)NgArqFr)rrrintbool)r~rs r1get_large_objectrQs* WS\\ . . .FF1I Mr3ct|||5}|dtdD}t|t|~tddddS#1swxYwYdS)N)rjr=rc3PK|]!}tt|V"dSr@)r$rrms r1roz0_test_deadlock_with_generator..\s6JJ13'"233A66JJJJJJr3r)r#rrrr)r=rrjr rs r1_test_deadlock_with_generatorrWs % ' ' '*2JJb JJJJJ V  V  sAA00A47A4rrrc(t|||dSr@)r)r=rrjs r1test_deadlock_with_generatorres "'9f=====r3ctjdd5|d5td||dddn #1swxYwYddddS#1swxYwYdSr)rrr)rzrrjs r1%test_deadlock_with_generator_and_daskrms  A   ??wv?? &dIv>>>?????????????????????????????????s3 AA AA AA AA Acttd5t|||}|dtdD}t j}|dtdD}dddn #1swxYwYt j|z dks Jd ~t dS) N)This Parallel instance is already runningrr=rc3PK|]!}ttdV"dSrrrs r1roz/test_multiple_generator_call..s2::1^WU^^A&&::::::r3rc3PK|]!}tt|V"dSr@rrms r1roz/test_multiple_generator_call..s0;;1  A;;;;;;r3rrFZThe error should be raised immediatly when submitting a new task but it took more than 2s.)rrr#rrr]r)r=rrjr grgen2s r1test_multiple_generator_callrys  A C C C<<FGyIII H::b ::: : :)++x;;c ;;;;; <<<<<<<<<<<<<<< 9;; 1 $ $ $ $ % $ $ OOOOOsA*B  BBct|||5}|dtdD}tj}ttd5|dtdD}dddn #1swxYwYtj|z dks Jd  dddn #1swxYwY~t dS) Nrc3PK|]!}ttdV"dSr*rrs r1roz7test_multiple_generator_call_managed..s2;;A^WU^^B'';;;;;;r3rrrc3PK|]!}tt|V"dSr@rrms r1roz7test_multiple_generator_call_managed..s0==Q+'"++a..======r3rrFr)r#rrr]rrr)r=rrjr rrg2s r1$test_multiple_generator_call_managedrsx &'% ' ' '  *2 H;;r;;; ; ;)++ LEGGG > >==%**=====B > > > > > > > > > > > > > > > y{{W$q((( ()(((                OOOOOs6A C #B C B C B 'C  CC return_as_1 return_as_2c t|||dtdD}t|||dtddD}|dkrt|}|dkrt|}tdt |tdDsJtdt |tddDsJdS) Nrc3VK|]$}tt|dzV%dSrFNrrms r1roz9test_multiple_generator_call_separated..sOAA"#  a1fAAAAAAr3rc3VK|]$}tt|dzV%dSrrrms r1roz9test_multiple_generator_call_separated..sOBB"#  a1fBBBBBBr3rJrc3(K|] \}}||kVdSr@r.r/rrns r1roz9test_multiple_generator_call_separated..s*88FCsax888888r3c3(K|] \}}||kVdSr@r.rs r1roz9test_multiple_generator_call_separated..s*==FCsax======r3)r#rrrrrA)r=rrrjrrs r1&test_multiple_generator_call_separatedrs6 AK@@@AA',RyyAAA  A B&'[ A A ABB',R}}BBB  B+++ 1II+++ BZZ 88c!U2YY&7&7888 8 8888 ==c"eBmm&<&<=== = ======r3zbackend, error))r4T)r5F)rFc |dkrttjdtd||}|dt dD}t j|d}|rttd n t}|5tj }td||d t dd D}t|d krt|}td t|t dd DsJ dddn #1swxYwYtj |z dksJd} |jr,| dkr&| dz } tjd|jr| dk&tj |z dksJ|jdkr |jsJdSdS)Nr4zRequires multiprocessingrFrc3PK|]!}ttdV"dSr*rrms r1roz.s277##777777r3rc tdS)NzGenerator collected)rr.r3r1z;test_multiple_generator_call_separated_gc..su-B'C'Cr3z The executor underlying Parallelrc3VK|]$}tt|dzV%dSrrrms r1roz.sO@ @ &'MGDMM!q& ! !@ @ @ @ @ @ r3rJrc3(K|] \}}||kVdSr@r.rs r1roz.s*@@Q3!8@@@@@@r3rrrprr)rrr@r#rrweakreffinalizerrrr]rrrrAaliverrrE) r=rrerrorr rg_wrrrretrys r1)test_multiple_generator_call_separated_gcrsq 6  .///7kBBBH77U2YY77777A  ACC D DD  $|#EFFFF!mm AA )++ ?HQ; ? ? ?@ @ +0R==@ @ @     / / /q A@@#ar2*?*?@@@@@@@@@#AAAAAAAAAAAAAAA& 9;; 1 $ $ $ $ E *   2 * 9;; 1 $ $ $ $!x!##q((!!!!)( "!sB!EE EcR|j}tdd||5}|dtjdgdzDt t j|dksJ|dtdDdddn #1swxYwYtdD]'}t j|sntd (td tdd| }|d tjdgdzD|d tdDtdD](}t j|sdStd )td )NrFr)rjrvr= temp_folderc3PK|]!}tt|V"dSr@rxrys r1roz(test_memmapping_leaks.. s5 G Gq ',   " " G G G G G Gr3rrc3NK|] }ttV!dSr@r$rhrs r1roz(test_memmapping_leaks..s3 7 7 "'/ " " $ $ 7 7 7 7 7 7r3rrz/temporary directory of Parallel was not removedruc3PK|]!}tt|V"dSr@rxrys r1roz(test_memmapping_leaks..#s5CC1glACCCCCCr3c3NK|] }ttV!dSr@rrs r1roz(test_memmapping_leaks..$s333Qgo  333333r3) rTr#rr6rrlistdirrrrAssertionError)r=rUrrs r1test_memmapping_leaksrs  ^F q'$ & & & 8)*  G GRY-=-=b-A-A,BQ,F G G GGGG2:f%%&&****  7 7eAhh 7 7 7777 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 3ZZPPz&!!  E b NOOO a999AACC)9)9")=)=(>(BCCCCCCA33%((333333 3ZZPPz&!!  EE b NOOOsBB''B+.B+)Nr4r5ctd|dtdD}|dtdDksJdS)NrFric3HK|]}td|VdS)c |dzSrEr.rGs r1rz2test_lambda_expression...5s !q&r3Nrrms r1roz)test_lambda_expression..4sJ2929)*!  !!!$$292929292929r3rcg|]}|dzS)rFr.rms r1r2z*test_lambda_expression..6s111!qAv111r3rq)r=rs r1test_lambda_expressionr.sp 2ha1112929.3Bii29292999G 11uRyy111 1 1 1 1 1 1r3cd}d}d|z td||}|fdt|D|jj|jjksJ|jj|jjksJ|fdt|D|jj|jjksJ|jj|jjksJdS) zCTest that a parallel backend correctly resets its batch statistics.rFig@r)rrjr=c3\K|]&}ttjV'dSr@rr/rn task_times r1roz6test_backend_batch_statistics_reset..B8>>gdj)$$>>>>>>r3c3\K|]&}ttjV'dSr@rrs r1roz6test_backend_batch_statistics_reset..Hrr3N)r#rrr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r=rjn_inputsrrs @r1#test_backend_batch_statistics_resetr 9sFHX IFG<<>>>eHoo>>>>>> J , J 4 5 5 5 5 J / J 7 8 8 8 8A>>>>eHoo>>>>>> J , J 4 5 5 5 5 J / J 7 8 8 8 8 8 8r3cdD]}tt|jtksJt|d}t|jtusJt|d}t|jtusJt|d}t|jtusJtdd d }t|jt usJ|d d5td }t|jt usJ|jdksJ dddn #1swxYwY|d d5td d}t|jt usJ|jd ksJ dddn #1swxYwY|d d5td }t|jtusJ|jdksJ dddn #1swxYwY|d d5td d}t|jtusJ|jd ksJ ddddS#1swxYwYdS)N)rrFr{r|threadsrjprefer processes sharedmemrjrequirerFr4)rjr=r r rprr)rer#rrrr"rj)rzrjrs r1$test_backend_hinting_and_constraintsrOs 4 4HF+++455GGGG F9 5 5 5AJ#33333 F; 7 7 7AJ>1111 FK 8 8 8AJ#333333 6)<<.MyCustomThreadingBackendTcdSr@r.rs r1 apply_asynczgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_async Dr3c|Sr@r.rs r1r(zltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsMr3NrSrTrUsupports_sharedmem use_threadsrr(r.r3r1MyCustomThreadingBackendrs=!         r3r rFrr rrc"eZdZdZdZdZdZdS)\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcdSr@r.rs r1rzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asyncrr3c|Sr@r.rs r1r(zmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsrr3Nrr.r3r1MyCustomProcessingBackendr"s="         r3r%rr)rjrrzUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r=r) r!r#rer readouterrrstriprrY)capsysrzr rr%outerrrs r19test_backend_hinting_and_constraints_with_custom_backendsr,s #6 ))++ , ,<< Ak 2 2 2AJ#;;;;; A{ 3 3 3AJ#;;;;;; <<<<<<<<<<<<<<<$7 **,, - - Ak 2 2 2AJ#<<<<<$$&&Sbyyyybyyyy A{B ? ? ?AJ#33333$$&&S@yy{{h&&&&byyyyy!$   KK2244kJJJJKKKKKKKKKKKKKKKKKKs7ABB B 8B;FFFGG  G cxtt5tddddn #1swxYwYtt5tddddn #1swxYwYtt5tdddddn #1swxYwYt}tt5tdddddn #1swxYwYtt5td dddddS#1swxYwYdSdS) Ninvalidrrrr)r rr4r&r,)rrYr#rr.r3r1,test_invalid_backend_hinting_and_constraintsr/s   ## """"###############   $$####$$$$$$$$$$$$$$$   ::  [9999:::::::::::::::  ~J   : : V[ 9 9 9 9 : : : : : : : : : : : : : : : J   E E . D D D D E E E E E E E E E E E E E E E E E E ~sT266A..A25A2 B++B/2B/C//C36C3D--D14D1c .td5}t|jj|jjfg}dkr|cdddS|fdt dD}||dzcdddS#1swxYwYdS)zCPerform nested parallel calls and introspect the backend on the wayrFr|rNc3VK|]#}ttdddz iV$dS)limitrNr.)r$_recursive_backend_info)r/rnrHr2s r1roz*_recursive_backend_info..sY''5G344OO519OOO''''''r3r)r#rerrSrrr)r2rHr this_levelrs`` r1r3r3s    'qAJ''0!*2JKL A::''''''''!'''''"1XX'''''GAJ& ''''''''''''''''''s.B /B  BBc||d5t}dddn #1swxYwYtdkrd}d}nd}d}|dz}|df|dfd|fd|fg}||ksJdS)NrFr|rrrBackendr)r3r)title)rzr=backend_types_and_levelssecond_level_backend_type max_leveltop_level_backend_typeexpected_types_and_levelss r1test_nested_parallelism_limitr=s  # # #==#:#<#< ==============={{a$7! $6! $]]__y8 # "A& i( i( ! $'@ @ @ @ @ @ @s )--cltjdd5tjt dtj}t dD]e}|d5t|}dddn #1swxYwYt|dksJtd|DsJf|d5t}dddn #1swxYwYt|dksJtd |DsJ ddddS#1swxYwYdS) NrFrgcArqr)datarc3(K|] \}}|dkVdSDaskDistributedBackendNr.r/rrs r1roz4test_nested_parallelism_with_dask..sH@@"477@@@@@@r3c3(K|] \}}|dkVdSrAr.rCs r1roz4test_nested_parallelism_with_dask..s@<<tQ33<<<<< > ><<ws3xxrx000q @ @A N N+B+M+M+M( N N N N N N N N N N N N N N N/00A5555@@&>@@@@@ @ @ @ @WV__ A A'>'@'@ $ A A A A A A A A A A A A A A A+,,1111<<":<<<<< < < < <<<<<<<<<<<<<<<<<<2 D)>B D)B ?D)C  D) C$ $D)'C$ (3D))D-0D-c^tdtdDS)z6A horrible function that does recursive parallel callsc3NK|] }ttV!dSr@)r$_recursive_parallelrms r1roz&_recursive_parallel..s4GG2g12244GGGGGGr3rFrq) nesting_limits r1rHrHs+ 8::GGeAhhGGG G GGr3cn||d5tt5}tdddn #1swxYwYdddn #1swxYwY|j}|dkr1ddlm}t ||tfrtj dt |tsJdS)NrFr|r4r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrHr&joblib.externals.loky.process_executorrKrjrrrRecursionError)rzr=rexcrKs r1test_thread_bomb_mitigationrP sK  # # #"" M " " "g  ! ! ! " " " " " " " " " " " " " " """""""""""""""" -C& QPPPPP c1=A B B N LL M M M c> * ******s1A> AA AA AAAcvi}dD]$}tj|||<%|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr)renvironr.r)env_varsvars r1_run_parallel_sumr\'sFH3,, s++ \#&& &&r3zNeed OpenMP helper compiledcptd|dtdD}ttdzd}|D]b\}}||ksJ|D]@\}}|dr|t |ksJ0|dksJ|dksJAcdS)NrFric3NK|] }ttV!dSr@)r$r\rs r1roz-test_parallel_thread_limit..3sE22)*"!""$$222222r3r_THREADSrX1)r#rrrr)itemsendswithr)r=rexpected_num_threadsworker_env_varsomp_num_threadsrrs r1test_parallel_thread_limitrf0s2ha11122.3Ahh222Gy{{a/33,3$$("66666*0022 $ $KD%}}Z(( $$8 9 999999|++++|||||  $$$r3czttd5|dddddS#1swxYwYdS)NzPlease install daskrr)rrY)rzs r1)test_dask_backend_when_dask_not_installedrhAs  "7 8 8 8s 044cNGddt}d}||5tjt|5t ddt dDdddn #1swxYwYddddS#1swxYwYdS)Nc"eZdZdZddZdZdS)3test_zero_worker_backend..ZeroWorkerBackendcdSrdr.)rLrrHs r1rz=test_zero_worker_backend..ZeroWorkerBackend.configureM1r3Nc td)NzNo worker availabler )rLfunccallbacks r1rz?test_zero_worker_backend..ZeroWorkerBackend.apply_asyncPs455 5r3cdSrdr.rs r1r(zDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsSrmr3r@)rSrTrUrrr(r.r3r1ZeroWorkerBackendrkLsF    6 6 6 6     r3rrz&ZeroWorkerBackend has no active workerrrFr|c3PK|]!}tt|V"dSr@rrms r1roz+test_zero_worker_backend..Ys0@@!{wr{{1~~@@@@@@r3)rrrrr#rr)rzrr expected_msgs r1test_zero_worker_backendruHsq,.check_globalsds!!r3rFr|c3FK|]}tVdSr@rr/rnrys r1roz.jE1414%&   141414141414r3z changed valuec3FK|]}tVdSr@rr{s r1roz.sr|r3)rxr#rrr)workers_global_variablerys @r1)test_globals_update_at_each_parallel_callr\s& *""" =??. . . . .0ha00014141414*/((14141444 & ' ',<+= = = = =) =??o - - - -0ha00014141414*/((14141444 & ' 'O+< < < < < < .sUGEGE67/.//11GEGEGEGEGEGEr3rFr rrrr@r#rrr(rr)r)rjrworkers_threadpool_infosexpected_child_num_threadss r1(test_threadpool_limitation_in_child_lokyrs122K ;1 CDDDDFxvFFFGEGE;@88GEGEGE E Ef % %F!$Y[[F%:A!>!>4k688888r3inner_max_num_threads)rrFrNct}t|dkrtjd|d|5t |dt dD}dddn #1swxYwYt |}| tt|zd }n|}t|||dS) Nrrrr4rr|c3NK|] }ttV!dSr@rrms r1roz>test_threadpool_limitation_in_child_context..sM;I;I:; 3G2 3 3 5 5;I;I;I;I;I;Ir3rFrr)rzrjrrrrs r1+test_threadpool_limitation_in_child_contextrsS122K ;1 CDDDD /D E E EII#:86#:#:#:;I;I?DQxx;I;I;I$I$I IIIIIIIIIIIIIIIf % %F$%()>%B%B""%:"4k688888s1BBBvar_name)rSrTrRcztddtj} dtj<t |fdt dD}|ddgksJ|dd 5t |fd t dD}dddn #1swxYwY|d d gksJ |tj=dS|tj<dS#|tj=n|tj<wxYw) NTreusec@tj|Sr@)rrYr.)rs r1_get_envz>test_threadpool_limitation_in_child_override.._get_envsz~~h'''r34r|c3HK|]}tVdSr@rr/rnrrs r1roz?test_threadpool_limitation_in_child_override..sG*;*;,- GH  h ' '*;*;*;*;*;*;r3rFr4rrc3HK|]}tVdSr@rrs r1roz?test_threadpool_limitation_in_child_override..sG.?.?01!!!(++.?.?.?.?.?.?r3r`)rshutdownrrYr.r#rr)rzrjroriginal_var_valuerrs ` @r1,test_threadpool_limitation_in_child_overriders%%%..000(((116" 8)(&)))*;*;*;*;*;16q*;*;*;;;3*$$$$ WV1 5 5 5 ? ?-hf---.?.?.?.?.?5:1XX.?.?.???G ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?3*$$$$$  % 8$$$#5BJx   % 8$$#5BJx 5 5 5 5s1AD!4C! D!C%%D(C%) D!D:)rFrr{cd}||td}tdD]%}||td}||ksJ&dS)Nctdt|fdtdDdS)Nrr|c3RK|]!}ttV"dSr@)r$sum)r/rnrAs r1rozAtest_loky_reuse_workers..parallel_call..s1CCA  QCCCCCCr3r)rjrAs @r1 parallel_callz.test_loky_reuse_workers..parallel_callsJ "IICCCCrCCCCCCCCr3Trr)rrr)rjrfirst_executorrexecutors r1test_loky_reuse_workersrs DDD M&*666N2YY** f(t444>)))))**r3)F)rr)rpr@)rVrrr]rrrr5 tracebackrmathrrpicklerrrr,r rrQr r rjoblib._multiprocessing_helpersrjoblib.test.commonrrrrrjoblib.testingrrrrrjoblib.externals.lokyrrWrposix ImportError _openmp_test_helper.parallel_sumrrjoblib._parallel_backendsrrr r!r"joblib.parallelr#r$r%r&r'r(r)r*r+rmRETURN_GENERATOR_BACKENDSpoprkeysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrVr6rr>rCrH ExceptionrJr[r_rbrgrtrvrymarkrrrrrrrrrrrrrr r!r'r+rGrTrYrtrrrrrrrrrrrrrrrrrextendrrrrrrrr_parallel_backendsrrrrr rrr*r1r:rArErNrL SQUARE_LOCAL SQUARE_LAMBDArWrBrOrPrRrYr\rarbrfrhrnr|rrrrrrrrrrrrrrrrrrrrrr rr,r/r3r=rErHno_coverrPr\rfrhrurrrrrrrrr.r3r1rs &&&&&& """"""(((((( ......--------33333355555555++++++++++++++>;;;;;;LLLL EEE>>>>>>>LLLKKK877777666666<<<<<<999999111111--------++++++,,,,,,555555777777779999999999*HMOO/666Vff]X]__555KK(KKKK:)62$ }4 72}7nbnW55666/*III             222... """ Y*:;<<% b"""--- . :000 .. =<. Y*++ X~~~&& Y %%77&%'&,+7 Y*++,+.2  47776 Y@@@AA  BA  000  _h''  x((??)(('? 999 Xyyy!!FF"!F Y)**//+*/: 5 5 5  [///**55+* 5& Y)**99+*9  Y)**33+*3 Y())A"A"*)A"H Y!!EE"!E.''' Y!!  )///0 JJJK L M M   M M"!   Y)**+*:444 Y())33*)3===999 >>>  Y)**CDD--ED+*-*     +       Y*++HH,+H$ Y)** X}}}%%::&%+*:%%% 4 4 4$=>>>99?>9;;;*$5QQQ#7 ''--EE!HH---  Y899 Y)?;<<44=<:94(#4 Y*:;<<44=<4( Y*:;<<44=<4( 111>>>>>)5>>>/;;; Y00011 Y*:;<< , ,=<11 , X}}}%% Y)** Y*:;<<  =<+*&% 4777 _/00 _/00 Y*:;<< ' '=<1010 ' Y*:;<<  %e}55!!65=<!> \===))((*)(   8 / / / 0 0 0 0+++  Y())33*) 3. 0,  Y()) , ,*) ,-)@    Y("*BB7)LMM ][, FGG  "?"?"?@@ , ,A@HGNM , (9P FbgooGOOFO$$&&F''Q6X   =, FbgooGOOFO$$&&F''-:4&&&888    Y())88*) 80  / /  /    999-- ; ;.- ; (Aq6** Y566,GHHNNIH +* NAr7++ Y*:;<< t$=>>>OO?>=<,+O Y*++ X~~~&&'&,+     Y122 [;(=>?? X~~~&&>>'&@?32 > Ar7++ [;(=>?? Y*:;<< t$=>>>???>=<@?,+  ? Y122 [;(=>?? X~~~&&'&@?32. Y122 [;(=>?? X~~~&&'&@?320 Y122 ][*?@AA ][*?@AA X~~~&&>>'&BABA32>*       ][*?@AA ][*?@AA/"/"BABA /"d  Y())(P(P*) (PV Y%'Zt[!!...22 2 Y())99*)9( Y*:;<<++=<+\ Y*:;<<3K3K=<3KlEEE* ' ' ' ' Y-.. Y*:;<<AA=</.A*  Y*:;<< t$=>>><<?>=< <&HHHH  Y*:;<<   &+1F++=< +0''' Y2>$vGG %BCCC $ $DCHG $ Y*:;<< 4(ABBBCB=<  Y*:;<<AA=<A&===B,,, ; ; ;  X~~~&&88'& 8&   $ooo66 X2w Y*:;<<88=< 76  82 X2w Z---.. Y*:;<<66=<..  6> Xzzz""**#"***s6BB! B!%B,,B65B6:B??C C