
    Ug"                     	   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dl m!Z! d dl m"Z" d dl m#Z# d dl m$Z$ d dl m%Z% d dl m&Z& d dl m'Z( d Z)d Z*d Z+d Z,d Z-eed                         Z.e eej        dk    pdd          d                         Z/eed                         Z0ed             Z1ee edeej2        gdd g!          d"                                     Z3ee edeej2        gdd g!          d#                                     Z4ee ed$dd g          d%                                     Z5ee ed$dd g          d&                                     Z6ee ed$dd g          d'                                     Z7ee ed$dd g          d(                                     Z8eed)                         Z9eed*                         Z:ee ed$dd g          d+                                     Z;ee ed$dd g          d,                                     Z<ee ed$dd g          d-                                     Z=ee edeej2        gdd g!          d.                                     Z>ee ed$ ej?        dej@        A                    d/          0          d g          d1                                     ZBee edeej2        gdd g!          d2                                     ZCeee edeej2        gdd g!          d3                                                 ZDeee edeej2        gdd g!          d4                                                 ZEee edeej2        gdd g!          d5                                     ZFd6 ZGee edeej2        gdd g!          d7                                     ZHd8 ZIee ed9 eJ ejK        eej2        g eLd:                              d;  ejK        dd g eMeN eLd:                              D             !          d<                                     ZOd= ZPd> ZQe eej        dk    d?           ed@dAdBg          dC                                     ZRedD             ZSdE ZTeedF                         ZUdS )G    N)sleep)
with_numpynp)with_multiprocessing)with_dev_shm)raisesparametrizeskipif)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutorhas_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap_get_backing_memmapc                  2    t          j        dd           d S )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     Z/var/www/surfInsights/venv3-11/lib/python3.11/site-packages/joblib/test/test_memmapping.pysetup_moduler    !   s    %c======r   c                  ,    t          j                     d S N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler$   %   s    ,.....r   c                 (    t          |           J | S r"   r   )arrays    r   check_memmap_and_send_backr'   )   s    u%%111Lr   c                 ^    | \  }}}t           j                            ||         |           dS )zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr1   .   s2      $D(HJ!!$x.(;;;;;r   c                     | \  }}}||         |k    sJ ||xx         dz  cc<   t           j                            ||         d|z             dS )zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       Nr)   r,   s       r   inplace_doubler4   9   sa      $D(H>X%%%%NNNaNNNJ!!$x.!h,?????r   c                    t           j        j        }|                     d          j        }t          j        |t           j        dd          }dt          j        |j        d         |j	                  z  |dd<   |
                                 t          j        |t           j        d	d
dd          }t          j        d                              |j                  |dd<   |ddddddf         }t          j        |          }|j        }t          d| j        dd          fd} ||          }	t          |	          sJ t!          |	t           j                  sJ  ||	|            ||          }
t          |
          sJ  ||
|            ||          }t!          |t           j                  rJ t          |          sJ  |||            ||          }t!          |t           j                  rJ t          |          sJ  |||           |dz  }t          |          rJ  ||          }t          |          rJ t!          |t           j                  rJ  |||dz             t          j        |          }t          |          rJ  ||          }t!          |t           j                  sJ t          |          rJ  |||           dS )z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r9   N)         r+Fr?   )r9   r:   r;   orderoffset<      r3   cTc                 *     |           \  }} || S r"   r   xconsr-   reducers      r   reconstruct_array_or_memmapzEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapi        WQZZ
dtT{r   r=   )r   r*   r+   joinstrpathmemmapfloat64aranger:   r9   flushreshapeasarrayTr   r   
isinstancendarray)tmpdirr+   filenamebufferabrG   drM   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrL   s                    @r    test_memmap_based_array_reducingrh   H   sD    6{{;''/H Yxrz4HHHF 	&,q/FFFFF111I
LLNNN 		("*I3q	2 	2 	2A9R==  ))AaaaD
 	
!B$"ac/A 	
1A	A (fnc4HHG    
 21!44O00000ory11111*** 21!44O00000*** 21!44O/295555500000***11!44O/295555500000*** 
QB#B'''''22266#$455555*BI66666'Q/// 
BB#B'''''22266&
33333#$455555',,,,,r   win32r   z2PermissionError only easily triggerable on Windows)reasonc                    |                      d          j        }d                    |          }t          j        t
          j        d|gt          j        t          j                  }|                                 |	                                \  }}|j
        dk    sJ |dk    sJ d                    |          }||                                v sJ d S )	Nr6   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r[   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rO   rP   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)rZ   r[   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;''/H: 	!!; < 	#.$4Z_ *	1 	1 	1AFFHHH}}HC<1#::::
3
:
:8
D
DC#**,,r   c                 n   t           j        j        }|                     d          j        }t          j        |t           j        dd          }t          j        d                              |j	                  |d d <   |dd         }|d d ddf         }|d d d d d d df         }|d	d
d         }t          d | j        dd          fd} ||          }	t          |	          sJ t          |	t           j                  sJ  ||	|            ||          }
t          |
          sJ  ||
|            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||            ||          }t          |          sJ  |||           d S )Nr6   )d      r   r=   r7   r8   i r   
   r>   rE   r=   r?   rG   Tc                 *     |           \  }} || S r"   r   rI   s      r   rM   zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rN   r   )r   r*   r+   rO   rP   rQ   rR   rS   rU   r:   r   r   rX   )rZ   r+   r[   r]   r^   rG   r_   erM   r`   ra   rb   rc   e_reconstructedrL   s                 @r   )test_high_dimension_memmap_array_reducingr      s#    6{{;''/H 		("*4D	 	 	A9&''//88AaaaD 	
!B$A	!!!QrT'
A	!!!QQQ1*A	!Aa%A (fnc4HHG     21!44O00000ory11111***11!44O00000***11!44O00000***11!44O00000***11!44O00000*****r   c                    |                      d          j        }dt          j        z  }t          j        dz   }t	          j        |d||z             }t          |dd|d|d d d	
	  	        }t          |t          j                  sJ |j        |k    sJ t          |dd|d|dz  fd|d	
	  	        }t          |          j        |k    sJ d S )Nr6   r>   rE   r7   )r;   r:   uint8rCF)r9   r;   rC   rB   r:   stridestotal_buffer_lenunlink_on_gc_collectr3   )r3   )
rO   rP   mmapALLOCATIONGRANULARITYr   rQ   r   rX   rC   r   )rZ   fnamesizerC   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr      s    KK$$,Et))D'!+F5t4&=AAAJ%e7-33d.2T;@B B BJ j"),,,,,&&&&,W3vSqylD4"  
 0118FBBBBBBr   factorymultiprocessingloky)idsc           	         t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           |	                    t          fd
t          j        d                   D                         |dt          j        j                  z             t          j        |t           j        dd          |	                    t          fdt          j        d                   D                        t          j        |          g k    sJ  |dt          j        j                  z              |dt          j        j                  z             t          j        |t           j        ddd          t!          t"                    5  |	                    t$          fdt          j        d                   D                        ddd           n# 1 swxY w Y   t!          t&          t(          f          5  |	                    t          fdt          j        d                   D                        ddd           n# 1 swxY w Y   |                                 ~dS # |                                 ~w xY w)z@Check that subprocess can access and update shared memory memmappoolr   r3   
max_nbytestemp_folderr6   r=   r>   r7   r8         ?c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S rE   r   ranger:   ).0ijr]   s      r   
<listcomp>z)test_pool_with_memmap.<locals>.<listcomp>  W     ; ; ;#$(-agaj(9(9; ;#$ !"Aq63/ ; ; ; ;r   r   )r>   r=   rG   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S )rE          @r   )r   r   r   r^   s      r   r   z)test_pool_with_memmap.<locals>.<listcomp>#  r   r   r   r      )r9   r:   r;   rC   c                     g | ]}|d fS )g      @r   r   r   rG   s     r   r   z)test_pool_with_memmap.<locals>.<listcomp>4  s    GGGAsGGGr   Nc                     g | ]}|d fS )r   r   r   s     r   r   z)test_pool_with_memmap.<locals>.<listcomp>9  s    "J"J"J1Aq#;"J"J"Jr   )r   r*   r+   mkdirrP   rO   rQ   float32fillmapr4   r   r:   onesoslistdirr   AssertionErrorr1   RuntimeError
ValueError	terminate)	r   rZ   r+   pool_temp_folderr{   r[   r]   r^   rG   s	         @@@r   test_pool_with_memmapr   
  sy    6 ||F++3q.>???A(;;{++3IhbjTJJJ	s	n ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	< 	1a"'!'"2"22333 IhbjSIII	n ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	< z*++r1111 	1a"'!'"2"223331a"'!'"2"22333 IhbjC"$ $ $ N## 	I 	IEE+GGGGU171:5F5FGGGHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 \:.// 	L 	LEE."J"J"J"Jagaj8I8I"J"J"JKKK	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	
AA 	
sV   F4K 2A H>2K >IK IK $A J0$K 0J44K 7J48K K)c                 N   t           j        j        }|                    d          j        } | dd|          }	 |                    d          j        }t          j        |t           j        dd                              d	           t          j	                  t          t           j                  rJ t                    sJ |                    t          fd
t          j        d                   D                         |dt          j        j                  z              |dt          j        j                  z             t#          j        |          g k    sJ 	 |                                 ~dS # |                                 ~w xY w)z?Check that subprocess can access and update shared memory arrayr   r   r3   r   r6   r   r7   r8   r   c                 X    g | ]&}t          j        d                    D ]	}||fdf
'S r   r   )r   r   r   r]   a_views      r   r   z4test_pool_with_memmap_array_view.<locals>.<listcomp>V  sW     ; ; ;#$(-agaj(9(9; ;#$ !'A4 ; ; ; ;r   r   N)r   r*   r+   r   rP   rO   rQ   r   r   rV   rX   r   r   r4   r   r:   r   r   r   r   )r   rZ   r+   r   r{   r[   r]   r   s         @@r    test_pool_with_memmap_array_viewr   @  s    6 ||F++3q.>???A;;{++3IhbjTJJJ	s Afbi00000#F+++++	n ; ; ; ; ;(-agaj(9(9; ; ; 	< 	< 	<
 	1a"'!'"2"2233361rwqw'7'7#7888 z*++r11111 	
AA 	
s   D8F F$backendc                 v   d                     |           }t          j        t          j        d|gt          j        t          j                  }|                                 |                                \  }}|j        dk    s4J |	                                dz   |	                                z               d S )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r^   rl   rm   r   

)
rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r   rz   r{   r|   r}   s        r   -test_permission_error_windows_reference_cycler   g  s    $ 	% & 	#.$4Z_ *	1 	1 	1AFFHHH}}HC<1cjjllV3cjjllBr   c                    d                     |           }t          d          D ]}t          j                                        }t          j                            t                    |d<   t          j	        t          j        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    s
J |            |dk    sJ t          j        d d         d	vrd
|vsJ d S )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r=   
PYTHONPATHrl   rn   ro   envr   r   )r=      r   )r=   r   rE      resource_tracker)rp   r   r   environcopypathdirname__file__rq   rr   rs   rt   ru   rv   rw   rx   version_info)r   rz   _r   r{   r|   r}   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s	   ( 	) , 1XX 2 2jooGOOH55LcndC8$.O$.O> > > 	
==??S|q   #   czzzzBQB'===
 'c11112 2r   c                    t          j        t          d                     t          d| d          fdt	          d          D                       \  } t          d| d          fdt	          d          D                       \  }t
          j                            |          t
          j                            |          k    sJ d S )N      Y@r3   r   n_jobsr   r   c              3   T   K   | ]"} t          t                    d           V  #dS r[   Nr   getattrr   r   r&   s     r   	<genexpr>z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  Q       F F01
++F F F F F Fr   rE   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  r   r   r   rS   intr   r   r   r   r   )r   
filename_1
filename_2r&   s      @r   #test_parallel_isolated_temp_foldersr     s     Ic#hhEE81g"EEE F F F F5:1XXF F F  LZ F81g"EEE F F F F5:1XXF F F  LZ 7??:&&"'//**E*EEEEEEEr   c                    t          j        t          d                    t          d| d          5 } |fdt	          d          D                       \  } |fdt	          d          D                       \  }d d d            n# 1 swxY w Y   t
          j                            |          t
          j                            |          k    sJ d S )Nr   r3   r   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  I       
 
45GGUJ//
 
 
 
 
 
r   rE   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  r   r   r   )r   r{   r   r   r&   s       @r   &test_managed_backend_reuse_temp_folderr     s6   
 Ic#hhE	G	;	;	; 
qq 
 
 
 
9>q
 
 
 
 
 q 
 
 
 
9>q
 
 
 
 
	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 7??:&&"'//**E*EEEEEEEs   ABBBc                     t          j        t          d                    } t                      }t                      }d }t	          j        || |f          }t	          j        || |f          }|                                 |                                 |                                 |                                 t          |          dk    sJ t          |          dk    sJ ||k    sJ d S )Nr   c                 4    t          ddd          5 }t          d          D ]Z} | fdt          d          D                       \  }|                    t          j                            |                     [	 d d d            d S # 1 swxY w Y   d S )Nr   r3   r   )r   r   r   c              3   T   K   | ]"} t          t                    d           V  #dS r   r   r   s     r   r   z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  sI        <=$GG$$UJ77     r   rE   )r   r   addr   r   r   )r&   	temp_dirsr{   r   r[   s   `    r   concurrent_get_filenamezJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  s    fQ2>>> 	9!2YY 9 9Q    AFq    
 bgooh778888	9	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9s   A+BBB)targetr-   rE   )	r   rS   r   set	threadingThreadstartrO   len)r&   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr     s    Ic#hhE9 9 9 
	&e5G-H
 
 
B 
	&e5G-H
 
 
B HHJJJHHJJJGGIIIGGIII!""a''''!""a''''!3333333r   c            	         d} g d}|D ]\  }}}t          j        t          j        d|                     ||          gt           j        t           j                  }|                                 |                                \  }}|j        |k    sJ |	                                            d|vsJ |	                                            d S )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserE   )r  r  rE   rl   )f1f2rm   r   )
rq   rr   rs   rt   rp   ru   rv   rw   rx   ry   )rz   functions_and_returncodesr  r  rx   r{   r|   r}   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr    s    1Cd! ! ! 8 < <B
cndCJJ"J4L4LM$.OJOM M M	==??S|z)))3::<<)))"#---szz||----< <r   c                    d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|j        dk    s
J |            |dk    sJ t          j        d d         d	vrd
|vsJ d S d S )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rl   r   r   r   r=   r   r   )rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   rv   rw   rx   r   )r   rz   r   r{   r|   r}   s         r   'test_many_parallel_calls_on_same_objectr  L  s    $ 	% & *//

C11C	s#		 	 	A FFHHH}}HC<1c#::::
#999
 ##---- :9
 .-r   c                     t          j        t          d                     t          d| d          fdt	          d          D                       \  }t          |          J d S )Ng     @@r3   r   r   c              3   R   K   | ]!} t          t                              V  "d S r"   )r   r'   )r   r   r.   s     r   r   z8test_memmap_returned_as_regular_array.<locals>.<genexpr>  sX       CE CE67+*++D11CE CE CE CE CE CEr   rE   )r   r   r   r   r   r   )r   resultr.   s     @r   %test_memmap_returned_as_regular_arrayr  |  s     73s88DBxq'cBBB CE CE CE CE;@88CE CE CE E EHVv&&.....r   c                    | dk    r3t           j                            d          rt          j        d           d                    |           }t          j        t           j        d|gt          j	        t          j	                  }|
                                 |                                \  }}|                                }|                                }|j        dk    sJ |d	z   |z               d
|vs
J |            d S )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rl   rm   r   r   resource_tracker)rs   platform
startswithpytestxfailrp   rq   rr   rt   ru   rv   rw   ry   rx   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr    s   " &S\44U;; 	,	
 	
 	

$ 	% & 	#.$4Z_ *	1 	1 	1AFFHHH}}HC
**,,C
**,,C<1cFlS0S(((#(((((r   c                 *   t          j        |j                  g k    sJ  | dd|j        d          }	 t          j        |j                  g k    sJ t           j                            |j                  rJ t          j        dt          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t          j        |j                  g k    sJ t          j        d
t          j                  j	        dk    sJ |
                    t          fdt          j        d	                   D                        t           j                            |j                  sJ t          j        |j                  }t!          |          dk    sJ t          j        dgd
z  d          }|
                    t$          |g          }|d	         rJ 	 |                                 t          d          D ]7}t)          d           t           j                            |j                  s n(8t+          d                    |j                            ~dS # |                                 t          d          D ]7}t)          d           t           j                            |j                  s n(8t+          d                    |j                            ~w xY w)z0Check that large arrays are not copied in memoryr=   (   r3   )r   r   verboser>   r<   r   c                     g | ]}|d fS r   r   )r   r   smalls     r   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>      KKKUAsOKKKr   r   r      c                     g | ]}|d fS r  r   r   r   larges     r   r   z9test_memmapping_pool_for_large_arrays.<locals>.<listcomp>  r  r   rE   abcobjectr   皙?z#temporary folder {} was not deletedN)r   r   rP   r   exists_temp_folderr   r   r   nbytesr   r1   r   r:   rR   isdirr   r&   r   r   r   r   rp   )	r   rZ   r{   dumped_filenamesobjectsresultsr   r  r  s	          @@r   %test_memmapping_pool_for_large_arraysr)    s    :fn%%++++ 	bfnaHHHA(z&.))R////7>>!.11111,,,|r!!!!	kKKKKU5;q>5J5JKKKLLL z&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL w}}Q^,,,,,:an55#$$)))) (E7S=999%%,wi881: 	
r 	 	A"III7>>!.11  !5<<Q^LL   AA 	
r 	 	A"III7>>!.11  !5<<Q^LL   s   GJ BLz,https://github.com/joblib/joblib/issues/1086)marksc                    d                     |           }t          j                                        }t          j                            t                    |d<   t          j        t          j
        d|gt          j        t          j        |          }|                                 |                                \  }}|                                |                                }}|                    d          d         }|j        dk    sJ |p|            |dk    sJ t          j                            |          rJ d S )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rl   r   
r    )rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   rv   rw   ry   splitrx   r"  )r   rz   r   r{   r|   r}   r[   s          r   &test_child_raises_parent_exits_cleanlyr/    s   :-Z 	[ \ *//

C11C#.$4Z_ *S	: 	: 	:AFFHHH}}HCzz||SZZ\\Cyyq!H<1cjS"9999w~~h'''''''r   c                     | dd|j                   }	 t          j        |j                   g k    sJ t          j        dt          j                  j        dk    sJ |                    t          fdt          j
        d                   D                        t          j        |j                   g k    sJ 	 |                                 ~dS # |                                 ~w xY w)	z2Check that large arrays memmapping can be disabledr=   Nr   r   r<   r  c                     g | ]}|d fS r  r   r  s     r   r   zBtest_memmapping_pool_for_large_arrays_disabled.<locals>.<listcomp>^  r  r   r   )rP   r   r   r   r   rR   r$  r   r1   r   r:   r   )r   rZ   r{   r  s      @r   .test_memmapping_pool_for_large_arrays_disabledr2  N  s     	d???A z&.))R//// 2:...|s""""	kKKKKU5;q>5J5JKKKLLL z&.))R///// 	
AA 	
s   B*C C/c                    t           j        }	 t          d          t           _         | dd          }	 |j        }d}|                    |          sJ t
          j                            |          sJ t          j	        dt          j
                  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d	k    sJ t          j	        dt          j
                  d
z  }|j        dk    sJ |                    t          |gdz             t          t          j        |                    d
k    sJ 	 |                                 ~n# |                                 ~w xY wt#          d          D ]2}t
          j                            |          s nt%          d           3t'          d          	 |t           _        dS # |t           _        w xY w)z1Check that memmapping uses /dev/shm when possibleg    ~Ar=   r   r   z"/dev/shm/joblib_memmapping_folder_r   r<   r  rE   r3   r!  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r#  r  r   r   r"  r   r   rR   r$  r   idr   r   r   r   r   r   )r   	orig_sizer{   r   folder_prefixr]   r^   r   s           r   'test_memmapping_on_large_enough_dev_shmr:  i  s    1I+6 -0II)GA"%%%	  !~@M#..}=====7>>"233333 2:...A8s????EE"qcBh rz"233449999
 2:...2A8s????EE"qcBhrz"2334499999 KKMMM KKMMMEEEEs 	M 	MA7>>"233 "IIII !KLLL 
 -6)))I)5555s$   &G0 D*E6  G0 6FAG0 0G>c                    t           j        }	 t          d          t           _         | dd          }	 |j        }|                    d          rJ 	 |                                 ~n# |                                 ~w xY wt          j                            |          rJ 	 |t           _        d S # |t           _        w xY w)Ng@6Dr=   r   r4  z/dev/shm)	r5  r6  r   r#  r  r   r   r   r"  )r   r8  r{   r   s       r   $test_memmapping_on_too_small_dev_shmr<    s     1I6 -0JJ)GA"%%%	  !~'22:>>>>>> KKMMM KKMMMEEEE 7>>"2333333,5)))I)5555s"   &B4 A* B4 *B$B4 4Cc                 v   t           j        j        } | dd|j                  }	 |                    t           j        d          }|                                }t          |          rJ  ||t          j        d                     |                                 ~dS # |                                 ~w xY w)z:Check that large arrays are not copied in memory in returnr=   r   r   )  r-   r>  N)	r   r*   r+   rP   apply_asyncr   getr   r   )r   rZ   r+   r{   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnrC    s     6 	bfn===AmmBG'm22		'.....5"'$--000	AA 	
s   A$B! !B8c                 .    t          |           sJ | |z  S )z4Multiplication function to be executed by subprocessr   )r]   n_timess     r   _worker_multiplyrF    s     """""w;r   c                    t           j        j        } | dd|j                  }	 t          j        t          j        d                              d          d          dddd	ddf         }|                    t          |df
          	                                }t          |          rJ  ||d|z             |                                 ~dS # |                                 ~w xY w)zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r=   r   r   ip  )r>  r3   r=   rA   )rB   NrE   r?  )r   r*   r+   rP   rV   rS   rU   r@  rF  rA  r   r   )r   rZ   r+   r{   r]   r^   s         r   6test_workaround_against_bad_memmap_with_copied_buffersrH    s     6bfn===A Jry..|<< " " ""#!!RaR(,
 MM*!QM88<<>>'*****1a!e$$$	AA 	
s   BC C.c                     | S r"   r   )args    r   identityrK    s    Jr   zfactory,retry_nor=   c                 @    g | ]\  }}d                      ||          S )z{}, {})rp   )r   rJ   ys      r   r   r     s6     	: 	: 	:41aA		 	: 	: 	:r   c                    |                     d          j        }dt          j        z  }t          j        dz   }t	          |d|d|          } | d|j                  }|                    t          |f	                                          }t          |t          j
                  sJ |j        |k    sJ t          j                            ||           |                                 d S )
Nr6   r>   rE   r7   r   )r;   r:   r9   rC   r3   r   r?  )rO   rP   r   r   r   r@  rK  rA  rX   r   rQ   rC   r*   r+   r   )	r   retry_norZ   r   r   rC   objr{   r
  s	            r    test_pool_memmap_with_big_offsetrR    s     KK$$,Et))D'!+F
e$d'#% % %C 	v~...A]]83&]115577Ffbi(((((=F""""J!!#v...KKMMMMMr   c                 &   d}t          || j                  \  }}|du sJ ||                     d          j        k    sJ t          |d           \  }}t          j                            d          r|du sJ |                    |          sJ d S )Ntest.tmpdirFrO  r  )r   rP   rO   rs   r  r  endswith)rZ   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrY    s    $+,<fnMMK&++m44<<<<<+,<$OOOK
|u%% #U"""" 01111111r   c                 &   d}ddl }t          |j        j        d          r |                    |j        j        d           t          |d          \  }}t          j                            d          r|du sJ |	                    |          sJ dS )z`Check that _get_temp_dir works when os.statvfs is not defined

    Regression test for #902
    rT  r   NstatvfsrO  r  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rs   r  r  rU  )rZ   monkeypatchrV  joblibrW  rX  s         r   !test_pool_get_temp_dir_no_statvfsrb  $  s    
 %%%%%v),i88 FF699EEE+,<$OOOK
|u%% #U"""" 01111111r   z1This test fails with a PermissionError on Windows	mmap_moder@   r7   c                    d d t          d          D             } t          | dd          fdt          |          D                       }t          |          D ]%\  }}t          j                            ||           &d S )Nc                     || d d <   | S r"   r   )arrvalues     r   funcz4test_numpy_arrays_use_different_memory.<locals>.func:  s    AAA
r   c                 :    g | ]}t          j        d d          S ))r   r   rR   r<   )r   zeros)r   r   s     r   r   z:test_numpy_arrays_use_different_memory.<locals>.<listcomp>>  s'    EEEabhxy111EEEr   r   r   r3   )rc  r   r   c              3   P   K   | ] \  }} t                    ||          V  !d S r"   r   )r   r   rf  rh  s      r   r   z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>@  sR       D? D?"(!Sc1D? D? D? D? D? D?r   )r   r   	enumerater   r*   r+   )rc  arraysr(  r   rf  rh  s        @r   &test_numpy_arrays_use_different_memoryro  5  s    
   FE599EEEFChqCCC D? D? D? D?,5f,=,=D? D? D? ? ?G G$$ . .3

%%c1----. .r   c                     dd} t          j        d          }t                                          |d                               |          dk    sJ |}                    |          dk    sJ                     |d                               |          dk    sJ ~t          j                     t          j                  dk    sJ                     |          dk    sJ ~ | j                   t          j        d          }                    |d           t          j                  dk    sJ                     |          dk    sJ t          t                    5                      t          j        d                     d d d            n# 1 swxY w Y   ~ | j                   d t          fd	t          d
          D                       }t          j                    dk    r4t          t          j        dd          rdnd}t          |          |k     sJ d S d S )Nr   c                     t          |          D ]:}t          |           dk    r d S t          j                     t	          d           ;t          |           dk    sJ d S )Nr   r!  )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collectz>test_weak_array_key_map.<locals>.assert_empty_after_gc_collectJ  sd    w 	 	A9~~""JLLL"IIII9~~""""""r   *   r]   r^   rE   rG   c                 ,   t          j        d          }t          t                    5  |                     |           d d d            n# 1 swxY w Y   |                     ||           |                     |          |k    sJ t          |          S )Nrw  )r   r   r   KeyErrorrA  r   r7  )mr   r]   s      r   get_set_get_collectz4test_weak_array_key_map.<locals>.get_set_get_collectq  s    GBKKH 	 	EE!HHH	 	 	 	 	 	 	 	 	 	 	 	 	 	 		auuQxx1}}}}!uus   AAAc                 (    g | ]} |          S r   r   )r   r   r{  rz  s     r   r   z+test_weak_array_key_map.<locals>.<listcomp>y  s'    EEEA))!Q//EEEr   r>  CPythonnogilFi  )r   )r   r   r   r   rA  rr  rs  r   _datar   ry  r   r  python_implementationr   rs   flags)rv  r]   r^   rG   
unique_idsmax_len_unique_idsr{  rz  s         @@r   test_weak_array_key_mapr  G  sl   # # # # 	AAEE!SMMM5588s????	A5588s????EE!SMMM5588s????	JLLLqw<<15588s????	!!!'***
AEE!SMMMqw<<15588s????			  	bgbkk               	
!!!'***   EEEEEtEEEFFJ%''944 %,CIw$F$FOSSC:!33333 54 43s   5(F))F-0F-c                      t                      } t          t          j                  5  t          j        |            d d d            d S # 1 swxY w Y   d S r"   )r   r   picklePicklingErrordumps)rz  s    r   #test_weak_array_key_map_no_picklingr    s    A	$	%	%  Q                 s   A

AAc                 ^   t          |                     d                    t          j        dd          }|                               fdd              } t          d          fd	|fD                       } t          d          fd
|fD                       }t          j                            ||           fd t          d          fdt          d          D                       }t          j                            |d         |           d S )Nzarr.datr   r   r<   c                      t                    5 } t          j        |                                 dt          j        d          }d d d            n# 1 swxY w Y   t	          j        dt          j        |d          S )Nr   )accessrC   r   )r9   r\   rC   )openr   filenoACCESS_READr   rY   r   )fdmmtestfiles     r   _read_arrayz%test_direct_mmap.<locals>._read_array  s    (^^ 	Nr299;;$2B1MMMB	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	Nz%rx1EEEEs   5AAAc                     | dz  S )Nr3   r   )rJ   s    r   rh  ztest_direct_mmap.<locals>.func  s    !tr   r3   )r   c              3   H   K   | ]} t                    |          V  d S r"   rl  r   rJ   rh  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s3      ;;!]WT]]1--;;;;;;r   c              3   H   K   | ]} t                    |          V  d S r"   rl  r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s3       A Aaq!1!1 A A A A A Ar   c                                    S r"   r   )r  s   r   workerz test_direct_mmap.<locals>.worker  s    {}}r   c              3   F   K   | ]} t                                V  d S r"   rl  )r   r   r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s1       E Eq!2!2 E E E E E Er   rE   r   )	strrO   r   rS   tofiler   r*   r+   r   )	rZ   r]   rf  refr(  r  rh  r  r  s	        @@@@r   test_direct_mmapr    sq    6;;y))**H
	"G$$$AHHXF F F F F
   +--C (!


;;;;s;;;
;
;C !ha    A A A AC5 A A AAAGJ!!'3///     !ha    E E E EE!HH E E EEEGJ!!'!*c22222r   )Vr   r   rs   r  rr  r  	itertoolstimer   rq   r   r   r  joblib.test.commonr   r   r   r   joblib.testingr   r	   r
   joblib.backportsr   joblib.parallelr   r   joblib.poolr   joblib.executorr   TestExecutorr\  r   r   r   r   r   r   r^  r5  r    r$   r'   r1   r4   rh   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r  r  r  r  r)  parammarkr  r/  r2  r:  r<  rC  rF  rH  rK  listproductr   r   r  rR  rY  rb  ro  r  r  r  r   r   r   <module>r     s
   				  



  				                         - - - - - - - - 3 3 3 3 3 3 + + + + + + 6 6 6 6 6 6 6 6 6 6 ( ( ( ( ( ( - - - - - - - - & & & & & & F F F F F F ; ; ; ; ; ; @ @ @ @ @ @ ; ; ; ; ; ; 4 4 4 4 4 4 7 7 7 7 7 7 : : : : : : ( ( ( ( ( (> > >/ / /  
< < <@ @ @ I- I-  I-X 	 'RCE E E) )E E )X *+ *+  *+Z C C C* Y)MN#V,. . ./ /. .  /d Y)MN#V,. . .   . .   F Y*F344C C 54  C@ Y*F344,2 ,2 54  ,2^ Y*F344
F 
F 54  
F Y*F344
F 
F 54  
F #4 #4  #4L C< C<  C<L Y*F344*. *. 54  *.Z Y*F344/ / 54  / Y*F3441. 1. 54  1.h Y)MN#V,. . .1 1. .  1h +##E $  	
 	
 	
 	 G( G(   G(T Y)MN#V,. . . . .  . Y)MN#V,. . ..6 .6. .   
.6b Y)MN#V,. . .6 6. .   
60 Y)MN#V,. . . . .  ,   Y)MN#V,. . . . .  2   D			=>a
J 
J K K	: 	:+<9+<	F#SSeeAhh%7%7,9 ,9 	: 	: 	:	; ; ; ; ;  &	2 	2 	22 2 2"  )% & & &[4,''. . ('& & . 94 94 94x   3 3  3 3 3r   