U
    |hG                     @   s&  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZmZ dd Z dVddZ!dd Z"dd Z#dd Z$dd Z%dd Z&ej'(ddd gd!d" Z)d#d$ Z*G d%d& d&e+Z,dWd'd(Z-d)d* Z.d+d, Z/ej'(d-e0e1dd.d/ Z2d0d1 Z3d2d3 Z4d4d5 Z5d6d7 Z6d8d9 Z7d:d; Z8d<d= Z9d>d? Z:d@dA Z;dBdC Z<dDdE Z=ej'(dFdGdHgej'j>ej?dIkoej?dJkdKdLdMdN Z@dOdP ZAej'(dQdRdSgdTdU ZBdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_config)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterinccleanupc                  O   s   d S N )argskwargsr   r   9/tmp/pip-unpacked-wheel-9qhk7zno/joblib/test/test_dask.pynoop   s    r   皙?c                 C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s    r#   c                    sF   | dd }i }| D ]&\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>$       zcount_events.<locals>.<lambda>c                    s   g | ]}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>'   s    z count_events.<locals>.<listcomp>)runitemslenlist)r,   clientZworker_eventsZevent_countsweventsr   r+   r   count_events#   s
     r5   c                 C   s   t  \}\}}t|d | d}tdd t dd tdD }|dd	 tdD ks`ttt t d
d tdD  W 5 Q R X t dd tdD }|dd	 tdD kstW 5 Q R X W 5 Q R X W 5 Q R X d S )Naddressloopr   backendc                 s   s   | ]}t t|V  qd S r   r   r   r)   ir   r   r   	<genexpr>0   s     ztest_simple.<locals>.<genexpr>
   c                 S   s   g | ]}t |qS r   r   r<   r   r   r   r-   1   s     ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t t|d kV  qdS )   N)r   r#   r<   r   r   r   r>   4   s   c                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>   7   s     c                 S   s   g | ]}t |qS r   r@   r<   r   r   r   r-   8   s     )	r   r   r   r
   rangeAssertionErrorpytestraisesr    r8   sabr2   seqr   r   r   test_simple,   s    
rK   c                 C   s   t jtjkstt \}\}}t|d | d}tddp t ^}|j}t	|t sZt|j
|ksht|jdksvt|dd ttdD  |jd	kstW 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )
Nr6   r7   r   r9   r(   c                 s   s   | ]}t d d  V  qdS )c                   S   s   d S r   r   r   r   r   r   r%   M   r&   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r)   _r   r   r   r>   L   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @r?   )r   Zcompute_batch_sizer   rC   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizerB   int)r8   rG   rH   rI   r2   rR   r:   r   r   r   #test_dask_backend_uses_autobatching;   s    
rT   c                   C   s   t  S r   r   r   r   r   r   random2S   s    rU   c                 C   sx   t  h\}\}}t|d | dD}tdd. t dd tdD \}}||ksVtW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr6   r7   r   r9   c                 s   s   | ]}t t V  qd S r   )r   rU   r<   r   r   r   r>   [   s     z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   rB   rC   )r8   rG   rH   rI   r2   xyr   r   r    test_dont_assume_function_purityW   s
    rY   mixedTFc                 C   s   ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRtt \}\}}t|d	 | d
`}tdd tddd|}	W 5 Q R X dd }
|	dd}|
|
}tdd |D stW 5 Q R X W 5 Q R X d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r;   r<   r   r   r   r-   c   s     z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt t|n
t t|qS )r	   )r   absr   r<   r   r   r   r-   f   s    Zmixed_batch_of_inc_4_callsr6   r7   r   r9   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r1   Ztransition_log)Zdask_schedulerr   r   r   fr   s    ztest_dask_funcname.<locals>.f42c                 s   s   | ]}d |d kV  qdS )Zbatch_of_incr   Nr   )r)   tupr   r   r   r>   v   s     z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskr[   rB   reprrC   r   r   r   r
   replaceZrun_on_schedulerr^   )r8   rZ   r[   tasksZ
batch_reprrG   rH   rI   r2   rN   r_   r$   r   r   r   test_dask_funcname_   s"    
rf   c               	      sV  dd t dD } tdtd d fdd	tdd	d
}t|}ztdd t	 fdd| D }W 5 Q R X | dd t dD kst
td|}t| dkst
tdd |D st
tdd  t	  fdd| D }W 5 Q R X td|}t| dkst
tdd |D s4t
W 5 |jdd |jdd X d S )Nc                 S   s   g | ]}g qS r   r   rM   r   r   r   r-      s     z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d k	rj |  | t j | S r   )ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr(   r	   )	n_workersthreads_per_worker   timeoutr   r9   c                 3   s   | ]}t  |V  qd S r   rL   r)   rk   )ro   r   r   r>      s    z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g qS r   r   rM   r   r   r   r-      s     receive-from-scatterr   c                 S   s   g | ]}t |d kqS r'   r0   r)   rr   r   r   r-      s     c                 3   s   | ]}t | d V  qdS ))rl   NrL   ru   )rm   ro   r   r   r>      s    c                 S   s   g | ]}t |d kqS r'   rw   rx   r   r   r   r-      s     )N)rB   rD   importorskipZarangerS   r   r   closer   r
   rC   r5   sumvaluesr^   )listsr   r2   rescountsr   )rm   ro   rn   r   'test_no_undesired_distributed_cache_hity   s0    


r   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )rW   count)selfrW   r   r   r   __init__   s    zCountSerialized.__init__c                 C   s   | j t|d| S )NrW   )rW   getattr)r   otherr   r   r   __add__   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr(   )r   r   rW   )r   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r      s   r   c                 C   s   | | | | | S r   r   )rH   rI   cder   r   r   add5   s    r   c                 C   s@  t d}t d}t d}t \}\}}t|d | d}td||gdj tt}||||dd	d
||||d	dd
|||||d	d
||||||d
g}	dd |	D }
t |	}W 5 Q R X tt	  td| dd W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X ||
kst
|jdkst
|jdks,t
|jdks<t
d S )Nr(   r	   rA   r6   r7   r   )r:   scatterr\      )r   r   c                 S   s   g | ]\}}}|||qS r   r   )r)   funcr   r   r   r   r   r-      s   z'test_manual_scatter.<locals>.<listcomp>)r:   r8   r   )r\      )r   r   r   r   r   r   r
   rD   rE   	TypeErrorrC   r   )r8   rW   rX   zrG   rH   rI   r2   r_   re   expectedresultsr   r   r   test_manual_scatter   s.    (r   c           	         sv  t d}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d^}tdd t d	d
 t	|D  W 5 Q R X t
d|}||d  ||d   dkstW 5 Q R X W 5 Q R X t \}\}}t|d | dn}tdd" t  fdd
tdD  W 5 Q R X t
d|}||d  dksHt||d  dks^tW 5 Q R X W 5 Q R X d S )Nrh   rO   )ZdtyperA   r6   r7   r   r9   c                 s   s&   | ]\}}t t||||d V  qdS ))optNr   r   )r)   r=   rl   r   r   r   r>      s   z$test_auto_scatter.<locals>.<genexpr>rv   r	   c                 3   s$   | ]}t t d d |V  qd S )NrA   r   r<   Zdata1r   r   r>      s     r   r   )rD   rz   onesrS   Zuint8r   r   r   r
   	enumerater5   rC   rB   )	loop_in_threadrn   Zdata2Zdata_to_processrG   rH   rI   r2   r   r   r   r   test_auto_scatter   s&    


4&
r   retry_noc                    s   t dd d}fdd fddt h\}\}}t|d | dD}td	d
. dt fddt|D }W 5 Q R X W 5 Q R X W 5 Q R X d S )Nrh   r?   c                    s
     | S r   )r|   )rW   r=   j)rn   r   r   my_sum  s    z#test_nested_scatter.<locals>.my_sumc              	      sD   t  }tdd& t  fddtD }W 5 Q R X t|S )Nr   r9   c                 3   s&   | ]}t  |d  |V  qd S r   rL   )r)   r   )arrayr=   r   r   r   r>     s    zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rB   r|   )r   r=   r2   r   )NUM_INNER_TASKSr   )r   r=   r   outer_function_joblib  s    z2test_nested_scatter.<locals>.outer_function_joblibr6   r7   r   r9   i'  c                 3   s$   | ]}t  |d  |V  qd S r   rL   r<   )my_arrayr   r   r   r>      s
   
 z&test_nested_scatter.<locals>.<genexpr>)rD   rz   r   r   r   r   r
   rB   )r8   r   ZNUM_OUTER_TASKSrG   rH   rI   rN   r   )r   r   r   rn   r   r   test_nested_scatter  s    
	
r   c                    s  dd  t  \}\}}t|d | dZ}tddD tdd fd	d
tdD }|D ]}tt|dksZtqZW 5 Q R X W 5 Q R X t|d | dZ}tddD tdd fdd
tdD }|D ]}tt|dkstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s   s   | ]}t tj V  qd S r   r   osgetpidrM   r   r   r   r>   (  s     zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tj V  qd S r   r   rM   r   r   r   r>   )  s     )setr
   rB   )Zpidsr   r   r   get_nested_pids'  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr6   r7   r   r9   r	   r   c                 3   s   | ]}t   V  qd S r   rL   rM   r   r   r   r>   /  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r?   c                 3   s   | ]}t   V  qd S r   rL   rM   r   r   r   r>   9  s   )r   r   r   r
   rB   r0   r   rC   )r   rG   rH   rI   r2   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager&  s     *r   c           	         s   dd   fddt  \}\}}t|d | d}tddv t >} |d	ksZt|jd
ksht|fddtdD }W 5 Q R X |D ] \}}|d	kst|d
kstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                 S   s
   | j jjS r   )rP   	__class__r   pr   r   r   _backend_typeE  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec               
      s,   t  }  | | jfW  5 Q R  S Q R X d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsH  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr6   r7   r   r9   r   c                 3   s   | ]}t   V  qd S r   rL   rM   )r   r   r   r>   R  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   rC   r   rB   )	r8   rG   rH   rI   r2   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobsA  s    r   c              
   C   sF   t t}tdd W 5 Q R X W 5 Q R X dt|j ksBtd S )Nr   r9   zcreate a dask client)rD   rE   r    r   strvaluelowerrC   )r8   infor   r   r   test_errors[  s    r   c                 C   s   t  \}\}}t|d | d}tdd< tdddd td	D }t|d
 d
 d
 tsdtW 5 Q R X tdd< tdddd td	D }t|d
 d
 d
 tstW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr6   r7   r   r9   r	   r   c                 s   s   | ]}t td dV  qd S )NZnested_requirer   outerrM   r   r   r   r>   h  s    z.test_correct_nested_backend.<locals>.<genexpr>r(   r   c                 s   s   | ]}t td dV  qdS )Z	sharedmemr   Nr   rM   r   r   r   r>   n  s   )	r   r   r   r
   rB   rQ   r   rC   r   )r8   rG   rH   rI   r2   resultr   r   r   test_correct_nested_backendc  s    $r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s   | ]}t t V  qd S r   )r   middlerM   r   r   r   r>   u  s    zouter.<locals>.<genexpr>r(   r
   rB   r   r   r   r   r   t  s    r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   s   | ]}t t V  qd S r   )r   innerrM   r   r   r   r>   {  s    zmiddle.<locals>.<genexpr>r(   r   )r   r   r   r   r   z  s    r   c                   C   s   t  jS r   )r
   rP   r   r   r   r   r     s    r   c              
   C   sP   t | ddd8 tdd" tdddd	 td
D  W 5 Q R X W 5 Q R X d S )NFT)r8   	processesZset_as_defaultr   r9   r\   r   c                 s   s   | ]}t t|V  qd S r   )r   idr<   r   r   r   r>     s     z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rB   r7   r   r   r   test_secede_with_no_processes  s    r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r6   )rN   r   r   r   r   _worker_address  s    r   c                 C   s   t  \}\}}t|d | d}td|d d4 t dd tdD }||d gd ksbtW 5 Q R X td|d d4 t dd tdD }||d gd kstW 5 Q R X W 5 Q R X W 5 Q R X d S )	Nr6   r7   r   )r:   workersc                 s   s   | ]}t t|V  qd S r   r   r   r<   r   r   r   r>     s    z-test_dask_backend_keywords.<locals>.<genexpr>r?   c                 s   s   | ]}t t|V  qd S r   r   r<   r   r   r   r>     s    rV   rF   r   r   r   test_dask_backend_keywords  s    
 
r   c              
   C   s   t d| dj}tdd t dd tdD  W 5 Q R X t }|jjjrhtd t |d	 k sBt	qB|j
rrt	W 5 Q R X d S )
NF)r   r8   r   r9   c                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>     s     z/test_scheduler_tasks_cleanup.<locals>.<genexpr>r?   g{Gz?r   )r   r   r
   rB   r   r   Z	schedulerre   r   rC   Zfutures)r8   r2   startr   r   r   test_scheduler_tasks_cleanup  s    "
r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc              	   C   s   t dddd}t|}| dkr.|jddd n| dkr@|d z2tdd	 t d
d tdD  W 5 Q R X W 5 |  |  X d S )Nr   Fr	   rp   r   rq   r   )Zminimummaximumr   r   r9   c                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>     s     z(test_wait_for_workers.<locals>.<genexpr>r?   )r   r   ZadaptZscaler{   r   r
   rB   )r   r   r2   r   r   r   test_wait_for_workers  s    
&r   c               
   C   s   t dddd} t| }ztddd< d}tjt|d	 t d
d tdD  W 5 Q R X W 5 Q R X tddd< d}tjt	|d	 t dd tdD  W 5 Q R X W 5 Q R X W 5 |  |   X d S )Nr   Fr	   r   r   g?)r:   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>     s     z0test_wait_for_workers_timeout.<locals>.<genexpr>r?   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>     s     )
r   r   r{   r   rD   rE   TimeoutErrorr
   rB   RuntimeError)r   r2   msgr   r   r   test_wait_for_workers_timeout  s    ,0r   r:   Zlokymultiprocessingc              	      s   t dd}t|}zZ fdd}||}| }t|dksBt|d j}t|t	sZtd	t
|ksjtW 5 |jdd |jdd X d S )
Nr	   )rp   rr   rs   c               	      s:   t jdd$} td ddd tdD  W 5 Q R X | S )NTrecordr	   )r   r:   c                 s   s   | ]}t t|V  qd S r   r;   r<   r   r   r   r>     s    zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>r?   )warningscatch_warningsr
   rB   r   r9   r   r   func_using_joblib_parallel  s
    zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr(   r   zdistributed.worker.daemon)r   r   r{   Zsubmitr   r0   rC   messagerQ   UserWarningr   )r:   r   r2   r   Zfutr   warningr   r9   r   /test_joblib_warning_inside_dask_daemonic_worker  s    


r   )r   )r   r   )C
__future__r   r   r   r   r   rD   r   uuidr   r   r    r
   r   r   rR   r   r   Z_daskr   rz   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r   r#   r5   rK   rT   rU   rY   markZparametrizerf   r   objectr   r   r   r   r1   rB   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   r   <module>   sd   


	
9
%
