o
    ?ez                     @   s  d Z ddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlm!Z! ddl"m#Z# ddl$m%Z% ddl$m&Z& ddl$m'Z' ddl$m(Z( ddl$m)Z) ddl$m*Z+ ddl$m,Z, ddl$m-Z- ddl$m.Z. ddl/m0Z0 dd l1m2Z2 dd!l1m3Z3 dd"l4m5Z5 dd#l4m6Z6 G d$d% d%e7Z8dOd'd(Z9dPd)d*Z:d+d, Z;d-d. Z<d/d0 Z=d1d2 Z>d3d4 Z?d5d6 Z@d7d8 ZAd9d: ZBd;ejCd<eDfd=d>ZEd;ejCd<eDfd?d@ZFd;ejCd<eDfdAdBZGG dCdD dDejHZIG dEdF dFejHZJG dGdH dHejHZKG dIdJ dJeIZLdKdL ZMdMdN ZNdS )Qz5Library for testing DistributionStrategy descendants.    N)
config_pb2)	event_pb2session)dataset_ops)collective_all_reduce_strategy)distribute_lib)distribute_utils)mirrored_strategy)reduce_util)tpu_strategy)backprop)context)def_function)test)dtypes)errors)ops)	test_util)	tf_record)	array_ops)gradients_impl)init_ops)init_ops_v2)math_ops)summary_ops_v2)variable_scope)variable_v1)	variables)gfile)	optimizer)training_util)nest)
tf_inspectc                   @   s   e Zd ZdS )_TestExceptionN)__name__
__module____qualname__ r(   r(   o/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/distribute/strategy_test_lib.pyr$   :   s    r$   Fc                 C   s   |rt  s| S t | S N)r   executing_eagerlyr   function)fnrun_in_functionr(   r(   r)   _maybe_run_in_function@   s   r/   c                 C   s   t  r*   )r$   _r(   r(   r)   _raise_exception_fnI   s   r2   c                   C      t  t d S r*   )r   get_replica_context
merge_callr2   r(   r(   r(   r)   _merge_raises_fnO      r6   c                 C      | j t d S r*   )extendedcall_for_each_replicar2   distr(   r(   r)   _call_raises_fnV      r=   c                   C   r3   r*   )r   r4   r5   r=   r(   r(   r(   r)   _merge_call_raises_fn]   r7   r?   c                 C   r8   r*   )r9   r:   r6   r;   r(   r(   r)   _call_merge_raises_fnd   r>   r@   c                   C   r3   r*   )r   r4   r5   r@   r(   r(   r(   r)   _merge_call_merge_raises_fnl   r7   rA   c                 C   sp   |  t| t|}| |d tttj	
||d }g }|D ]}t }|| || q%|S )z(Reads summary events from log directory.   r   )
assertTruer   ZExistsZListDirectory	assertLenlistr   Ztf_record_iteratorospathjoinr   EventZParseFromStringappend)Z	test_caselogdirfilesrecordsresultreventr(   r(   r)   _events_from_logdirp   s   

rQ   c                 C   s$   t jt ||d}tj|| ddS )zFUtitlity for create variables that works like variable in keras layer.)dtypeT)initial_valuename	trainable)	functoolspartialr   ZGlorotUniformr   Variable)rT   shaperR   initializerr(   r(   r)    create_variable_like_keras_layer~   s   
r[   c                 C   s&   t | j}d|jd t|j  v S )NZvar_list)r#   getfullargspecZminimizeargslendefaults)Zoptimizer_objZarg_specr(   r(   r)   is_optimizer_v2_instance   s   r`   strategyreturnc                 C      t | tjtjfS r*   )
isinstancemirrored_libZMirroredStrategyZMirroredStrategyV1ra   r(   r(   r)   is_mirrored_strategy   s   
rg   c                 C   rc   r*   )rd   mwms_libZCollectiveAllReduceStrategyZCollectiveAllReduceStrategyV1rf   r(   r(   r)   !is_multi_worker_mirrored_strategy   s   ri   c                 C   s   t | tjtjtjfS r*   )rd   r   ZTPUStrategyZTPUStrategyV1ZTPUStrategyV2rf   r(   r(   r)   is_tpu_strategy   s
   rj   c                   @   s|   e Zd ZdZdd Z		dddZdd	 Zd
d Zdd Zdd Z		dddZ
			dddZdd ZdddZdd ZdS ) DistributionTestBasez:Some tests that should work with any DistributionStrategy.c                    s      a tddtjdfdd}t|tdd t	dgg fd	d
}t
dD ]}| \}}|dkrC|\}|\}q5t| d }	t| d }
| |
|	 W d    d S 1 shw   Y  d S )NkernelrB   rB   rT   rY   rR   c                    &   t t|  g t d }|| S N      ?r   Zreshaper   Zmat_mulidentityxyrl   r(   r)   loss      z<DistributionTestBase._test_minimize_loss_eager.<locals>.lossc                 S   s   |  d| S )N皙?Z
assign_subvgr(   r(   r)   update      z>DistributionTestBase._test_minimize_loss_eager.<locals>.updaterq   c               
          j jfd} g }g }| D ]W\}} j |}|| t|g:  j jtjj	||d}t j j
||fdd | j | W d   n1 sRw   Y  W d   n1 saw   Y  q||fS zPerform one optimization step.r]   )destinationsF)r]   groupNr9   r:   Zread_varrJ   r   Zcontrol_dependenciesZ	reduce_tor   ReduceOpSUMr   Zg_vZbefore_listZ
after_listr~   r}   Zfetcheddgrad_fnoner   r(   r)   step   s&   

z<DistributionTestBase._test_minimize_loss_eager.<locals>.step
   r   rB   )scoper[   r   float32r   implicit_gradr    Zget_filtered_grad_fnr   rs   rangeabsnumpy
assertLess)selfr   rx   r   ibabeforeaftererror_beforeerror_afterr(   )r   r   rl   r   r   r)   _test_minimize_loss_eager   s&   



"z.DistributionTestBase._test_minimize_loss_eagerFrz   c                    s  t  }||_d|j_t  t 	  | j
|d|}  h tddtjdfdd}t|fdd	td
gg fdd}| \}}	t   tdD ]}
|||	f\}}|
dkrm|\}|\}q[t|d }t|d }| || W d    n1 sw   Y  W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Ng333333?)configrl   rm   rn   c                    ro   rp   rr   rt   rw   r(   r)   rx      ry   z<DistributionTestBase._test_minimize_loss_graph.<locals>.lossc                    s   |   | S r*   r{   r|   learning_rater(   r)   r      r   z>DistributionTestBase._test_minimize_loss_graph.<locals>.updaterq   c               
      r   r   r   r   r   r(   r)   r      s&   

z<DistributionTestBase._test_minimize_loss_graph.<locals>.stepr   r   rB   )r   ZConfigProtoZallow_soft_placementZgpu_optionsZper_process_gpu_memory_fractionr   Z
graph_moder   ZGraph
as_defaultcached_sessionr   r[   r   r   r   r   r   rs   r   global_variables_initializerrunr   r   r   )r   r   Zsoft_placementr   r   sessrx   r   Z
before_outZ	after_outr   r   r   r   r   r   r   r(   )r   r   rl   r   r   r   r)   _test_minimize_loss_graph   sD   




 P z.DistributionTestBase._test_minimize_loss_graphc           	   
      s^  t  } fdd}|  }| z t f t }t	 s&|j
  td t| |j|}||}t	 sR|   || |   t| |}| |d | |d jjd jd | |d jjd jd W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nc                     sB       t j} td| W  d   S 1 sw   Y  dS )z#Function executed for each replica.r   N)r   r   r4   replica_id_in_sync_groupsummary_opswriteZ
replica_idZsummary_writerr(   r)   run_fn  s   


$zHDistributionTestBase._test_summary_for_replica_zero_only.<locals>.run_fnr      rB   r   g        )tempfilemkdtempr   r   r   Zalways_record_summariesr!   Zget_or_create_global_stepr   r+   rZ   r   Zset_stepZcreate_file_writerr9   r:   unwrapinitcloserQ   rD   assertEqualsummaryvaluetagZsimple_value)	r   r   rK   r   r   global_stepoutputZ	unwrappedeventsr(   r   r)   #_test_summary_for_replica_zero_only  s.   





P z8DistributionTestBase._test_summary_for_replica_zero_onlyc                    sv      - dgt jj  fdd} j| dgt jj  W d    d S 1 s4w   Y  d S )NFc                     s>    t j} | t jj |   d| < d S )NT)	evaluater   r4   r   r   r^   r9   worker_devicesZassertFalser   r   Zexpected_devicesr   r(   r)   mark_devices_fn3  s   z>DistributionTestBase._test_replica_id.<locals>.mark_devices_fnT)r   r^   r9   r   r:   assertAllEqual)r   r   r   r(   r   r)   _test_replica_id/  s   
"z%DistributionTestBase._test_replica_idc              	   C   s  |  } | t |jt W d    n1 sw   Y  | t |jt W d    n1 s6w   Y  | t |jt W d    n1 sQw   Y  | t |jt W d    n1 slw   Y  W d    d S W d    d S 1 sw   Y  d S r*   )	r   assertRaisesr$   r9   r:   r2   r6   r?   rA   )r   r<   r(   r(   r)   _test_call_and_merge_exceptions>  s    
"z4DistributionTestBase._test_call_and_merge_exceptionsc                    s    dg fdd}|S )Nr   c                    sj    |  | j | j dur!| j   S d | j d  d7  <   S )zInput fn for testing.Nr   rB   )ZassertIsNotNoner   num_replicas_in_syncZnum_input_pipelinesZinput_pipeline_id)Zinput_contextdataset_or_callable_fnexpected_input_pipeline_idexpected_num_input_pipelinesexpected_num_replicas_in_syncr   Zworker_id_counterr(   r)   	_input_fnS  s   
zGDistributionTestBase._input_fn_to_test_input_context.<locals>._input_fnr(   )r   r   r   r   r   r   r(   r   r)   _input_fn_to_test_input_contextI  s   z4DistributionTestBase._input_fn_to_test_input_contextc           
      C   s   |r| j n| j}||}t rft|}|D ]}| t|t	|}	|||	 q| 
t | |t	| W d    n1 sEw   Y  t|}|D ]}| t|t	|}	|||	 qPd S t|}| j||jj|d|d d S )NTtest_reinitializeignore_order)assertCountEqualr   Z!distribute_datasets_from_functionr   r+   iterr   rE   experimental_local_resultsnextr   StopIterationr   Zmake_initializable_iterator_test_input_fn_iteratorr9   r   )
r   ra   input_fnexpected_valuesr   Zassert_sameiterableiteratorexpected_valuecomputed_valuer(   r(   r)   _test_input_fn_iterablee  s0   


z,DistributionTestBase._test_input_fn_iterableNTc           
         s  fdd}||j  |D ]$}|  | fddtt|D }	|r,||	 q||	 qtj |  | fddtt|D  W d    n1 sWw   Y  |r||j  |D ]&}|  | fddtt|D }	|r||	 qe||	 qed S d S )Nc                    s   r | S  | S r*   )r   r   ru   )r   r   r(   r)   <lambda>      z>DistributionTestBase._test_input_fn_iterator.<locals>.<lambda>c                       g | ]}t | qS r(   r	   Zselect_replica.0rO   next_elementr(   r)   
<listcomp>  r   z@DistributionTestBase._test_input_fn_iterator.<locals>.<listcomp>c                    r   r(   r   r   r   r(   r)   r     r   c                    r   r(   r   r   r   r(   r)   r     s    )	rZ   get_nextr   r^   r   r   r   r   OutOfRangeError)
r   r   Zdevicesr   r   r   r   r   r   r   r(   )r   r   r   r)   r     s@   






z,DistributionTestBase._test_input_fn_iteratorc              	      s   |  J tjdg tjt dtjj	d | 
t   fdd}|j|\}}| 
|| ||}| 
|}| dt| | W d    d S 1 sQw   Y  d S )Nr   F)rY   rR   rZ   rU   Zaggregationc                     s     d}   }| |fS NrB   )Z
assign_addZ
read_value)Ztrain_opr   r   r(   r)   model_fn  s   
z?DistributionTestBase._test_global_step_update.<locals>.model_fn)rB   )r   r   Zget_variabler   Zint64r   Zzeros_initializerr   ZVariableAggregationZONLY_FIRST_REPLICAr   r   r9   r:   r   r   r   r^   )r   ra   r   Z	train_opsr   Zglobal_step_tensorsZglobal_step_valuesr(   r   r)   _test_global_step_update  s"   


"z-DistributionTestBase._test_global_step_updatec                    s  t |tjsd |p }|  |}tddgddgddgddgd	dgdd
gg}tg d}d}|jj	sB||j
 }|jj||f|pM d}	|	d}	|jj}
|	j||
d}	||	}|j  fdd}|||\}}|| || |||\}}|| || tj ||| W d    n1 sw   Y  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nzn/a: V1 onlyrB   r               r         )r   r   r   r   rB   r   r   )drop_remainderc                    sL   |  tdd  |\}}| || |f\}}t|t|fS )Nc                 S   s   | S r*   r(   )zr(   r(   r)   r     s    zWDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate.<locals>.<lambda>)experimental_runr/   r   r   npZconcatenate)ra   r   ru   rv   r.   r   r(   r)   run_and_concatenate  s   
zEDistributionTestBase._test_numpy_dataset.<locals>.run_and_concatenate)rd   r   Z
StrategyV1skipTestr   r   r   Zasarrayr9   Z_global_batch_sizer   Zexperimental_make_numpy_datasetrepeatZ"experimental_require_static_shapesbatchZmake_dataset_iteratorr   rZ   r   r   r   r   )r   ra   r   r.   r   r   ru   rv   Z
batch_sizeZdsr   r   r   Zx_1Zy_1Zx_2Zy_2r(   r   r)   _test_numpy_dataset  s:   
.


Pz(DistributionTestBase._test_numpy_datasetc              	   C   s   t jtjfD ]P}| B |d}| d|j |dtjjd}| d|j |dtjjdd}| d|j |dtjjdd}| d|j W d    n1 sQw   Y  qd S )Nrq   T)synchronizationF)r   rU   )	r   Z
VariableV1r   rX   r   r   rU   ZVariableSynchronizationZON_READ)r   ra   clsv1v2Zv3Zv4r(   r(   r)   _test_trainable_variable  s"   


z-DistributionTestBase._test_trainable_variable)Frz   F)NTF)NF)r%   r&   r'   __doc__r   r   r   r   r   r   r   r   r   r   r  r(   r(   r(   r)   rk      s&    3
:"
!
)
&rk   c                   @   sp   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )OneDeviceDistributionTestBasezESome tests that should work with any one-device DistributionStrategy.c                 C   s   | dd }| dg| || |j dd |fd}| t|j|}| dg|d  | dg|d	  |j d
d |d}| dg| || d S )Nc                   S   s
   t dS )N      @)r   rs   r(   r(   r(   r)   r     s   
 z9OneDeviceDistributionTestBase._test_run.<locals>.<lambda>r  c                 S      | d | |  dS Nr   )r   r   r(   r   r(   r(   r)   r          r   g       @r   g      0@r   c                 S      |d|   d S Nr   r(   r   r   r(   r(   r)   r         kwargs      E@)r   r   r   r   r"   map_structure)r   ra   out1out2	out2_valsout3r(   r(   r)   	_test_run  s   z'OneDeviceDistributionTestBase._test_runc                 C   &   | j |tdddgfdddgfd d S )Nr  r       E@inputsexpected_test_collective_comms_all_sumr   ra   r(   r(   r)   _test_all_reduce_sum     
z2OneDeviceDistributionTestBase._test_all_reduce_sumc                 C      | j |tdgdgd d S Nr  r  expected_grads _test_collective_comms_gradientsr  r  r(   r(   r)   _test_all_reduce_sum_gradients     
z<OneDeviceDistributionTestBase._test_all_reduce_sum_gradientsc                 C   r!  r"  $_test_collective_comms_gradient_taper  r  r(   r(   r)   "_test_all_reduce_sum_gradient_tape  r(  z@OneDeviceDistributionTestBase._test_all_reduce_sum_gradient_tapec                 C   r  )N       @      5@g      6@r  r  	_all_meanr  r(   r(   r)   _test_all_reduce_mean  r   z3OneDeviceDistributionTestBase._test_all_reduce_meanc                 C   r!  Ng      @r#  r&  r/  r  r(   r(   r)   _test_all_reduce_mean_gradients  r(  z=OneDeviceDistributionTestBase._test_all_reduce_mean_gradientsc                 C   r!  r1  r*  r/  r  r(   r(   r)   #_test_all_reduce_mean_gradient_tape  r(  zAOneDeviceDistributionTestBase._test_all_reduce_mean_gradient_tapec              	      sn   |  fdd |    | tt|j|| }| |d g|d  | |d g|d  d S )Nc                       t j S r*   r   DatasetZfrom_tensorsr0   r  r(   r)   r   "      zFOneDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>r   rB   )make_input_fn_iteratorr   
initializerE   mapr   r   r   )r   ra   comm_fnr  r  outputsr(   r9  r)   r     s   

z4OneDeviceDistributionTestBase._test_collective_commsc                    sb   t  r	| d  fdd}|fdd|   | || ||| d S )Nz5`tf.gradients` is not supported with eager execution.c                    (   t d} ||  }t||gd S Nr  r   r   rs   r   Z	gradientscru   rv   r>  r(   r)   r   1     
zLOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.stepc                    r6  r*   r7  r0   r9  r(   r)   r   7  r:  zPOneDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>)	r   r+   r   r;  r   r<  r   r   r   r   ra   r>  r  r$  r   r(   r>  r  r)   r&  ,  s   


z>OneDeviceDistributionTestBase._test_collective_comms_gradientsc                    sP    fdd}| fdd|   | || ||| d S )Nc                    T   t d}t }||  ||  }W d    n1 sw   Y  |||S Nr  r   rs   r   ZGradientTapewatchZgradientrD  ru   Ztaperv   rE  r(   r)   r   C     


zPOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.stepc                    r6  r*   r7  r0   r9  r(   r)   r   K  r:  zTOneDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>)r;  r   r<  r   r   r   rG  r(   rH  r)   r*  @  s   

zBOneDeviceDistributionTestBase._test_collective_comms_gradient_tapec                 C   s   t  r	| d tdd\}}|dd }dd }|||}tj|d j	d}|
|  |
| W d    d S 1 sCw   Y  d S )	N:cross-device tests are not supported with eager execution.r   r   c                 S      t jdS Nr   r   r8  r   r0   r(   r(   r)   r   Z  r:  z[OneDeviceDistributionTestBase._test_device_and_input_device_are_colocated.<locals>.<lambda>c                 S      | d S r   r(   r   r(   r(   r)   r   [      rB   target)r   r+   r   r   create_local_clusterr;  r   session_libSessionrV  r   r<  )r   ra   workersr1   r  r>  run_opr   r(   r(   r)   +_test_device_and_input_device_are_colocatedT  s   "zIOneDeviceDistributionTestBase._test_device_and_input_device_are_colocatedc           	      C   s   t  r	| d tdd\}}|dd }dd }t |j}t	
d |||}W d    n1 s8w   Y  tj|d jd	}||  || W d    d S 1 s^w   Y  d S )
NrO  r   r   c                 S   rP  rQ  rR  r0   r(   r(   r)   r   g  r:  ziOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_function.<locals>.<lambda>c                 S   rS  r   r(   r   r(   r(   r)   r   h  rT  z)/job:worker/replica:0/task:1/device:CPU:0rB   rU  )r   r+   r   r   rW  r;  r   r,   r   r   devicerX  rY  rV  r   r<  )	r   ra   rZ  r1   r  r>  r   r[  r   r(   r(   r)   9_test_device_and_input_device_are_colocated_with_functiona  s"   "zWOneDeviceDistributionTestBase._test_device_and_input_device_are_colocated_with_functionN)r%   r&   r'   r  r  r  r'  r+  r0  r3  r5  r  r&  r*  r\  r^  r(   r(   r(   r)   r    s    r  c                   @   s|   e Zd ZdZdddZdddZdddZdd	d
ZdddZdddZ		dddZ
	dddZ	dddZ	dddZdS )TwoDeviceDistributionTestBasezESome tests that should work with any two-device DistributionStrategy.Fc                 C   s   | tdd |}| ddg| || |j tdd ||fd}| t|j|}| ddg|d  | ddg|d	  |j td
d ||d}| ddg| || d S )Nc                   S   s   t  jd S r   )r   r4   r   r(   r(   r(   r)   r   w  s    z9TwoDeviceDistributionTestBase._test_run.<locals>.<lambda>rB   r   c                 S   r  r  r(   r   r(   r(   r)   r   |  r  r   r   r   r   c                 S   r	  r
  r(   r  r(   r(   r)   r     r  r  r      )r   r/   r   r   r   r"   r  )r   ra   r.   r  r  r  r  r(   r(   r)   r  u  s&    z'TwoDeviceDistributionTestBase._test_runc                 C   s4   | j |tddgddgddggfdddgf|d	 d S )
Nrq         @     C@r,       D@r  r  r  r  r  r.   r  r   ra   r.   r(   r(   r)   r       

z2TwoDeviceDistributionTestBase._test_all_reduce_sumc                 C       | j |tddgddg|d d S Nrq   ra  r  r  r$  r.   r%  re  r(   r(   r)   r'       
z<TwoDeviceDistributionTestBase._test_all_reduce_sum_gradientsc                 C   rg  rh  r)  re  r(   r(   r)   r+    rj  z@TwoDeviceDistributionTestBase._test_all_reduce_sum_gradient_tapec                 C   s4   | j |tddgddgddggfdddgf|d d S )	Nrq   ra  rb  r,  rc  r-  g     5@rd  r.  re  r(   r(   r)   r0    rf  z3TwoDeviceDistributionTestBase._test_all_reduce_meanc                 C   rg  Nrq   ra  r,  ri  r2  re  r(   r(   r)   r3    rj  z=TwoDeviceDistributionTestBase._test_all_reduce_mean_gradientsc                 C   rg  rk  r4  re  r(   r(   r)   r5    s   
zATwoDeviceDistributionTestBase._test_all_reduce_mean_gradient_tapec              
      s   |  fdd |    | tt|j|t|| }| |d |d g|d  | |d |d g|d  d S )Nc                    r6  r*   r   r8  Zfrom_tensor_slicesr0   r9  r(   r)   r     r:  zFTwoDeviceDistributionTestBase._test_collective_comms.<locals>.<lambda>r   rB   )	r;  r   r<  rE   r=  r   r   r/   r   )r   ra   r>  r  r  r.   r?  r(   r9  r)   r    s   

 z4TwoDeviceDistributionTestBase._test_collective_commsc                    sl   t  r|s| d  fdd}|fdd|   | || ||t	|| d S )NzP`tf.gradients` is not supported with eager execution without using tf.functions.c                    r@  rA  rB  rC  rE  r(   r)   r     rF  zLTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.stepc                    r6  r*   rl  r0   r9  r(   r)   r     r:  zPTwoDeviceDistributionTestBase._test_collective_comms_gradients.<locals>.<lambda>)
r   r+   r   r;  r   r<  r   r   r   r/   r   ra   r>  r  r$  r.   r   r(   rH  r)   r&    s"   


z>TwoDeviceDistributionTestBase._test_collective_comms_gradientsc                    sV    fdd}| fdd|   | || ||t|| d S )Nc                    rI  rJ  rK  rM  rE  r(   r)   r     rN  zPTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.stepc                    r6  r*   rl  r0   r9  r(   r)   r     r:  zTTwoDeviceDistributionTestBase._test_collective_comms_gradient_tape.<locals>.<lambda>)r;  r   r<  r   r   r   r/   rm  r(   rH  r)   r*    s    
zBTwoDeviceDistributionTestBase._test_collective_comms_gradient_tapeNr  )r%   r&   r'   r  r  r  r'  r+  r0  r3  r5  r  r&  r*  r(   r(   r(   r)   r_  r  s     








r_  c                   @   s@   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dS )&RemoteSingleWorkerMirroredStrategyBasez!Tests for a Remote single worker.c                 C   s   d S r*   r(   )r   r(   r(   r)   _get_num_gpus  s   z4RemoteSingleWorkerMirroredStrategyBase._get_num_gpusc                 C   s   |  |  |j d S r*   )r   ro  r   r   distributionr(   r(   r)   _testNumReplicasInSync  s   z=RemoteSingleWorkerMirroredStrategyBase._testNumReplicasInSyncc                 C   s(   t  r| | d S | j|dd d S )Ng?r   )r   r+   r   r   rp  r(   r(   r)   _testMinimizeLoss  s   z8RemoteSingleWorkerMirroredStrategyBase._testMinimizeLossc              	   C   s   |  ? td}td td}W d    n1 sw   Y  t r+d}nd}| |j| | |jd W d    d S 1 sFw   Y  d S )Nrq   z/cpu:0z)/job:worker/replica:0/task:0/device:CPU:0z/job:worker/replica:0/task:0)r   r   rs   r   r]  r   r+   r   )r   rq  r   r   r]  r(   r(   r)   _testDeviceScope  s   

"z7RemoteSingleWorkerMirroredStrategyBase._testDeviceScopec                    s   dd }|    d fddtdd D }|  #}| j|  d d}||}| ||jj|| W d    d S 1 sBw   Y  d S )	Nc                   S   rP  Nd   rR  r(   r(   r(   r)   r     r:  z\RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<lambda>rB   c                    s&   g | ]  fd dt D  qS )c                       g | ]} | qS r(   r(   r   jr   r(   r)   r         ziRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<listcomp>.<listcomp>)r   )r   num_gpusnum_workersrz  r)   r     s    z^RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDataset.<locals>.<listcomp>r   rv  r   r   r   )ro  r   r   r   r;  r   r9   r   )r   rq  Z
dataset_fnr   r   r   r   r(   r|  r)   #_testMakeInputFnIteratorWithDataset  s$   


"zJRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithDatasetc           	   	      s   dd }|   }d}g }tdd|D ] | fddt|D |  q|  &}| j||| |d d}||}| j||jj||d	d
d W d    d S 1 sRw   Y  d S )Nc                  S   s   t jd} t | }|jS ru  )r   r8  r   Zmake_one_shot_iteratorr   )Zdatasetitr(   r(   r)   r-     s   
zWRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable.<locals>.fnrB   r   rv  c                    rw  r(   r(   rx  rz  r(   r)   r   !  r{  z_RemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallable.<locals>.<listcomp>r  FTr   )	ro  r   rJ   r   r   r;  r   r9   r   )	r   rq  r-   r}  r~  r   r   r   r   r(   rz  r)   $_testMakeInputFnIteratorWithCallable  s&   "

"zKRemoteSingleWorkerMirroredStrategyBase._testMakeInputFnIteratorWithCallableN)
r%   r&   r'   r  ro  rr  rs  rt  r  r  r(   r(   r(   r)   rn    s    rn  c                 C      t  }|tjj| S r*   )r   r4   
all_reducer   r   r   r   ctxr(   r(   r)   r  2     r  c                 C   r  r*   )r   r4   r  r   r   ZMEANr  r(   r(   r)   r/  7  r  r/  r  r*   )Or  rV   rF   r   r   r   Ztensorflow.core.protobufr   Ztensorflow.core.utilr   Ztensorflow.python.clientr   rX  Ztensorflow.python.data.opsr   Ztensorflow.python.distributer   rh   r   r	   r
   re   r   r   Ztensorflow.python.eagerr   r   r   r   Ztensorflow.python.frameworkr   r   r   r   Ztensorflow.python.lib.ior   Ztensorflow.python.opsr   r   r   r   r   r   r   r   r   r   Ztensorflow.python.platformr   Ztensorflow.python.trainingr    r!   Ztensorflow.python.utilr"   r#   	Exceptionr$   r/   r2   r6   r=   r?   r@   rA   rQ   r[   r`   ZStrategyboolrg   ri   rj   ZTestCaserk   r  r_  rn  r  r/  r(   r(   r(   r)   <module>   s   

	
  \ytL