o
    ?ek                  	   @   sb  d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddlmZ! ddl"m#Z# ddl$m%Z& ddl'm(Z( ddl)m*Z* da+da,e
j-Z-dd Z.e.ej/ej0Z0e.ej1ej2Z2e.ej3ej4Z4e
j5Z5			dddZ6d d! Z7d"d# Z8dd%d&Z9ej:d'd(d)Z;de;fd*d+Z<	$dd,d-Z=e=d$d.e9dd/d0Z>e=d$d.e9dd/dd1Z?e=d$d2e9dd/d0Z@ejAd3ejBdd/ZCejAd4d5d6 dd/ZejAd7d8d6 d.d/ZDejAd9d:d6 dd/ZEejAd;d<d6 d.d/ZFejAd=e6d(d>d$d?ZejAd@e6d(d$dAd$d?ZGejAdBe6d(d$dCd$d?ZHejAdDe6d.d>d$d?ZIejAdEe6d(d$dFd$d?ZJejAdGe6d.d$dFd$d?ZKejAdHe6d(d>d$d$dIZLeAdJdKd6 ZMejAdLdMd6 d.d/ZNejAdNdOd6 d.d/ZOejAdPdQd6 dd/ZPejAdRdSd6 d(d/ZQejAdTdUd6 d(dVZReAdWdXd6 ZSdYeS_ ejAdZd[d6 d(d/ZTejAd\d]d6 d.d/ZUejAd^e9dd/d$d.e>d$d_ZVejAd`e9d.d/d$d.d.e>ddaZWejAdbe9d.d/d$d.d.e?ddaZXejAdce9d(d/d$d.d(e>d$ddZYejAdee9d(ddfd$d.d(e>d$dgZZejAdhe9dd/d$d2e@d$d_Z[de;fdidjZ\e\dkd2d(dlZ]e\dmd.d(dlZ^e\dnd2d(d.doZ_e\dpd.d(d.doZ`dqdrgZadsdt ZbeCeeDeOeQeUgZceeDeOeQgZdeeIeGeLgZeedee Zfecee ZgeOeQeVeWeeIeUgZheYe[gZieOeQeeIgZjeVeWeYgZkdudv Zldwdx Zmdydz Znd{d| Zod}d~ Zpe*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd e*dg dqerd dS )z1Strategy combinations for combinations.combine().    N)
config_pb2)tf2)central_storage_strategycluster_resolver)collective_all_reduce_strategy)combinations)distribute_lib)mirrored_strategy)multi_process_runner)multi_worker_test_base)one_device_strategy)parameter_server_strategy_v2)sharded_variable)	test_util)tpu_strategy)tpu_cluster_resolver)context)remote)device)errors)flags)device_assignment)
server_lib)	tf_exportFc                    s    fdd}|S )Nc                     s$   t  r| i |S  | i |S N)r   enabled)argskwargstf1_clstf2_cls s/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/distribute/strategy_combinations.pycreator5   s   z!_version_chooser.<locals>.creatorr"   )r    r!   r$   r"   r   r#   _version_chooser4   s   r%   c                    s    fdd}|S )Nc               	      s  t j} zt }d}W n+ ty6   d}tjt| dr| jpdt| dr'| jp(d t| dr0| jp1d d}Y nw t	sMt
| ddsA|rHt| da	t|ad }rYtjttjd}t rjtj||fd	i}ntj||fi } r}r}td
 |_|S )NTFtpu zoneproject)r&   r(   r)   )Zcore_assignmentZ"experimental_spmd_xla_partitioningz1Packed Variable is not compatiable with SPMD mode)r   FLAGSr   ZTPUClusterResolver
ValueErrorhasattrr&   r(   r)   _did_connect_to_clustergetattrr   Zconnect_to_clusterZinitialize_tpu_system	_topologydevice_assignment_libZDeviceAssignmentZSINGLE_CORE_ASSIGNMENTr   r   tpu_libZTPUStrategyV2ZTPUStrategyV1Z%_enable_packed_variable_in_eager_mode)r*   resolverZdid_automatically_resolver   strategyenable_packed_variableenable_spmd_xla_paritioningr   steps_per_runuse_single_corer"   r#   _create_tpu_strategyU   sT   


z7_get_tpu_strategy_creator.<locals>._create_tpu_strategyr"   )r7   r8   r5   r6   r   r9   r"   r4   r#   _get_tpu_strategy_creatorN   s   5r:   c                 C   sd   t dd | D }|tt dkrt  td| tj j	d7  _	tj
 j	d7  _	t
| S )Nc                 s   s&    | ]}t j|jd krdV  qdS )CPU   N)	tf_deviceZ
DeviceSpecZfrom_stringZdevice_type).0dr"   r"   r#   	<genexpr>   s    
z>_mirrored_strategy_with_collective_key_base.<locals>.<genexpr>r;   順 )sumlenr   Zlist_logical_devicesZ_reset_contextr   set_logical_devices_to_at_leastmirrored_libMirroredStrategyV1_collective_key_baseMirroredStrategy)devicesZrequired_cpus_numsr"   r"   r#   +_mirrored_strategy_with_collective_key_base   s   rJ   c                 C   s8   t j jd7  _t j jd7  _t| }dd |j_|S )NrA   c                   S      dS NFr"   r"   r"   r"   r#   <lambda>       z7_mirrored_strategy_with_no_merge_call.<locals>.<lambda>)rE   rF   rG   rH   extended_use_merge_call)rI   outr"   r"   r#   %_mirrored_strategy_with_no_merge_call   s
   rR   Tc                    s   fdd  fdd}|S )Nc               	      s   t  } |  }| jr|td| j d  }t j|  | j| j|| j	d i| jp)dd}dt
_t jdd t  t|d}W d    n1 sKw   Y  sXd	d
 |j_z	t   W |S  tyk   Y |S w )Nz%s://GPUgrpc)cluster_spec	task_typetask_idmasterenvironmentnum_accelerators	rpc_layerFr'   )Zservice_typer   c                   S   rK   rL   r"   r"   r"   r"   r#   rM      rN   z[_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirrored.<locals>.<lambda>)r   TFConfigClusterResolverrX   r[   rC   SimpleClusterResolverrU   rV   rW   rY   CollectiveAllReduceExtendedZ_enable_check_healthr   Zconfigure_coordination_serviceZ
eager_modeCollectiveAllReduceStrategyrO   rP   r   Zget_barrierwaitr+   )	tf_configrX   r2   r3   required_gpususe_merge_callr"   r#   _create_multi_worker_mirrored   s8   
zI_get_multi_worker_mirrored_creator.<locals>._create_multi_worker_mirroredc               
      sV   z  W S  t jy* }  zd| jv r%ttjdkr%dtjd v r%td d } ~ ww )NCould not start gRPC serverr<   Zbazelr   Cannot start std servers.)r   UnknownErrormessagerC   sysargvunittestSkipTest)e)re   r"   r#    skip_if_cannot_start_grpc_server   s   

zL_get_multi_worker_mirrored_creator.<locals>.skip_if_cannot_start_grpc_serverr"   )rc   rd   ro   r"   )re   rc   rd   r#   "_get_multi_worker_mirrored_creator   s   -rp         )Zmin_shard_bytesZ
max_shardsc                    s    dd   fdd}|S )Nc                 S   s   t j| |dS )N)variable_partitioner)r   ZParameterServerStrategyV2)r2   rs   r"   r"   r#   _create_ps_strategy   s   z5_get_ps_strategy_creator.<locals>._create_ps_strategyc               
      s   t  rtjdd} tjt| didd} |S t }|	 
 } | s,d S tjt| di|j|j|j|jp?dd}|jdv ryt }d|_ztj| |j|jd|d}W n tjyt } zd	|jv rotd
 d }~ww |   |S )NrT   )num_workersnum_psr[   rS   )rZ   r[   )rZ   rV   rW   rY   r[   )ZworkerZps   )Zjob_nameZ
task_indexprotocolconfigrf   rg   )framework_test_utilZis_xla_enabledr   Zcreate_in_process_clusterr   r]   r   ZClusterSpecr\   rU   as_dictrV   rW   rY   r[   r   ZConfigProtoZinter_op_parallelism_threadsServerr   rh   ri   rl   rm   join)Zcluster_defr2   ra   Zworker_configserverrn   rt   rv   ru   rc   rs   r"   r#   _create_parameter_server   sR   





z:_get_ps_strategy_creator.<locals>._create_parameter_serverr"   )ru   rv   rc   rs   r   r"   r   r#   _get_ps_strategy_creator   s   =r   c                    s   g   fdd}|S )a  Returns a callable that returns the pool runner.

  It creates the pool runner only upon first invocation. This avoids creating it
  when this file is imported.

  Args:
    has_chief: whether there should be a chief.
    num_workers: the number of workers excluding the chief.
    initializer: initializer of each process.
    share_gpu: whether to share GPU between the workers.

  Returns:
    A callable that returns the runner.
  c                     s8    st jddd} tj| d} |  d S )Nr   F)	has_chiefru   rv   Zhas_eval)initializer	share_gpu)r   Zcreate_cluster_specr   ZMultiProcessPoolRunnerappend)rU   runner	containerr   r   ru   r   r"   r#   get_or_createL  s   
z,_deferred_pool_runner.<locals>.get_or_creater"   )r   ru   r   r   r   r"   r   r#   _deferred_pool_runner8  s   r   r<   )rc   )r   ru   r   )r   ru   r   r      ZDefaultZOneDeviceCPUc                   C      t dS N/cpu:0OneDeviceStrategyr"   r"   r"   r#   rM   w      rM   ZOneDeviceGPUc                   C   r   N/gpu:0r   r"   r"   r"   r#   rM   z  r   ZOneDeviceOnWorker1CPUc                   C   r   )Nz"/job:worker/replica:0/task:1/cpu:0r   r"   r"   r"   r#   rM   ~  r   ZOneDeviceOnWorker1GPUc                   C   r   )Nz"/job:worker/replica:0/task:1/gpu:0r   r"   r"   r"   r#   rM     r   ZTPU)r7   )required_tpuZTPUPackedVar)r7   r5   Z
TPUUseSPMD)r7   r6   Z
TPUOneStepZ
TPUOneCore)r7   r8   ZTPUOneStepOneCoreZCloudTPU)r   Zuse_cloud_tpuZMirrored1CPUc                   C   
   t dgS r   rJ   r"   r"   r"   r#   rM        
 ZMirrored1GPUc                   C   r   r   r   r"   r"   r"   r#   rM     r   ZMirroredCPUAndGPUc                   C      t ddgS Nr   r   r   r"   r"   r"   r#   rM         ZMirrored2CPUsc                   C   r   )Nr   /cpu:1r   r"   r"   r"   r#   rM     r   ZMirrored2GPUsc                   C   r   Nr   z/gpu:1r   r"   r"   r"   r#   rM     r   ZMirrored2GPUsNoMergeCallc                   C   r   r   )rR   r"   r"   r"   r#   rM     r   )required_physical_gpusZMirrored2CPUc                   C   r   )Nr   z/cpu:2r   r"   r"   r"   r#   rM     r   zYMirrored strategy with 2 virtual CPUs.

    Should set up logical devices before use
    ZCentralStorage2GPUsc                   C   r   r   CentralStorageStrategyr"   r"   r"   r#   rM     r   ZCentralStorageCPUAndGPUc                   C   r   r   r   r"   r"   r"   r#   rM     r   ZMultiWorkerMirrored2x1CPU)r   ru   pool_runner_fnno_xlaZMultiWorkerMirrored2x1GPU)r   ru   rc   r   r   Z MultiWorkerMirrored2x1GPUNoShareZMultiWorkerMirrored2x2GPU)r   ru   rc   r   r   Z$MultiWorkerMirrored2x2GPUNoMergeCallrb   )r   ru   r   r   r   ZMultiWorkerMirrored4x1CPUc                 C   s"   t j| t||||d||d|dS )N)ru   rv   rc   rs   T)rc   ru   r   rv   )r   NamedDistributionr   )nameru   rv   rc   rs   r"   r"   r#   parameter_server_strategy_fn  s   r   ZParameterServer3Worker2PSCPU)ru   rv   ZParameterServer1Worker2PSCPUZParameterServer3Worker2PS1GPU)ru   rv   rc   ZParameterServer1Worker2PS1GPUgrapheagerc                 C   s   t d|  d S )Nr;   )r   rD   )Znum_virtual_cpusr"   r"   r#   set_virtual_cpus_to_at_least>     r   c                   C   s   t jtddgdS Nr   r   distributionmode)r   combinestrategies_minus_tpur"   r"   r"   r#   strategy_minus_tpu_combinations|  s   r   c                   C   s   t jtdgdS )Nr   r   )r   r   tpu_strategiesr"   r"   r"   r#   tpu_strategy_combinations  r   r   c                   C      t  t  S r   )r   r   r"   r"   r"   r#   all_strategy_combinations  s   r   c                   C   s   t jttttgddgdS r   )r   r   r   one_device_strategy_gpu"mirrored_strategy_with_gpu_and_cpumirrored_strategy_with_two_gpusr"   r"   r"   r#   /all_strategy_minus_default_and_tpu_combinations  s   r   c                   C   r   r   )r   r   r"   r"   r"   r#   'all_strategy_combinations_minus_default  s   r   zN__internal__.distribute.combinations.central_storage_strategy_with_gpu_and_cpu)v1)central_storage_strategy_with_gpu_and_cpuzK__internal__.distribute.combinations.central_storage_strategy_with_two_gpus&central_storage_strategy_with_two_gpusz7__internal__.distribute.combinations.cloud_tpu_strategycloud_tpu_strategyz5__internal__.distribute.combinations.default_strategydefault_strategyzG__internal__.distribute.combinations.mirrored_strategy_with_cpu_1_and_2"mirrored_strategy_with_cpu_1_and_2zD__internal__.distribute.combinations.mirrored_strategy_with_two_cpusmirrored_strategy_with_two_cpuszG__internal__.distribute.combinations.mirrored_strategy_with_gpu_and_cpur   zC__internal__.distribute.combinations.mirrored_strategy_with_one_cpumirrored_strategy_with_one_cpuzC__internal__.distribute.combinations.mirrored_strategy_with_one_gpumirrored_strategy_with_one_gpuzD__internal__.distribute.combinations.mirrored_strategy_with_two_gpusr   zR__internal__.distribute.combinations.mirrored_strategy_with_two_gpus_no_merge_call-mirrored_strategy_with_two_gpus_no_merge_callzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_cpumulti_worker_mirrored_2x1_cpuzB__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpumulti_worker_mirrored_2x1_gpuzJ__internal__.distribute.combinations.multi_worker_mirrored_2x1_gpu_noshare%multi_worker_mirrored_2x1_gpu_nosharezB__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpumulti_worker_mirrored_2x2_gpuzP__internal__.distribute.combinations.multi_worker_mirrored_2x2_gpu_no_merge_call+multi_worker_mirrored_2x2_gpu_no_merge_callz8__internal__.distribute.combinations.one_device_strategyr   z<__internal__.distribute.combinations.one_device_strategy_gpur   z1__internal__.distribute.combinations.tpu_strategyr   zN__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_cpu)parameter_server_strategy_3worker_2ps_cpuzN__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_cpu)parameter_server_strategy_1worker_2ps_cpuzO__internal__.distribute.combinations.parameter_server_strategy_3worker_2ps_1gpu*parameter_server_strategy_3worker_2ps_1gpuzO__internal__.distribute.combinations.parameter_server_strategy_1worker_2ps_1gpu*parameter_server_strategy_1worker_2ps_1gpuz:__internal__.distribute.combinations.tpu_strategy_one_coretpu_strategy_one_corez<__internal__.distribute.combinations.tpu_strategy_packed_vartpu_strategy_packed_var)FFF)T)NT)s__doc__rj   rl   Ztensorflow.core.protobufr   Ztensorflow.pythonr   Ztensorflow.python.distributer   r   r   r   r	   r
   rE   r   r   r   Zone_device_libr   r   r   r   r1   Z-tensorflow.python.distribute.cluster_resolverr   Ztensorflow.python.eagerr   r   Ztensorflow.python.frameworkr   r=   r   rz   Ztensorflow.python.platformr   Ztensorflow.python.tpur   r0   Ztensorflow.python.trainingr   Z tensorflow.python.util.tf_exportr   r-   r/   r^   r%   rF   rH   ZCentralStorageStrategyV1r   ZOneDeviceStrategyV1r   r_   r:   rJ   rR   rp   ZMinSizePartitionerZDEFAULT_PARTITIONERr   r   Z_two_worker_poolZ_two_worker_pool_noshareZ_four_worker_poolr   Z_get_default_strategyr   r   Zone_device_strategy_on_worker_1Z#one_device_strategy_gpu_on_worker_1r   Ztpu_strategy_spmdZtpu_strategy_one_stepr   Ztpu_strategy_one_step_one_corer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Zmulti_worker_mirrored_4x1_cpur   r   r   r   r   Zgraph_and_eager_modesr   r   Z strategies_minus_default_and_tpur   Zall_strategies_minus_defaultZall_strategiesZtwo_replica_strategiesZfour_replica_strategiesZmultidevice_strategiesZmultiworker_strategiesr   r   r   r   r   Zexport_constant__name__r"   r"   r"   r#   <module>   s$  	
?
	@
L
$




	
	



























