o
    ?e4                     @   s   d Z ddlZddlmZ ddlmZ ddlmZ	 ddl
mZ ddl
mZ ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ i ZdZedddZdd Zdd Zdd Z dS )z>TPU specific APIs to be used in conjunction with TPU Strategy.    N)
config_pb2)session)cluster_resolver)context)def_function)
monitoring)device)errors)ops)
tf_logging)topology)tpu)compat) localz/tensorflow/tpu/worker_addressz;The worker address that the coordinator/client connects to.addressc              
      s
  |du st |tjrt|dstdtd t   t 	  t
  d | du rJt rFtjt j}|jdurFd|j |d} t| |sQJ t| j}|tv ratd| td| |tvrrd|   t rtjd	d
 fdd}t }|rtd td	 zEz!tt !  | }W d   n1 sw   Y  t"  W n t#j$y } zt#%dddt&| d}~ww W |durt| n|durt| w w t '  |( }nwt) sC| * }	| + }
t,j-dd}|
r|j./|
0  t1 2 ' t3j4||	d}|5t 6 }W d   n	1 s-w   Y  W d   n	1 s=w   Y  n"tt !  t j6 d	d}|W  d   S 1 s`w   Y  td t7j8|d}| 9| |t|< t:;d<| =  |S )a`  Implementation for tpu.experimental.initialize_tpu_system.

  Kept separate to avoid tpu_oss code duplication.

  Initialize the TPU devices.

  Args:
    cluster_resolver: A tf.distribute.cluster_resolver.TPUClusterResolver,
        which provides information about the TPU cluster.
    tpu_cluster_resolver_cls: a reference to
        tf.distribute.cluster_resolver.TPUClusterResolver so that an instance
        of it can be initialized if cluster_resolver is None.
  Returns:
    The tf.tpu.Topology object for the topology of the TPU cluster. If called
    inside tf.function, it returns the serialized topology object instead.

  Raises:
    RuntimeError: If running inside a tf.function.
    NotFoundError: If no TPU devices found in eager mode.
    TypeError: If tpu_cluster_resolver_cls is
        not tf.distribute.cluster_resolver.TPUClusterResolver.
  Ntpu_hardware_featureRtpu_cluster_resolver_cls is not tf.distribute.cluster_resolver.TPUClusterResolver.z6Deallocate tpu buffers before initializing tpu system.{}/replica:0/task:0r   z|TPU system %s has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.zInitializing the TPU system: %sFZ	autographc                      s   t j dddS )NF)job compilation_failure_closes_chipsZtpu_cancellation_closes_chips)r   initialize_system r   r   h/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/tpu/tpu_strategy_util.py_tpu_init_fnr   s
   z0initialize_tpu_system_impl.<locals>._tpu_init_fnzIt looks like tf.function behavior was disabled, perhaps using tf.config.run_functions_eagerly. tf.tpu.experimental.initialize_tpu_system requires tf.function to work. This primitive will override the disable.z9TPUs not found in the cluster. Failed in initialization: TZallow_soft_placementconfigtarget)r   r   z!Finished initializing TPU system.)Z
serializedr   )>
issubclasscluster_resolver_libClusterResolverhasattr	TypeErrorlogginginfor   _clear_cachesclear_kernel_cachegcZcollectexecuting_eagerlyr   
DeviceSpecfrom_stringdevice_namer   format
isinstancer   as_text_tpu_INITIALIZED_TPU_SYSTEMSwarning_LOCAL_MASTERSget_job_namer   functionfunctions_run_eagerlyrun_functions_eagerlyr
   r   _tpu_system_device_nameZ
async_waitr	   ZInvalidArgumentErrorZNotFoundErrorstrZ_initialize_logical_devicesnumpy#executing_eagerly_outside_functionsmastercluster_specr   ConfigProtocluster_defCopyFromas_cluster_defGraph
as_defaultsession_libSessionrunr   r   ZTopologyZset_tpu_topology_tpu_worker_addressZget_cellsetZ
get_master)r   tpu_cluster_resolver_clscurr_devicetpu_namer   run_eagerlyoutputeZserialized_topologyr>   r?   session_configsessZtpu_topologyr   r   r   initialize_tpu_system_impl+   s   







"

rS   c                   C   s   t  S )zReturns all currently initialized tpu systems.

  Returns:
     A dictionary, with tpu name as the key and the tpu topology as the value.
  )r3   copyr   r   r   r   get_initialized_tpu_systems   s   rU   c           
   	      sT  |du st |tjrt|dstdd | du r5t r1tj	t j
}|jdur1d|j |d} t| |s<J t| j}|tvrMtd|  td| t r|tvrbd|   tjdd	 fd
d}t }|r~td td z&tt  |  W d   n1 sw   Y  W |durt| n|durt| w w td t   t   nWt  s| ! }| " }t#j$dd}|r|j%&|'  t( ) & t*j+||d}	|	,t-  W d   n1 sw   Y  W d   n	1 sw   Y  nt.dtd |tv r(t|= dS dS )aW  Implementation for tpu.experimental.shutdown_tpu_system.

  Kept separate to avoid tpu_oss code duplication.

  Shuts down the TPU devices.

  This will clear all caches, even those that are maintained through sequential
  calls to tf.tpu.experimental.initialize_tpu_system, such as the compilation
  cache.

  Args:
    cluster_resolver: A tf.distribute.cluster_resolver.TPUClusterResolver,
        which provides information about the TPU cluster.
    tpu_cluster_resolver_cls: a reference to
        tf.distribute.cluster_resolver.TPUClusterResolver so that an instance
        of it can be initialized if cluster_resolver is None.

  Raises:
    RuntimeError: If no TPU devices found for eager execution or if run in a
        tf.function.
    TypeError: If tpu_cluster_resolver_cls is
        not tf.distribute.cluster_resolver.TPUClusterResolver.
  Nr   r   r   r   zDYou are shutting down a TPU system %s that has not been initialized.z Shutting down the TPU system: %sFr   c                      s   t j d d S )Nr   )r   shutdown_systemr   r   r   r   _tpu_shutdown_fn  s   z2shutdown_tpu_system_impl.<locals>._tpu_shutdown_fnzIt looks like tf.function behavior was disabled, perhaps using tf.config.run_functions_eagerly. tf.tpu.experimental.shutdown_tpu_system requires tf.function to work. This primitive will override the disable.zClearing out eager cachesTr   r   zinitialize_tpu_system is not supported within tf.functions.  You should call initialize_tpu_system outside of your tf.function. z"Finished shutting down TPU system.)/r!   r"   r#   r$   r%   r   r+   r   r,   r-   r.   r   r/   r0   r   r1   r2   r3   r&   r4   r'   r5   r6   r   r7   r8   r9   r
   r   r:   r(   r)   r=   r>   r?   r   r@   rA   rB   rC   rD   rE   rF   rG   rH   rV   RuntimeError)
r   rK   rL   rM   rW   rN   r>   r?   rQ   rR   r   r   r   shutdown_tpu_system_impl   s   









rY   )!__doc__r*   Ztensorflow.core.protobufr   Ztensorflow.python.clientr   rF   Z-tensorflow.python.distribute.cluster_resolverr   r"   Ztensorflow.python.eagerr   r   r   Ztensorflow.python.frameworkr   r	   r
   Ztensorflow.python.platformr   r&   Ztensorflow.python.tpur   r   Ztensorflow.python.utilr   r3   r5   ZStringGaugerI   rS   rU   rY   r   r   r   r   <module>   s2    	