o
    ?e+                     @   s  d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlZddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd ZG dd dej Z!			dddZ"e#e!e" dS )zKA Variable class that is replicated to logical cores for model parallelism.    )absolute_import)division)print_function)abcN)xla_sharding)tpu_util)context)config)ops)tensor_conversion_registry)control_flow_ops)gen_resource_variable_ops)gen_tpu_partition_ops)variable_scope	variables)save_context)basec                 K   sB   t |j | ||fi |W  d    S 1 sw   Y  d S N)r
   device)	update_fnvarvaluekwargs r   u/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/distribute/tpu_replicated_variable.py_on_device_update&   s   $r   c                   @   sR  e Zd ZdZdAddZdd Zedd Zedd	 Zed
d Z	edd Z
edd Zejdd Zejdd Zedd Zedd Zedd Zedd Zedd Zedd Zed d! Zed"d# Zd$d% Zd&d' Zed(d) Zed*d+ Zd,d- ZdBd0d1Zd2d3 Zd4d5 ZdCd7d8Z dCd9d:Z!dCd;d<Z"d=d> Z#d?d@ Z$d.S )DTPUReplicatedVariablea  Container for replicated `Variables` that are treated as a single variable.

  This class maintains a list of replicated variables that are stored on
  separate logic TPU devices. TF2XLA bridge accesses these variables as
  if they were a single variable.
  c                    s   t  tjr rtdd  D rtd  t fdd D r/tddd  D  t fdd D rFtd	d
d  D   | _|| _| jdd | _	d| _
dS )a  Treats `variables` as a replicated list of `tf.Variable`s.

    Example:

    ```
    variables = [
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
      tf.Variable(..., shape=(10, 100), dtype=tf.float32),
    ]
    replicated_variable = TPUReplicatedVariable(variables)
    assert replicated_variable.shape.as_list() == [10, 100]
    ```

    Args:
      variables: A list of `ResourceVariable`s that comprise this replicated
        variable. Variables should not be shared between different
        `TPUReplicatedVariable` objects.
      name: String. Name of this container. Defaults to "TPUReplicatedVariable".
    c                 s   s    | ]
}t |tj V  qd S r   )
isinstancevariables_libVariable.0vr   r   r   	<genexpr>I   s    
z1TPUReplicatedVariable.__init__.<locals>.<genexpr>zSArgument `variables` should be a non-empty list of `variables.Variable`s. Received c                 3        | ]}|j  d  j kV  qdS r   Ndtyper!   r   r   r   r$   N       zPAll elements in argument `variables` must have the same dtype. Received dtypes: c                 S      g | ]}|j qS r   r'   r!   r   r   r   
<listcomp>Q       z2TPUReplicatedVariable.__init__.<locals>.<listcomp>c                 3   r%   r&   shaper!   r   r   r   r$   S   r)   zPAll elements in argument `variables` must have the same shape. Received shapes: c                 S   r*   r   r-   r!   r   r   r   r+   V   r,   :r   N)r   r   Sequenceany	TypeError
ValueError_vars_namesplit_common_name_cached_value)selfr   namer   r   r   __init__3   s,   
zTPUReplicatedVariable.__init__c                 C   s
   t | jS )zBReturn an iterable for accessing the underlying sharded variables.)iterr4   r9   r   r   r   __iter__]   s   
zTPUReplicatedVariable.__iter__c                 C      | j S )z0The name of this object. Used for checkpointing.)r5   r=   r   r   r   r:   a   s   zTPUReplicatedVariable.namec                 C      | j d jS )z,The dtype of all `Variable`s in this object.r   )r4   r(   r=   r   r   r   r(   f      zTPUReplicatedVariable.dtypec                 C   r@   Nr   )r4   is_initializedr=   r   r   r   rC   k      z$TPUReplicatedVariable.is_initializedc                 C   r@   rB   )r4   	trainabler=   r   r   r   rE   o   rD   zTPUReplicatedVariable.trainablec                 C   r@   )zThe device this variable is on.r   )r4   r   r=   r   r   r   r   s   rA   zTPUReplicatedVariable.devicec                 c   s:    | j j  d V  W d    d S 1 sw   Y  d S r   )handlegraphZ
as_defaultr=   r   r   r   _handle_graphx   s   "z#TPUReplicatedVariable._handle_graphc                 c   sP    | j d ur#t| j g d V  W d    d S 1 sw   Y  d S d V  d S r   )r8   r
   Zcontrol_dependenciesr=   r   r   r   _assign_dependencies}   s   
"
z*TPUReplicatedVariable._assign_dependenciesc                 C   r@   rB   )r4   
constraintr=   r   r   r   rJ      rD   z TPUReplicatedVariable.constraintc                 C   r@   rB   )r4   _in_graph_moder=   r   r   r   rK      rD   z$TPUReplicatedVariable._in_graph_modec                 C   r@   rB   )r4   
_unique_idr=   r   r   r   rL      rD   z TPUReplicatedVariable._unique_idc                 C   r@   rB   )r4   rG   r=   r   r   r   rG      rD   zTPUReplicatedVariable.graphc                 C   r?   r   )r7   r=   r   r   r   _shared_name   s   z"TPUReplicatedVariable._shared_namec                 C      t jjS r   )r   ZVariableSynchronizationNONEr=   r   r   r   synchronization      z%TPUReplicatedVariable.synchronizationc                 C   rN   r   )r   ZVariableAggregationrO   r=   r   r   r   aggregation   rQ   z!TPUReplicatedVariable.aggregationc                 C   s   t  r
| jd gS | jS )zThe list of `Variables`.r   )r   in_save_contextr4   r=   r   r   r   r      s   zTPUReplicatedVariable.variablesc                 K   s   | j d }|j|||fi |}| j dd D ]}|| ||< ||j ||j< ||j q|| || < ||j || < ||  |S )zFor implementing `Trackable`.r      N)r4   _export_to_saved_model_graphrF   append)r9   Z
object_mapZ
tensor_mapoptionsr   Z	first_varZresource_listr#   r   r   r   rU      s   

z2TPUReplicatedVariable._export_to_saved_model_graphc                 C   s   t j| jd iS rB   )	trackableZVARIABLE_VALUE_KEYr4   r=   r   r   r   !_gather_saveables_for_saved_model      z7TPUReplicatedVariable._gather_saveables_for_saved_modelc                 C   r@   rB   )r4   r.   r=   r   r   r   r.      rD   zTPUReplicatedVariable.shapec                 C   s   t  st r| jd jS t d u rtdt	 / t
| dd }|d u s+t r:tjdd | jD g dd}n
tj|jgg dd}W d    n1 sNw   Y  t|S )	Nr   zQTPUReplicatedVariable.handle is not available outside tpu context or save contextZ_packed_varc                 S   r*   r   )rF   r!   r   r   r   r+      r,   z0TPUReplicatedVariable.handle.<locals>.<listcomp>F)Zpartition_dimsZ	is_packedT)r   rS   r   executing_eagerlyr4   rF   r   enclosing_tpu_contextNotImplementedErrorZoutside_or_skip_tpu_contextgetattrr	   Zget_soft_device_placementtpu_partition_opsZtpu_partitioned_input_v2Zpacked_handler   Z	replicate)r9   Z
packed_varZtensorr   r   r   rF      s"   


zTPUReplicatedVariable.handlec                 C   s   t | j| jS r   )r   Zread_variable_oprF   r(   r=   r   r   r   _read_variable_op   rZ   z'TPUReplicatedVariable._read_variable_opNFc                 C   s   t  du r
|  S |  S )z Converts a variable to a tensor.N)r   r\   
read_valuer`   )r9   r(   r:   as_refr   r   r   _dense_var_to_tensor   s   z*TPUReplicatedVariable._dense_var_to_tensorc                 C   s   | j d  S rB   )r4   ra   r=   r   r   r   ra      s   z TPUReplicatedVariable.read_valuec                    s4   t j|d| jd tjt fdd| jD  S )z;Converts the value to tensor and updates the variable list.Zvalue_in_tensor)r:   r(   c                 3   s$    | ]}t | fi V  qd S r   )r   r!   Zinput_tensorr   r   r   r   r$      s
    
z0TPUReplicatedVariable._update.<locals>.<genexpr>)r
   Zconvert_to_tensorr(   r   grouptupler   )r9   r   r   r   r   rd   r   _update   s   zTPUReplicatedVariable._updateTc                 C   J   t  d u s
t rdd }| j|||||dS t tj| ||||dS )Nc                 _      | j |i |S r   )assignr   akar   r   r   <lambda>       z.TPUReplicatedVariable.assign.<locals>.<lambda>r   use_lockingr:   ra   )r   r\   r   r[   rg   make_raw_assign_fnr   Zassign_variable_op)r9   r   rq   r:   ra   Z	assign_fnr   r   r   rj      $   zTPUReplicatedVariable.assignc                 C   rh   )Nc                 _   ri   r   )
assign_subrk   r   r   r   rn     ro   z2TPUReplicatedVariable.assign_sub.<locals>.<lambda>rp   )r   r\   r   r[   rg   rr   r   Zassign_sub_variable_op)r9   r   rq   r:   ra   Zassign_sub_fnr   r   r   rt      rs   z TPUReplicatedVariable.assign_subc                 C   rh   )Nc                 _   ri   r   )
assign_addrk   r   r   r   rn     ro   z2TPUReplicatedVariable.assign_add.<locals>.<lambda>rp   )r   r\   r   r[   rg   rr   r   Zassign_add_variable_op)r9   r   rq   r:   ra   Zassign_add_fnr   r   r   ru     rs   z TPUReplicatedVariable.assign_addc                 C   *   d dd t| jD }d| jj|f S )N,
c                 s        | ]\}}d ||f V  qdS )z  %d: %sNr   r"   ir#   r   r   r   r$   $      
z0TPUReplicatedVariable.__str__.<locals>.<genexpr>	%s:{
%s
}join	enumerater4   	__class____name__)r9   Z	debug_strr   r   r   __str__#     
zTPUReplicatedVariable.__str__c                 C   rv   )Nrw   c                 s   rx   )z  %d: %rNr   ry   r   r   r   r$   )  r{   z1TPUReplicatedVariable.__repr__.<locals>.<genexpr>r|   r}   )r9   Z
debug_reprr   r   r   __repr__(  r   zTPUReplicatedVariable.__repr__)r   NNF)FNT)%r   
__module____qualname____doc__r;   r>   propertyr:   r(   rC   rE   r   
contextlibcontextmanagerrH   rI   rJ   rK   rL   rG   rM   rP   rR   r   rU   rY   r.   rF   r`   rc   ra   rg   rj   rt   ru   r   r   r   r   r   r   r   +   sb    
*





















r   Fc                 C   s   | j |||dS )N)r(   r:   rb   )rc   )r   r(   r:   rb   r   r   r   %_tensor_conversion_tpu_replicated_var0  s   r   r   )$r   
__future__r   r   r   collectionsr   r   Z+tensorflow.python.compiler.xla.experimentalr   Ztensorflow.python.distributer   Ztensorflow.python.eagerr   Ztensorflow.python.frameworkr	   r
   r   Ztensorflow.python.opsr   r   r   r_   r   r   r   Ztensorflow.python.saved_modelr   Ztensorflow.python.trackabler   rX   r   r    r   r   Z#register_tensor_conversion_functionr   r   r   r   <module>   s<     
