o
    ?e{[                     @   s.  d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 dd	lmZ dd
lmZ ddlmZ ddlmZ dZG dd deZG dd deejZG dd deejZG dd deZG dd deejZd%ddZd%ddZd%dd ZG d!d" d"ejZG d#d$ d$ej Z!dS )&zcVarious classes representing TPU distributed values.

Note that the tests are in values_test.py .

    )packed_distributed_variable)tpu_replicated_variable)tpu_util)values)values_util)context)tape)ops)gen_resource_variable_ops)math_ops)variable_scopez{op_name} is only supported for distributed variable (variable created within certain `tf.distribute.Strategy` scope) with NONE  aggregation, got: {aggregation}.c                       s   e Zd ZdZ fddZ fddZ fddZdd	 Zed
d Z	edd Z
dd Z fddZ fddZ fddZedd Zd fdd	Z  ZS )TPUVariableMixinzMixin for TPU variables.c                    sH   t t| j|i | t r| jd tt| j | _	d S | j| _	d S )N_)
superr   __init__r	   Z#executing_eagerly_outside_functions_common_namestrid_primary
_handle_idselfargskwargs	__class__ h/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/distribute/tpu_values.pyr   +   s   zTPUVariableMixin.__init__c                    s,   t  d u rtt| |S td| d)Nz`TPUVariableMixin.z&` not accessible within a TPU context.)r   enclosing_tpu_contextr   r   __getattr__AttributeError)r   namer   r   r   r   5   s
   
zTPUVariableMixin.__getattr__c                    s"   t  d u rtt|  S td)Nz?`TPUVariableMixin.get()` is not supported within a TPU context.)r   r   r   r   getNotImplementedErrorr   r   r   r   r"   <   s
   zTPUVariableMixin.getc                 C   s   |   S N)
read_valuer$   r   r   r   _get_as_operandC   s   z TPUVariableMixin._get_as_operandc                 C   sp   t  }|du st r|  }t|tjr| S |j	S | j
du}| j}|r+| j
g}|| j| j||  |S )z2The handle by which this variable can be accessed.N)r   r   r   Zexecuting_eagerly_get_on_device_or_primary
isinstancepackedZPackedVarAndDeviceZon_device_handlehandleZ_packed_varZ_valuesZget_replicated_var_handler   r   Z_is_mirrored)r   tpu_contextvar	is_packedvalr   r   r   r+   F   s   
zTPUVariableMixin.handlec                 C   s   | j jS r%   )r+   devicer$   r   r   r   r0   \   s   zTPUVariableMixin.devicec                 C   sr   | j rt|  | j}t|ddr2t|  j t	|| j
W  d   S 1 s+w   Y  dS t	|| j
S )z!Reads the value of this variable.r.   FN)Z	trainabler   Zvariable_accessedr+   getattrr	   r0   r(   r
   Zread_variable_opdtype)r   r+   r   r   r   _read_variable_op`   s   
$z"TPUVariableMixin._read_variable_opc                    "   t  d u rtt|  S |  S r%   )r   r   r   r   r&   r3   r$   r   r   r   r&   m      zTPUVariableMixin.read_valuec                    r4   r%   )r   r   r   r   valuer3   r$   r   r   r   r6   s   r5   zTPUVariableMixin.valuec                    s   t  d u rtt|  S d S r%   )r   r   r   r   _as_graph_elementr$   r   r   r   r7   y   s   z"TPUVariableMixin._as_graph_elementc                 C   s8   t  r| jjS t| jjj| jjj| jjj| jjj	S r%   )
r   is_saving_non_distributedr   opr   ZDistributedVarOpr!   graph	tracebacktyper$   r   r   r   r9      s   zTPUVariableMixin.opNFc                    sV   t  du rtt| j|||dS |dur"|| jkr"t|  |S |r'| j	S |  S )z Converts a variable to a tensor.N)r2   r!   as_ref)
r   r   r   r   _dense_var_to_tensorr2   r   castr&   r+   )r   r2   r!   r=   r   r   r   r>      s   
z%TPUVariableMixin._dense_var_to_tensor)NNF)__name__
__module____qualname____doc__r   r   r"   r'   propertyr+   r0   r3   r&   r6   r7   r9   r>   __classcell__r   r   r   r   r   (   s"    



r   c                   @   st   e Zd ZdZdddZdddZdd	d
ZdddZdddZdddZ	dddZ
dddZdddZdddZdS )TPUDistributedVariablez-DistributedVariable subclass for TPUStrategy.FNTc                 C   0   t  r| j||||S | jj| ||||dS Nuse_lockingr!   r&   )r   r8   r   
assign_sub_policyr   r6   rJ   r!   r&   r   r   r   rK      
   
z!TPUDistributedVariable.assign_subc                 C   rG   rH   )r   r8   r   
assign_addrL   rM   r   r   r   rO      rN   z!TPUDistributedVariable.assign_addc                 C   rG   rH   )r   r8   r   assignrL   rM   r   r   r   rP      rN   zTPUDistributedVariable.assignc                 C   ,   t  r| j|||S | jj| |||dS N)rJ   r!   )r   r8   r   scatter_subrL   r   sparse_deltarJ   r!   r   r   r   rS      
   z"TPUDistributedVariable.scatter_subc                 C   rQ   rR   )r   r8   r   scatter_addrL   rT   r   r   r   rW      rV   z"TPUDistributedVariable.scatter_addc                 C   rQ   rR   )r   r8   r   scatter_mulrL   rT   r   r   r   rX      rV   z"TPUDistributedVariable.scatter_mulc                 C   rQ   rR   )r   r8   r   scatter_divrL   rT   r   r   r   rY      rV   z"TPUDistributedVariable.scatter_divc                 C   rQ   rR   )r   r8   r   scatter_minrL   rT   r   r   r   rZ      rV   z"TPUDistributedVariable.scatter_minc                 C   rQ   rR   )r   r8   r   scatter_maxrL   rT   r   r   r   r[      rV   z"TPUDistributedVariable.scatter_maxc                 C   rQ   rR   )r   r8   r   scatter_updaterL   rT   r   r   r   r\      rV   z%TPUDistributedVariable.scatter_updateFNTFN)r@   rA   rB   rC   rK   rO   rP   rS   rW   rX   rY   rZ   r[   r\   r   r   r   r   rF      s    








rF   c                       s   e Zd ZdZdd Ze fddZdd	d
ZdddZdddZ	dd Z
dd Zdd Zdd Zdd Zdd Zdd Z  ZS )TPUMirroredVariablezHHolds a map from replica to TPU variables whose values are kept in sync.c                 C   s   t | jtjS )zReturns whether each of the underlying variables is replicated or sharded to logical cores.

    If True, the handles of the underlying variables are not available outside a
    TPU context.
    )r)   r   r   ZTPUReplicatedVariabler$   r   r   r   *_is_replicated_or_sharded_to_logical_cores   s   z>TPUMirroredVariable._is_replicated_or_sharded_to_logical_coresc                    s(   |   rt d u r| jjS tt| jS r%   )r`   r   r   r   r0   r   r_   r$   r   r   r   r0      s
   
zTPUMirroredVariable.deviceFNTc                 C   r   t  }|  r|d u rdd }| j|||||dS |r0| jtjjkr0t t	j
| ||||dS t| ||||dS )Nc                 _      | j |i |S r%   )rK   vakar   r   r   <lambda>       z0TPUMirroredVariable.assign_sub.<locals>.<lambda>	update_fnr6   rJ   r!   r&   r6   rJ   r!   r&   rI   )r   r   r`   _updateaggregationr   VariableAggregationNONEmake_raw_assign_fnr
   assign_sub_variable_oprK   )r   r6   rJ   r!   r&   r,   assign_sub_fnr   r   r   rK      6   
zTPUMirroredVariable.assign_subc                 C   ra   )Nc                 _   rb   r%   )rO   rc   r   r   r   rg     rh   z0TPUMirroredVariable.assign_add.<locals>.<lambda>ri   rk   rI   )r   r   r`   rl   rm   r   rn   ro   rp   r
   assign_add_variable_oprO   )r   r6   rJ   r!   r&   r,   assign_add_fnr   r   r   rO      rs   zTPUMirroredVariable.assign_addc                 C   sv   t  }|  r|d u rdd }| j|||||dS t  r2| jtjjkr2t t	j
| ||||dS t| ||||dS )Nc                 _   rb   r%   )rP   rc   r   r   r   rg     rh   z,TPUMirroredVariable.assign.<locals>.<lambda>ri   rk   rI   )r   r   r`   rl   rm   r   rn   ro   rp   r
   assign_variable_oprP   )r   r6   rJ   r!   r&   r,   	assign_fnr   r   r   rP     s6   
zTPUMirroredVariable.assignc                 O      t  r| jj|i |S tr%   )r   r8   r   rS   r#   r   r   r   r   rS   0     zTPUMirroredVariable.scatter_subc                 O   rx   r%   )r   r8   r   rW   r#   r   r   r   r   rW   5  ry   zTPUMirroredVariable.scatter_addc                 O   rx   r%   )r   r8   r   r[   r#   r   r   r   r   r[   :  ry   zTPUMirroredVariable.scatter_maxc                 O   rx   r%   )r   r8   r   rZ   r#   r   r   r   r   rZ   ?  ry   zTPUMirroredVariable.scatter_minc                 O   rx   r%   )r   r8   r   rX   r#   r   r   r   r   rX   D  ry   zTPUMirroredVariable.scatter_mulc                 O   rx   r%   )r   r8   r   rY   r#   r   r   r   r   rY   I  ry   zTPUMirroredVariable.scatter_divc                 O   rx   r%   )r   r8   r   r\   r#   r   r   r   r   r\   N  ry   z"TPUMirroredVariable.scatter_updater]   )r@   rA   rB   rC   r`   rD   r0   rK   rO   rP   rS   rW   r[   rZ   rX   rY   r\   rE   r   r   r   r   r_      s    	


r_   c                       sR   e Zd ZdZdd Zd fdd	Zd fd	d
	Zd fdd	Z fddZ  Z	S )TPULazyDistributedVariablez:TPU Mirrored variable to be initialized lazily in a batch.c                 C   s$   t | ddrd S | j  d| _d S )N_is_lazily_initializedFT)r1   Z_lazy_scopeZinitialize_allr{   r$   r   r   r   _initialize_if_uninitializedW  s   

z7TPULazyDistributedVariable._initialize_if_uninitializedFNTc                       |    t ||||S r%   )r|   r   rK   rM   r   r   r   rK   ^     z%TPULazyDistributedVariable.assign_subc                    r}   r%   )r|   r   rO   rM   r   r   r   rO   d  r~   z%TPULazyDistributedVariable.assign_addc                    r}   r%   )r|   r   rP   rM   r   r   r   rP   j  s   z!TPULazyDistributedVariable.assignc                    s   |    t  S r%   )r|   r   r&   r$   r   r   r   r&   q  s   
z%TPULazyDistributedVariable.read_valuer]   )
r@   rA   rB   rC   r|   rK   rO   rP   r&   rE   r   r   r   r   rz   T  s    rz   c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	TPUSyncOnReadVariablezGHolds a map from replica to variables whose values are reduced on save.c                 O   D   t  d u rtjj| g|R i |S t tj| g|R i |S r%   )r   r   r   SyncOnReadVariablerK   rp   r
   rq   r   r   r   r   rK   y     z TPUSyncOnReadVariable.assign_subc                 O   r   r%   )r   r   r   r   rO   rp   r
   rt   r   r   r   r   rO     r   z TPUSyncOnReadVariable.assign_addc                 O   r   r%   )r   r   r   r   rP   rp   r
   rv   r   r   r   r   rP     s   zTPUSyncOnReadVariable.assignN)r@   rA   rB   rC   rK   rO   rP   r   r   r   r   r   v  s
    r   FNTc                 C       t tj}| j|||||dS Nri   )r   rp   r
   rq   rl   )r-   r6   rJ   r!   r&   rr   r   r   r   rK        rK   c                 C   r   r   )r   rp   r
   rt   rl   )r-   r6   rJ   r!   r&   ru   r   r   r   rO     r   rO   c                 C   r   r   )r   rp   r
   rv   rl   )r-   r6   rJ   r!   r&   rw   r   r   r   rP     r   rP   c                   @   s   e Zd ZdZ			dddZ			dddZdd	d
Z		dddZdddZdddZ	dddZ
dddZdddZdddZdddZdS )TPUOnWritePolicyzPolicy defined for `tf.VariableSynchronization.ON_WRITE` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.AUTO` or `tf.VariableSynchronization.ON_WRITE`.
  FNTc                 C   B   t  r|jtjjkrt tj|||||dS t	|||||dS Nrk   rI   )
r   r   rm   r   rn   ro   rp   r
   rq   rK   r   r-   r6   rJ   r!   r&   r   r   r   rK        
zTPUOnWritePolicy.assign_subc                 C   r   r   )
r   r   rm   r   rn   ro   rp   r
   rt   rO   r   r   r   r   rO     r   zTPUOnWritePolicy.assign_addc                 C   r   r   )
r   r   rm   r   rn   ro   rp   r
   rv   rP   r   r   r   r   rP     s   
zTPUOnWritePolicy.assignc                 C   sV   t |}t  r"| jtjjkrttj	|| jd|||||dS |j
||||dS )N)op_namerm   )rU   rJ   r!   )rj   r6   rJ   r!   )r   Zmake_raw_scatter_xxx_fnr   Z_aggregationr   rn   ro   r#   _scatter_error_msgformatrl   )r   Zraw_scater_xxx_fnr   r-   rU   rJ   r!   Zscater_xxx_fnr   r   r   _scatter_xxx  s"   
zTPUOnWritePolicy._scatter_xxxc                 C      |  tjd||||S )NrS   )r   r
   Zresource_scatter_subr   r-   rU   rJ   r!   r   r   r   rS        zTPUOnWritePolicy.scatter_subc                 C   r   )NrW   )r   r
   Zresource_scatter_addr   r   r   r   rW     r   zTPUOnWritePolicy.scatter_addc                 C   r   )Nr[   )r   r
   Zresource_scatter_maxr   r   r   r   r[     r   zTPUOnWritePolicy.scatter_maxc                 C   r   )NrZ   )r   r
   Zresource_scatter_minr   r   r   r   rZ     r   zTPUOnWritePolicy.scatter_minc                 C   r   )NrX   )r   r
   Zresource_scatter_mulr   r   r   r   rX     r   zTPUOnWritePolicy.scatter_mulc                 C   r   )NrY   )r   r
   Zresource_scatter_divr   r   r   r   rY     r   zTPUOnWritePolicy.scatter_divc                 C   r   )Nr\   )r   r
   Zresource_scatter_updater   r   r   r   r\     r   zTPUOnWritePolicy.scatter_updater]   r^   )r@   rA   rB   rC   rK   rO   rP   r   rS   rW   r[   rZ   rX   rY   r\   r   r   r   r   r     s*    	









r   c                       sp   e Zd ZdZ fddZ fddZ fddZdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Z  ZS )TPUOnReadPolicya{  Policy defined for `tf.VariableSynchronization.ON_READ` synchronization.

  This policy is created when `synchronization` is set to
  `tf.VariableSynchronization.ON_READ` and `aggregation` is set to any of the
  values allowed by the `tf.VariableAggregation` enum such as `NONE`, `SUM`,
  `MEAN` or `ONLY_FIRST_REPLICA`when creating a `tf.Variable` in `tf.distribute`
  scope.
  c                    H   t  d u rtt| j|g|R i |S t tj|g|R i |S r%   )r   r   r   r   rK   rp   r
   rq   r   r-   r   r   r   r   r   rK   /     zTPUOnReadPolicy.assign_subc                    r   r%   )r   r   r   r   rO   rp   r
   rt   r   r   r   r   rO   7  r   zTPUOnReadPolicy.assign_addc                    r   r%   )r   r   r   r   rP   rp   r
   rv   r   r   r   r   rP   ?  s   zTPUOnReadPolicy.assignc                 O      t r%   r#   r   r   r   r   rS   F     zTPUOnReadPolicy.scatter_subc                 O   r   r%   r   r   r   r   r   rW   I  r   zTPUOnReadPolicy.scatter_addc                 O   r   r%   r   r   r   r   r   r[   L  r   zTPUOnReadPolicy.scatter_maxc                 O   r   r%   r   r   r   r   r   rZ   O  r   zTPUOnReadPolicy.scatter_minc                 O   r   r%   r   r   r   r   r   rX   R  r   zTPUOnReadPolicy.scatter_mulc                 O   r   r%   r   r   r   r   r   rY   U  r   zTPUOnReadPolicy.scatter_divc                 O   r   r%   r   r   r   r   r   r\   X  r   zTPUOnReadPolicy.scatter_update)r@   rA   rB   rC   rK   rO   rP   rS   rW   r[   rZ   rX   rY   r\   rE   r   r   r   r   r   %  s    	r   r]   )"rC   Ztensorflow.python.distributer   r*   r   r   r   r   Ztensorflow.python.eagerr   r   Ztensorflow.python.frameworkr	   Ztensorflow.python.opsr
   r   r   r   objectr   ZDistributedVariablerF   ZMirroredVariabler_   rz   r   r   rK   rO   rP   ZOnWritePolicyr   ZOnReadPolicyr   r   r   r   r   <module>   s.   m@"


r