o
    ?eP)                     @   s   d Z ddlZddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
 ddl	mZ dd	l	mZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ G dd dejZG dd dejZedg dG dd dejZdS )zDTensor variable and saveable.    N)api)layout)context)def_function)dtypes)errors_impl)ops)math_ops)resource_variable_ops)base)saveable_object)	tf_exportc                       s&   e Zd ZdZ		d fdd	Z  ZS )	DSaveSpeczCDTensor SaveSpec that additionaly captures global_shape and layout.Nc                    s&   t  j|||||d || _|| _d S )N)tensor
slice_specnamedtypedevice)super__init__global_shaper   )selfr   r   r   r   r   r   r   	__class__ e/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/dtensor/python/d_variable.pyr   $   s   
zDSaveSpec.__init__)NN)__name__
__module____qualname____doc__r   __classcell__r   r   r   r   r   !   s
    r   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )_DVariableSaveablez8Class for defining how to save/restore DTensor variable.c                    s   t  j t W d    n1 sw   Y  _ _ fdd}tjj	
  fdd}j	 }ttd t||dg| tjjj	
 dd||g| tjjj	
 dd j  rttjn j jdg| d S )	Nc                    s<   t  j t| |W  d    S 1 sw   Y  d S N)r   r   r   pack)Ztensorsr   )	dvariabler   r   r#   @   s   
$z)_DVariableSaveable.__init__.<locals>.packc                     sh   j   dkr"t rtt  } q&t	  } n  } 
| r2t| tjS | S )NCPU)meshdevice_typeupperr   executing_eagerlyr   r#   unpackZ
read_valuecopy_to_meshshould_castr	   castr   bfloat16)Zhost_dtensorr$   Zhost_layoutZoriginal_layoutr   r   r   get_host_dtensorG   s   z5_DVariableSaveable.__init__.<locals>.get_host_dtensor r   )Zrank)r   r   r   r   r   r   r   )r   r   r   fetch_layout_original_layout
_dvariable
layout_libZLayoutZsharding_specsr&   Z	host_meshnum_local_devicesr   r!   r   r   Z
replicatedshapeZ	to_stringr,   r   r.   r   )r   r$   r   r#   r0   r6   r   r/   r   r   9   sF   





z_DVariableSaveable.__init__c                 C   s   | j jo	|jtjkS )a  Returns True if v has float32 dtype and is intructed to save as bf16.

    Args:
      v : The variable that determines whether to cast.

    Returns:
      True if current savable DVariable is instructed to save as bfloat16 and
        the variable has dtype float32.
    )r4   save_as_bf16r   r   float32)r   vr   r   r   r,   l   s   
z_DVariableSaveable.should_castc                    s\   |\}t j fdd} jj  dkr||} j jjr+t	j
| jjdS |S )z+Restores the same value into all variables.c                    s@   t  jj t|  jW  d    S 1 sw   Y  d S r"   )r   r   r4   r   r+   r3   )tr   r   r   _restore|   s   $z,_DVariableSaveable.restore.<locals>._restorer%   r   )r   functionr3   r&   r'   r(   r4   Zassignr8   r	   r-   r   )r   Zrestored_tensorsZrestored_shapesr   r=   r   r<   r   restorex   s   z_DVariableSaveable.restore)r   r   r   r   r   r,   r@   r    r   r   r   r   r!   6   s
    3r!   zexperimental.dtensor.DVariable)v1c                       sH   e Zd ZdZdd fdd
Zedd Zejdd Zd	d
 Z  Z	S )	DVariablezA replacement for tf.Variable which follows initial value placement.

    The class also handles restore/save operations in DTensor. Note that,
    DVariable may fall back to normal tf.Variable at this moment if
    `initial_value` is not a DTensor.
  Nr>   c          
   
      s  | dd}|dd}t|rC|}tt|tjr|j}tt|tj	r@|r)|s-t
dtj||tj|dgt| dd}n| }t|tjrL|j}tj||d}|j}d	| _t| t rt|rt|}	|dur||	krtddd
|	 d| d|	}n|durt||}ntddd|| _t|j t t!| j"|g|R d|i| W d   n1 sw   Y  n#|durt||}t t!| j"|g|R d|i| W d   dS W d   dS 1 sw   Y  dS )z4Overrides tf.Variable to fix VarHandleOp placements.r   Nr7   z)Expected shape and layout to be not None.r   )r7   offset)Z
shard_infor>   Fz:Conflicting layout are provided for initial value layout (z) and variable (z).z6Neither layout nor DTensor initial value are provided.r   )#popgetcallable
issubclasstype	functoolspartialfunc	trackableZCheckpointInitialValueCallable
ValueErrorr   Zcall_with_layoutZ	ShardInfolen
isinstanceZCheckpointInitialValueZwrapped_valuer   Zconvert_to_tensorr   _save_as_bf16r   r)   Z
is_dtensorr2   r   ZInvalidArgumentErrorZrelayoutr   Zdefault_meshr&   r   rB   r   )
r   initial_valuer   argskwargsr   r7   Z	unwrappedZvariable_deviceZvalue_layoutr   r   r   r      s   	



" zDVariable.__init__c                 C   s   | j S r"   )rP   r<   r   r   r   r8      s   zDVariable.save_as_bf16c                 C   s   |o| j tjk| _dS )z#Enables saving float32 as bfloat16.N)r   r   r9   rP   )r   r8   r   r   r   r8      s   c                 C   s   t jtt| iS r"   )rL   ZVARIABLE_VALUE_KEYrI   rJ   r!   r<   r   r   r    _gather_saveables_for_checkpoint   s   
z*DVariable._gather_saveables_for_checkpoint)
r   r   r   r   r   propertyr8   setterrT   r    r   r   r   r   rB      s    _

rB   )r   rI   Ztensorflow.dtensor.pythonr   r   r5   Ztensorflow.python.eagerr   r   Ztensorflow.python.frameworkr   r   r   Ztensorflow.python.opsr	   r
   Ztensorflow.python.trackabler   rL   Z!tensorflow.python.training.savingr   Z tensorflow.python.util.tf_exportr   ZSaveSpecr   ZSaveableObjectr!   ZResourceVariablerB   r   r   r   r   <module>   s$   
Y