o
    ?e                     @   s|   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z
 dd Zd	d
 Zdd Zdd Zdd Zdd Zdd ZdS )zOptimizer utilities.    )central_storage_strategy)distribute_lib)reduce_util)clip_ops)
tf_loggingc                 C   s   t | } t| }|r+t r dd |D }t jtjj	|}q-t
 jt|fd}ng }g }d}| D ]\}}|du rC|d|f q3||| |f |d7 }q3|t|ks[J d|S )zReturns all-reduced gradients aggregated via summation.

  Args:
    grads_and_vars: List of (gradient, variable) pairs.

  Returns:
    List of (gradient, variable) pairs where gradients have been all-reduced.
  c                 S   s   g | ]}|d  qS )r    ).0pairr   r   k/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/keras/optimizer_v2/utils.py
<listcomp>%   s    z,all_reduce_sum_gradients.<locals>.<listcomp>)argsr   N   zFailed to add all gradients)listfilter_empty_gradientsstrategy_supports_no_merge_callr   get_strategyextendedZ_replica_ctx_all_reduceds_reduce_utilReduceOpSUMZget_replica_contextZ
merge_call_all_reduce_sum_fnappendlen)grads_and_varsZfiltered_grads_and_varsgradsZreducedZreduced_with_nonesZreduced_posgvr   r   r
   all_reduce_sum_gradients   s*   	

r   c                 C   s   t | } | s| S g }g }| D ]\}}|du r|| q|||f qt |}|s6tddd | D f |rCtddd |D  |S )zDFilter out `(grad, var)` pairs that have a gradient equal to `None`.Nz+No gradients provided for any variable: %s.c                 S   s   g | ]\}}|j qS r   name)r   _r   r   r   r
   r   L   s    z*filter_empty_gradients.<locals>.<listcomp>zAGradients do not exist for variables %s when minimizing the loss.c                 S   s   g | ]}|j qS r   r   )r   r   r   r   r
   r   P   s    )tupler   
ValueErrorloggingwarning)r   filteredZvars_with_empty_gradsZgradvarr   r   r
   r   ;   s(   r   c                         du rdd S  fdd}|S )@Creates a gradient transformation function for clipping by norm.Nc                 S      | S Nr   r   r   r   r
   <lambda>W       z+make_gradient_clipnorm_fn.<locals>.<lambda>c                    4   t t tjtjfrtd fdd| D }|S )Nz:`clipnorm` is not supported with `CenteralStorageStrategy`c                    s    g | ]\}}t | |fqS r   )r   Zclip_by_normr   r   r   clipnormr   r
   r   a   s    zKmake_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fn.<locals>.<listcomp>
isinstancer   r   r   CentralStorageStrategyCentralStorageStrategyV1r"   r   clipped_grads_and_varsr0   r   r
   gradient_clipnorm_fnY   s   
z7make_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   r1   r8   r   r0   r
   make_gradient_clipnorm_fnT      r:   c                    r'   )r(   Nc                 S   r)   r*   r   r+   r   r   r
   r,   l   r-   z2make_global_gradient_clipnorm_fn.<locals>.<lambda>c                    sL   t t tjtjfrtdt|  \}}t	| \}}t
t||}|S )NzA`global_clipnorm` is not supported with `CenteralStorageStrategy`)r3   r   r   r   r4   r5   r"   zipr   Zclip_by_global_normr   )r   r   	variablesZclipped_gradsr    r7   r0   r   r
   r8   n   s   z>make_global_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   r9   r   r0   r
    make_global_gradient_clipnorm_fni   r;   r>   c                    r'   )zACreates a gradient transformation function for clipping by value.Nc                 S   r)   r*   r   r+   r   r   r
   r,      r-   z,make_gradient_clipvalue_fn.<locals>.<lambda>c                    r.   )Nz;`clipvalue` is not supported with `CenteralStorageStrategy`c                    s$   g | ]\}}t |   |fqS r   )r   Zclip_by_valuer/   	clipvaluer   r
   r      s    
zMmake_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fn.<locals>.<listcomp>r2   r6   r?   r   r
   gradient_clipvalue_fn   s   
z9make_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fnr   )r@   rA   r   r?   r
   make_gradient_clipvalue_fn~   r;   rB   c                 C   s   | j tjj|S r*   )r   Zbatch_reduce_tor   r   r   )distributionr   r   r   r
   r      s   r   c                  C   s    t  sdS t  } | j  S )zDReturns if the current Strategy can operate in pure replica context.T)r   Zhas_strategyr   r   Z_use_merge_call)Zstrategyr   r   r
   r      s   r   N)__doc__Ztensorflow.python.distributer   r   r   r   Ztensorflow.python.opsr   Ztensorflow.python.platformr   r#   r   r   r:   r>   rB   r   r   r   r   r   r
   <module>   s   #