o
    7?ev                     @   s   d Z ddlZddlm  mZ ddlmZ ddlm	Z
 ddlmZ ddlmZ ddlmZ ddlmZ e ed	g d
G dd deZdS )z,Sharpness Aware Minimization implementation.    N)data_adapter)deserialize)Model)register_keras_serializable)serialize_keras_object)keras_exportz4keras.models.experimental.SharpnessAwareMinimization)v1c                       s`   e Zd ZdZd fdd	Zdd Zdd	 Z fd
dZed fdd	Z	dd Z
dd Z  ZS )SharpnessAwareMinimizationab  Sharpness aware minimization (SAM) training flow.

    Sharpness-aware minimization (SAM) is a technique that improves the model
    generalization and provides robustness to label noise. Mini-batch splitting
    is proven to improve the SAM's performance, so users can control how mini
    batches are split via setting the `num_batch_splits` argument.

    Args:
      model: `tf.keras.Model` instance. The inner model that does the
        forward-backward pass.
      rho: float. The gradients scaling factor. Defaults to `0.05`.
      num_batch_splits: int. The number of mini batches to
        split into from each data batch. If None, batches are not split into
        sub-batches. Defaults to `None`.
      name: string. The name of the SAM model. Defaults to `None`.

    Reference:
      [Pierre Foret et al., 2020](https://arxiv.org/abs/2010.01412)
    皙?Nc                    s$   t  j|d || _|| _|| _d S )N)name)super__init__modelrhonum_batch_splits)selfr   r   r   r   	__class__ n/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/keras/src/models/sharpness_aware_minimization.pyr   6   s   
z#SharpnessAwareMinimization.__init__c              	   C   s(  t |\}}}| jdurt|| j}t|| j}n|g}|g}g }g }t||D ]\}	}
g }t }| |	}| |
|}W d   n1 sKw   Y  |	| | jj
}|||}| |}| j|d  }t||D ]\}}|| }| ||tj  |	| qpt }| |	}| |
|}W d   n1 sw   Y  |||}t|dkr|D ]}|	|g qnt||D ]	\}}|	| qt||D ]\}}| || tj  qq+g }|D ]}|	tj|dd q| jt|| tj|dd}| j||| dd | jD S )a  The logic of one SAM training step.

        Args:
          data: A nested structure of `Tensor`s. It should be of structure
            (x, y, sample_weight) or (x, y).

        Returns:
          A dict mapping metric names to running average values.
        Ng-q=r   )Zaxisc                 S   s   i | ]}|j | qS r   )r   result).0mr   r   r   
<dictcomp>   s    z9SharpnessAwareMinimization.train_step.<locals>.<dictcomp>)r   Zunpack_x_y_sample_weightr   tfsplitzipZGradientTaper   Zcompiled_lossappendtrainable_variablesgradient_gradients_order2_normr   _distributed_apply_epsilon_w
distributeget_strategylenZ
reduce_sumZ	optimizerZapply_gradientsconcatZcompiled_metricsZupdate_stateZmetrics)r   dataxyZsample_weightZx_splitZy_splitZgradients_all_batchesZpred_all_batchesZx_batchZy_batchZepsilon_w_cacheZtapepredZlossr   	gradientsZgradients_order2_normscaler   variable	epsilon_wZgradient_all_batchesr   r   r   
train_step<   sh   






	z%SharpnessAwareMinimization.train_stepc                 C   
   |  |S )zForward pass of SAM.

        SAM delegates the forward pass call to the wrapped model.

        Args:
          inputs: Tensor. The model inputs.

        Returns:
          A Tensor, the outputs of the wrapped model for given `inputs`.
        )r   )r   Zinputsr   r   r   call   s   
zSharpnessAwareMinimization.callc                    s&   t   }|t| j| jd |S )N)r   r   )r   
get_configupdater   r   r   )r   configr   r   r   r1      s   
z%SharpnessAwareMinimization.get_configc                    s2   t |}t|d|d}||d< t ||S )Nr   )custom_objects)copydeepcopydeserialize_layerpopr   from_config)clsr3   r4   r   r   r   r   r9      s   

z&SharpnessAwareMinimization.from_configc                 C   sV   t tj tjjjtjjjfr$dd }tjjj	|tj || d S |
| d S )Nc                 S   s   | j j|dd |fdd d S )Nc                 S   r/   N)
assign_add)r'   r(   r   r   r   <lambda>   s   
 zcSharpnessAwareMinimization._distributed_apply_epsilon_w.<locals>.distribute_apply.<locals>.<lambda>F)argsgroup)extendedr2   )strategyvarr-   r   r   r   distribute_apply   s   
zQSharpnessAwareMinimization._distributed_apply_epsilon_w.<locals>.distribute_apply)
isinstancer   r"   r#   ZexperimentalZParameterServerStrategyZCentralStorageStrategyZ__internal__ZinterimZmaybe_merge_callr<   )r   rB   r-   rA   rC   r   r   r   r!      s   
z7SharpnessAwareMinimization._distributed_apply_epsilon_wc                 C   s   t t dd |D }|S )Nc                 S   s   g | ]}|d urt |qS r;   )r   norm)r   Zgradr   r   r   
<listcomp>   s    zESharpnessAwareMinimization._gradients_order2_norm.<locals>.<listcomp>)r   rE   stack)r   r*   rE   r   r   r   r       s   z1SharpnessAwareMinimization._gradients_order2_norm)r
   NNr;   )__name__
__module____qualname____doc__r   r.   r0   r1   classmethodr9   r!   r    __classcell__r   r   r   r   r	      s    F
	r	   )rK   r5   Ztensorflow.compat.v2compatv2r   Zkeras.src.enginer   Zkeras.src.layersr   r7   Zkeras.src.modelsr   Z$keras.src.saving.object_registrationr   Z"keras.src.saving.serialization_libr   Z tensorflow.python.util.tf_exportr   r	   r   r   r   r   <module>   s   
