o
    ?eD+                     @   s   d Z ddlZddlZddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dae  Z!dZ"de#fddZ$G dd dej%Z&dS )zHook for asynchronous checkpointing.

This hook dispatches checkpoint writing operations in a separate thread to
allow execution to continue on the main thread.
    N)AnyListOptionalText)	event_pb2)session)
meta_graph)ops)
tf_logging)metrics)basic_session_run_hooks)monitored_session)saver)session_run_hook)training_util)SummaryWriterCacheZasync_checkpoint_v1returnc                 C   s   t t||  d dS )z@Returns the duration between start and end time in microseconds.i@B r   )maxint)Zstart_time_secondsZend_time_seconds r   g/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/tpu/async_checkpoint.py_get_duration_microseconds1   s   r   c                   @   s   e Zd ZdZ						d"dedee dee deej ded	ee	j
 d
eeej  fddZdd Zdd ZdejdefddZdefddZdejdefddZdejfddZd#ddZd d! ZdS )$AsyncCheckpointSaverHookz+Saves checkpoints every N steps or seconds.N
model.ckptcheckpoint_dir	save_secs
save_stepsr   checkpoint_basenamescaffold	listenersc           	      C   s   t j||}td| |rtdt| |dur#|dur#td|| _d| _d| _	|| _
|| _|| _tj||d| _|p@g | _d| _d| _d| _d| _t tdu rat aW d   dS W d   dS 1 slw   Y  dS )a  Initializes a `CheckpointSaverHook`.

    Args:
      checkpoint_dir: `str`, base directory for the checkpoint files.
      save_secs: `int`, save every N secs.
      save_steps: `int`, save every N steps.
      saver: `Saver` object, used for saving.
      checkpoint_basename: `str`, base name for the checkpoint files.
      scaffold: `Scaffold`, use to get saver object.
      listeners: List of `CheckpointSaverListener` subclass instances. Used for
        callbacks that run immediately before or after this hook saves the
        checkpoint.

    Raises:
      ValueError: One of `save_steps` or `save_secs` should be set.
      ValueError: At most one of `saver` or `scaffold` should be set.
    z1Create AsyncCheckpointSaverHook saving to path
%sz with %d listener(s).Nz+You cannot provide both saver and scaffold.)Z
every_secsZevery_steps   )ospathjoinlogginginfolen
ValueError_saver_save_thread_write_graph_thread_checkpoint_dir
_save_path	_scaffoldr   ZSecondOrStepTimer_timer
_listeners_steps_per_run_summary_writer_global_step_tensor_last_checkpoint_step_END_TIME_OF_LAST_WRITE_LOCK_END_TIME_OF_LAST_WRITEtime)	selfr   r   r   r   r   r   r   Z	save_pathr   r   r   __init__9   s8   

"z!AsyncCheckpointSaverHook.__init__c                 C   s
   || _ d S N)r0   )r7   Zsteps_per_runr   r   r   _set_steps_per_runo   s   
z+AsyncCheckpointSaverHook._set_steps_per_runc                 C   sB   t | j| _t | _| jd u rtd| jD ]}|	  qd S )Nz9Global step should be created to use CheckpointSaverHook.)
r   getr+   r1   r   Z_get_or_create_global_step_readr2   RuntimeErrorr/   begin)r7   lr   r   r   r=   r   s   



zAsyncCheckpointSaverHook.beginr   coordc                 C   s   | | j}dd }tj|| gd| _| j  |  r!|  jnd }t	 }t
j|jdd|d}| jd u r;td| j| | j| | || | j| d S )Nc                 S   s    t t jdd| jd d S )NTZ
add_shapeszgraph.pbtxt)r   Zwrite_graphr	   get_default_graphas_graph_defr+   )r7   r   r   r   _write_graph_fn   s   zFAsyncCheckpointSaverHook.after_create_session.<locals>._write_graph_fn)targetargsTr@   )Z	graph_def	saver_def!Summary writer is not initialised)runr2   	threadingThreadr*   start
_get_saverrF   r	   rA   r   Zcreate_meta_graph_defrB   r1   r'   Z	add_graphZadd_meta_graph_saver.   update_last_triggered_step)r7   r   r?   global_steprC   rF   graphZmeta_graph_defr   r   r   after_create_session{   s"   

z-AsyncCheckpointSaverHook.after_create_sessionrun_contextc                 C   s   t | jS r9   )r   ZSessionRunArgsr2   )r7   rR   r   r   r   
before_run   s   z#AsyncCheckpointSaverHook.before_run
run_valuesc                 C   sT   |j | j}| j|r&| j| td| | |j |r(|	  d S d S d S )NzTriggering checkpoint. %s)
r   rH   r2   r.   Zshould_trigger_for_steprN   r$   r%   rM   Zrequest_stop)r7   rR   rT   rO   r   r   r   	after_run   s   z"AsyncCheckpointSaverHook.after_runc                 C   sv   | j rtd | j   | jrtd | j  || j}| j|kr-| j||dd | j	D ]}|
|| q0d S )Nz.Waiting for any pending checkpoints to finish.z.Waiting for any pending write_graph to finish.F)asynchronous)r)   r$   r%   r#   r*   rH   r2   r3   rM   r/   end)r7   r   Z	last_stepr>   r   r   r   rW      s   





zAsyncCheckpointSaverHook.endTc                    s   fdd}t     fdd}|s_|  |  dS jdur:jjdd j r:td |  dS _tj|d	_j	  |  dS )
z1Saves the latest checkpoint, returns should_stop.c                     s
  t d j t }  jD ]}| q  j jd  jdu r,t	d j
tjtjj jd  jD ]}| q>t }tjtt| |d t tjttt| d W d   n1 slw   Y  | at d||   t d j dS )	zRun the saver process.z"Saving checkpoints for %d into %s.)rO   NrG   )statusZcheckpoint_pathZ	api_labelmicrosecondsz*Checkpoint actual writing time: (%.3f sec)z#Checkpoint finished for %d into %s.)r$   r%   r,   r6   r/   Zbefore_saverL   saver1   r'   Zadd_session_logr   Z
SessionLog
CHECKPOINTZ
after_saver   ZAddAsyncCheckpointWriteDuration_ASYNC_CHECKPOINT_V1r   r4   ZAddTrainingTimeSavedr5   )
start_timer>   end_time)r7   r   stepr   r   _save_fn   sD   



z0AsyncCheckpointSaverHook._save.<locals>._save_fnc                     s    t   } tjtt | d d S )NrY   )r6   r   ZAddCheckpointWriteDurationr]   r   )Zblocking_end_time)blocking_start_timer   r   end_of_blocking_time   s   
z<AsyncCheckpointSaverHook._save.<locals>.end_of_blocking_timeNg?)timeoutz4Saver thread still in progress, skipping checkpoint.)rD   )
r6   r3   r)   r#   is_aliver$   r%   rI   rJ   rK   )r7   r   r`   rV   ra   rc   r   )rb   r7   r   r`   r   rM      s$   +




zAsyncCheckpointSaverHook._savec                 C   sr   | j d ur| j S | jd ur| jjS tjj}t|}|s#td|t	|dkr0td||d | _ |d S )Nz_No items in collection {}. Please add a saver to the collection or provide a saver or scaffold.r    zgMore than one item in collection {}. Please indicate which one to use by passing it to the constructor.r   )
r(   r-   r   r	   Z	GraphKeysZSAVERSZget_collectionr<   formatr&   )r7   Zcollection_keyZsaversr   r   r   rL      s$   



z#AsyncCheckpointSaverHook._get_saver)NNNr   NN)T)__name__
__module____qualname____doc__r   r   r   	saver_libZSaverr   ZScaffoldr   r   ZCheckpointSaverListenerr8   r:   r=   session_libSessionr   rQ   rS   r   ZSessionRunContextrU   rW   rM   rL   r   r   r   r   r   6   sF    
6	
	
Hr   )'rj   r!   rI   r6   typingr   r   r   r   Ztensorflow.core.utilr   Ztensorflow.python.clientr   rl   Ztensorflow.python.frameworkr   r	   Ztensorflow.python.platformr
   r$   Z0tensorflow.python.saved_model.pywrap_saved_modelr   Ztensorflow.python.trainingr   r   r   rk   r   r   Z%tensorflow.python.training.summary_ior   r5   Lockr4   r]   r   r   ZCheckpointSaverHookr   r   r   r   r   <module>   s,   