o ?eQj@sdZddlmZddlmZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlmZddlmZddlmZddlmZdd lm Z e j!dd kr}d>d d Z"ndd l#m"Z"ddZ$d?ddZ%        d@ddZ&ddZ'dAddZ(dBddZ)dCd d!Z*Gd"d#d#e+Z,d$d%Z-Gd&d'd'e+Z.d(d)Z/ia0da1e2Z3da4e5Z6da7e8Z9d*d+Z:d,d-Z;d.d/ZGd4d5d5e+Z?Gd6d7d7e?Z@d>d8d9ZAd:d;ZBGd.chunk_readwb)r)r N)ropenwrite)urlfilenamerdatarrfdrrrr urlretrieve0s    "r#)r#cCsJttttttf}t|tjt j f|rdSt |p$t|t p$t|tjS)z'Check if `x` is a Keras generator type.F)strlisttupledictset frozenset isinstancerZTensornpZndarrayr isgeneratorSequencetypingIterator)xZbuiltin_iteratorsrrris_generator_or_sequenceXs  r1.autoc Cs|durdS|dkrddg}t|tr|g}t|}t|}|D]^}|dkr,tj}tj}|dkr6tj}tj}||r~||5}z| |Wn#tj t t fyit j|rht j|rct |t|wWddS1svwYdSq dS)a6Extracts an archive if it matches tar, tar.gz, tar.bz, or zip formats. Args: file_path: path to the archive file path: path to extract the archive file archive_format: Archive format to try for extracting the file. Options are 'auto', 'tar', 'zip', and None. 'tar' includes tar, tar.gz, and tar.bz files. The default 'auto' is ['tar', 'zip']. None or an empty list will return no matches found. Returns: True if a match was found and an archive extraction was completed, False otherwise. NFr3tarzipT)r*r$rtarfiler is_tarfilezipfileZipFile is_zipfile extractallTarError RuntimeErrorKeyboardInterruptospathexistsisfileremoveshutilrmtree) file_pathr@archive_formatZ archive_typeZopen_fnZ is_match_fnarchiverrr_extract_archivebsD          rIFdatasetsc  s| durtjtjdd} |dur|dur|}d}tj| } t| tjs/tjdd} tj| |} t| t|}|rLtj| |} | d} ntj| |} d}tj| ru|durtt | ||dstt d |d |d d }nd }|rt d |Gdddt fdd}d}z.ProgressTrackerN)__name__ __module__ __qualname__progbarrrrrProgressTrackersrRcs8jdur|dkr d}t|_dSj||dS)Nr )rQrupdate)r block_sizerrRrr dl_progresss zget_file..dl_progressz!URL fetch failure on {}: {} -- {}r4)rG)r?r@join expanduseraccessW_OK_makedirs_exist_okrrA validate_fileprintobjectr#urlliberror HTTPError ExceptionformatcodemsgURLErrorerrnoreasonr>rCrQrI)fnameoriginZuntarZmd5_hash file_hashZ cache_subdirZhash_algorithmextractrG cache_dirZ datadir_basedatadirZ untar_fpathfpathdownloadrV error_msgerrUrget_filesz:         rscCstj|dddS)NT)exist_ok)r?makedirs)rnrrrr[sr[cCs<|dkrtS|dkr|durt|dkrtStS)z+Returns hash algorithm as hashlib function.sha256r3N@)hashlibrvlenrL)rMrkrrr_resolve_hashers rzrvcstt|tr t|}n|}t|dtfdddD]}||qWd|S1s1wY|S)aCalculates a file sha256 or md5 hash. Example: ```python _hash_file('/path/to/file.zip') 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' ``` Args: fpath: path to the file being validated algorithm: hash algorithm, one of `'auto'`, `'sha256'`, or `'md5'`. The default `'auto'` detects the hash algorithm in use. chunk_size: Bytes to read at a time, important for large files. Returns: The file hash rbcs SN)rrrZ fpath_filerr?s z_hash_file..N)r*r$rzriterrS hexdigest)rorMrhasherrrr~r _hash_file&s     rcCs*t||}tt|||t|krdSdS)aValidates a file against a sha256 or md5 hash. Args: fpath: path to the file being validated file_hash: The expected hash string of the file. The sha256 and md5 hash algorithms are both supported. algorithm: Hash algorithm, one of 'auto', 'sha256', or 'md5'. The default 'auto' detects the hash algorithm in use. chunk_size: Bytes to read at a time, important for large files. Returns: Whether the file is valid TF)rzr$r)rorkrMrrrrrr\Es r\c@s0eZdZdZddZddZddZdd Zd S) ThreadsafeIterzEWrap an iterator with a lock and propagate exceptions to all threads.cCs||_t|_d|_dSr})it threadingLocklock _exception)selfrrrr__init__^s  zThreadsafeIter.__init__cCs|Sr}rrrrr__iter__kszThreadsafeIter.__iter__cCs|Sr})__next__rrrrnextnszThreadsafeIter.nextc Csd|j%|jr |jz t|jWWdSty'}z||_d}~ww1s+wYdSr})rrrrrb)rrrrrrrqs zThreadsafeIter.__next__N)rNrOrP__doc__rrrrrrrrr[s   rctfdd}|S)Ncst|i|Sr})r)akwfrrgszthreadsafe_generator..g functoolswraps)rrrrrthreadsafe_generator}src@s8eZdZdZeddZeddZddZdd Zd S) r-a1Base object for fitting to a sequence of data, such as a dataset. Every `Sequence` must implement the `__getitem__` and the `__len__` methods. If you want to modify your dataset between epochs you may implement `on_epoch_end`. The method `__getitem__` should return a complete batch. Notes: `Sequence` are a safer way to do multiprocessing. This structure guarantees that the network will only train once on each sample per epoch which is not the case with generators. Examples: ```python from skimage.io import imread from skimage.transform import resize import numpy as np import math # Here, `x_set` is list of path to the images # and `y_set` are the associated classes. class CIFAR10Sequence(Sequence): def __init__(self, x_set, y_set, batch_size): self.x, self.y = x_set, y_set self.batch_size = batch_size def __len__(self): return math.ceil(len(self.x) / self.batch_size) def __getitem__(self, idx): batch_x = self.x[idx * self.batch_size:(idx + 1) * self.batch_size] batch_y = self.y[idx * self.batch_size:(idx + 1) * self.batch_size] return np.array([ resize(imread(file_name), (200, 200)) for file_name in batch_x]), np.array(batch_y) ``` cCt)zGets batch at position `index`. Args: index: position of the batch in the Sequence. Returns: A batch NotImplementedError)rindexrrr __getitem__ zSequence.__getitem__cCr)zbNumber of batch in the Sequence. Returns: The number of batches in the Sequence. rrrrr__len__szSequence.__len__cCsdS)z-Method called at the end of every epoch. Nrrrrr on_epoch_endzSequence.on_epoch_endc#s,fddttDD]}|VqdS)z2Create a generator that iterate over the Sequence.c3s|]}|VqdSr}r).0irrr sz$Sequence.__iter__..N)rangery)ritemrrrrszSequence.__iter__N) rNrOrPrrrrrrrrrrr-s-   r-ccs |D]}|Vqq)zIterates indefinitely over a Sequence. Args: seq: `Sequence` instance. Yields: Batches of data from the `Sequence`. r)seqrrrriter_sequence_infinites  rcr)NcsHttd}a|i|}|a|WdS1swYdS)NT)_FORCE_THREADPOOL_LOCK_FORCE_THREADPOOL)argskwargsZold_force_threadpooloutrrrwrappeds  $z.dont_use_multiprocessing_pool..wrappedr)rrrrrdont_use_multiprocessing_poolsrcCs|rtrtjjStjSr})rmultiprocessingdummyZPool)use_multiprocessingrrrget_pool_classsrcCstdurtatS)z,Lazily create the queue to track worker ids.N)_WORKER_ID_QUEUErQueuerrrrget_worker_id_queuesrcCs|adSr}_SHARED_SEQUENCES)seqsrrr init_pool srcCs t||S)aQGet the value from the Sequence `uid` at index `i`. To allow multiple Sequences to be used at the same time, we use `uid` to get a specific one. A single Sequence would cause the validation to overwrite the training Sequence. Args: uid: int, Sequence identifier i: index Returns: The value at index `i`. r)uidrrrr get_indexs rc@sleZdZdZ dddZddZdd d Zd d ZdddZddZ e ddZ e ddZ e ddZ d S)SequenceEnqueueraBase class to enqueue inputs. The task of an Enqueuer is to use parallelism to speed up preprocessing. This is done with processes or threads. Example: ```python enqueuer = SequenceEnqueuer(...) enqueuer.start() datas = enqueuer.get() for data in datas: # Use the inputs; training, evaluating, predicting. # ... stop sometime. enqueuer.stop() ``` The `enqueuer.get()` should be an infinite stream of datas. FcCs||_||_tdurztddaWn tydaYnwtttr+t|_td7ant tj |_tj d7_ Wdn1sEwYd|_ d|_ d|_ d|_d|_dS)Nrrr )sequencer_SEQUENCE_COUNTERrValueOSErrorr*rrZget_lockvalueworkers executor_fnqueue run_thread stop_signal)rrrrrrr8s(     zSequenceEnqueuer.__init__cCs|jduo |j Sr})ris_setrrrr is_runningVszSequenceEnqueuer.is_runningr csd|jr ||_nfdd|_|_t||_t|_tj |j d|_ d|j _ |j dS)zStarts the handler's workers. Args: workers: Number of workers. max_queue_size: queue size (when full, workers could block on `put()`) cs tdS)NF)r)_rrrres z(SequenceEnqueuer.start..)targetTN)r_get_executor_initrrrrrEventrThread_runrdaemonstart)rrZmax_queue_sizerrrrYs  zSequenceEnqueuer.startcCs|jt|j<dS)z&Sends current Iterable to all workers.N)rrrrrrr_send_sequencemszSequenceEnqueuer._send_sequenceNcCsl|j|jj|jjd|j_|jjWdn1s$wY|j |dt |j <dS)zStops running threads and wait for them to exit, if necessary. Should be called by the same thread which called `start()`. Args: timeout: maximum time to wait on `thread.join()` rN) rr(rmutexclearunfinished_tasksnot_fullnotifyrrWrr)rtimeoutrrrstoprs    zSequenceEnqueuer.stopcCs|r |dSdSr})rrrrrr__del__s zSequenceEnqueuer.__del__cCr)?Submits request to the executor and queue the `Future` objects.rrrrrrrzSequenceEnqueuer._runcCr)Gets the Pool initializer for multiprocessing. Args: workers: Number of workers. Returns: Function, a Function to initialize the pool rrrrrrrrz#SequenceEnqueuer._get_executor_initcCr)zCreates a generator to extract data from the queue. Skip the data if it is `None`. # Returns Generator yielding tuples `(inputs, targets)` or `(inputs, targets, sample_weights)`. rrrrrrs zSequenceEnqueuer.get)F)r rr})rNrOrPrrrrrrrrrrrrrrrr#s      rcsBeZdZdZd fdd ZddZddZd d Zd d ZZ S)OrderedEnqueueraBuilds a Enqueuer from a Sequence. Args: sequence: A `tf.keras.utils.data_utils.Sequence` object. use_multiprocessing: use multiprocessing if True, otherwise threading shuffle: whether to shuffle the data at the beginning of each epoch Fctt|||||_dSr})superrrshuffle)rrrr __class__rrrs zOrderedEnqueuer.__init__csfdd}|S)rcs(tdt|dtfd}t||SNT)Z initializerZinitargs)rinit_pool_generatorr _DATA_POOLSaddrpoolrrrpool_fns   z3OrderedEnqueuer._get_executor_init..pool_fnrrrrrrrrs z"OrderedEnqueuer._get_executor_initcCs( td|jjdks|jrdSq)zWait for the queue to be empty.T皙?rN)timesleeprrrrrrrr _wait_queues  zOrderedEnqueuer._wait_queuecCsttt|j}| |jrt|t|t <}|D]}|j r0WddS|j j |t|j|fddq ||j rR WddSWdn1s\wY|j|qrTNblock)r%rryrrrrandomrrrrrrput apply_asyncrrrr)rrexecutorrrrrrs,     zOrderedEnqueuer._runc cs|rCz|jjddd}|r|j|dur |VWntjy+Ynty<}z||d}~ww|sdSdS)Creates a generator to extract data from the queue. Skip the data if it is `None`. Yields: The next element in the queue, i.e. a tuple `(inputs, targets)` or `(inputs, targets, sample_weights)`. TrrN)rrr task_doneEmptyrbr)rinputsrrrrrrs"  zOrderedEnqueuer.get)FF) rNrOrPrrrrrr __classcell__rrrrrsrcCsV|at}d|j|_|durtj||j|dur)|j |jddddSdS)aInitializer function for pool workers. Args: gens: State which should be made available to worker processes. random_seed: An optional value with which to seed child processes. id_queue: A multiprocessing Queue of worker ids. This is used to indicate that a worker process was created by Keras and can be terminated using the cleanup_all_keras_forkpools utility. zKeras_worker_{}NTrr) rrcurrent_processrcnamer+rseedidentr)Zgens random_seedZid_queueZ worker_procrrrrs rcCs tt|S)aKGets the next value from the generator `uid`. To allow multiple generators to be used at the same time, we use `uid` to get a specific one. A single generator would cause the validation to overwrite the training generator. Args: uid: int, generator identifier Returns: The next value of generator `uid`. )rr)rrrr next_samples rcs>eZdZdZ  d fdd ZddZdd Zd d ZZS) GeneratorEnqueueraBuilds a queue out of a data generator. The provided generator can be finite in which case the class will throw a `StopIteration` exception. Args: generator: a generator function which yields data use_multiprocessing: use multiprocessing if True, otherwise threading random_seed: Initial seed for workers, will be incremented by one for each worker. FNcrr})rr rr)r generatorrrrrrr4s zGeneratorEnqueuer.__init__csfdd}|S)zGets the Pool initializer for multiprocessing. Args: workers: Number of works. Returns: A Function to initialize the pool cs*tdt|jtfd}t||Sr)rrrrrrrrrrrCs   z5GeneratorEnqueuer._get_executor_init..pool_fnrrrrrr:s z$GeneratorEnqueuer._get_executor_initcCsh|t|t} |jr WddS|jj|t |j fddq 1s-wYdSr) rrrrrrrrrrr)rrrrrrKs zGeneratorEnqueuer._runc csz#|r"|jjdd}|j|dur|V|sWdSWdStydg}|jdkrE||jjdd|jdks4|D]}|qGdd|D}|D] }|dur`|VqWYdSty}z| dt |vryt d|d}~ww) rTrNrcSsg|] }|r|qSr)Z successfulr)rfuturerrr osz)GeneratorEnqueuer.get..zgenerator already executingzyYour generator is NOT thread-safe. Keras requires a thread-safe generator when `use_multiprocessing=False, workers > 1`. ) rrrr StopIterationqsizeappendwaitrbrr$r=)rrZ last_onesrrrrrrrVs<     zGeneratorEnqueuer.get)FN) rNrOrPrrrrrrrrrrr 's  r )NN)r2r3)FNNrJr3Fr3Nr})rvr{)r3r{)Drabcr contextlibrrrxrZmultiprocessing.dummyr?rrrDsysr6rrr.r_weakrefr8numpyr+Ztensorflow.python.frameworkrZsix.moves.urllib.requestrZtensorflow.python.keras.utilsrZ+tensorflow.python.keras.utils.generic_utilsrZ&tensorflow.python.keras.utils.io_utilsr version_infor#urllib.requestr1rIrsr[rzrr\r^rrr-rrrWeakSetrrr(Z _WORKER_IDSrRLockrrrrrrrrrrr rrrrs         % 3   " N  Y