o
    ?e                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
 ddl	mZ ddlmZ ddlmZ dd
dZdd Zdd Zdd Zd	S )z5Input-pipeline utilities for Distribution strategies.    )
distribute)dataset_ops)AutoShardPolicy)traverse)op_def_registry)ops)dataNc                 C   sf   t | tjr| ||S |  jjtjkr1|du rd}t | t	j
r)t| |||S t| |||S | S )aB  Shard the input pipeline by sharding the underlying list of files.

  Args:
    dataset: A `tf.data.Dataset` instance, typically the result of a bunch of
      dataset transformations.
    num_shards: A `tf.int64` scalar `tf.Tensor`, representing the number of
        shards operating in parallel. Same usage as in `tf.data.Dataset.shard`.
    index: A `tf.int64` scalar `tf.Tensor`, representing the worker index.
      Same usage as in `tf.data.Dataset.shard`.
    num_replicas_in_sync: An integer representing the total number of replicas
      across all workers. This is used in the rewrite when sharding by data.

  Returns:
    A modified `Dataset` obtained by updating the pipeline sharded by the
    files. The input dataset will be returned if we cannot automatically
    determine a good way to shard the input dataset.
  N   )
isinstancedistribute_typesZDistributedDatasetInterfaceZ
auto_shardoptionsZexperimental_distributeZauto_shard_policyr   ZOFF
data_typesZ	DatasetV1r   Z_AutoShardDatasetV1Z_AutoShardDataset)datasetZ
num_shardsindexZnum_replicas_in_sync r   g/home/www/facesmatcher.com/pyenv/lib/python3.10/site-packages/tensorflow/python/distribute/input_ops.pyauto_shard_dataset   s   


r   c                 C   s8   t | }t| jj|}|| jj jd }t|| jS )z&Returns a cloned version of `dataset`.r   )	r   Zobtain_all_variant_tensor_ops_clone_helperZ_variant_tensoropoutputsr   Z_VariantDatasetZelement_spec)r   variant_tensor_ops
remap_dictZnew_variant_tensorr   r   r   _clone_dataset>   s   
r   c                 C   s   | j pt| jS N)op_defr   gettype)r   r   r   r   _get_op_defF   s   r   c           
      C   s   i }| j D ]}|j}||v rt||}|| qg }| j D ]}|j}||v r4|| jd }|| q||j|j  qt }|j	| j
|dd | jD | j| jjt| d}	|	|| < |S )a  Helper method that recursively clones `op_to_clone`.

  Args:
    op_to_clone: The op we want to clone.
    variant_tensor_ops: A list of ops that we have to clone along the way.

  Returns:
    A dictionary mapping old_ops to new_ops created. Includes op_to_clone
    as a key.
  r   c                 S   s   g | ]}|j qS r   )Zdtype).0or   r   r   
<listcomp>f   s    z!_clone_helper.<locals>.<listcomp>)nameattrsr   )Zinputsr   r   updater   appendZvalue_indexr   Zget_default_graphZ	create_opr   r!   Znode_defattrr   )
Zop_to_cloner   r   Zinput_tensorZinput_tensor_opZrecursive_mapZinputs_listZremapped_inputgZnew_opr   r   r   r   J   s0   



r   r   )__doc__Z'tensorflow.python.data.experimental.opsr   Ztensorflow.python.data.opsr   Z"tensorflow.python.data.ops.optionsr   Ztensorflow.python.data.utilr   Ztensorflow.python.frameworkr   r   Ztensorflow.python.typesr   r   r   r   r   r   r   r   r   r   r   <module>   s   
"