
    gjq                     2   d Z ddlZddlZddlZddlZddlZddlmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZ erddlm Z m!Z!  ej"        e#          Z$dZ%g dZ&dZ'dZ(dZ)dZ* e+            Z,dZ-dZ.dZ/e-fde0de0de1de2fdZ3de0de2fdZ4 G d d          Z5 G d d          Z6 G d  d!          Z7 G d" d#          Z8dddd$dd%d&d'd(e2d)e2d*e9e         d+e9e         d,e2d-e2dz  d.e2e:z  dz  d/e2dz  d0e:d1e2dz  dd2fd3Z;dS )4ac  Streamed, multi-commit upload of a folder on top of the Xet upload protocol.

How it works:

- The **coordinator** (caller's thread) walks the list of files and asks the Hub, 256 files at a
  time, what each file is (regular git blob, xet file, ignored). Regular files are accumulated
  directly; xet files are registered into a `XetSession` upload-commit, which chunks, deduplicates,
  retries and uploads them in the background while the coordinator keeps going. No Python-side
  sha256 computation: `hf_xet` computes it during chunking (single read pass over each file).
- Whenever enough files have accumulated (adaptive batch size), the batch is handed over to the
  **committer** thread which joins the xet uploads, drops unchanged files, and creates a git
  commit for the batch. While a batch is being committed, the coordinator is already uploading
  the next one.
- Interrupted uploads are resumable by simply re-running the same call: already-committed files
  are dropped (no-op detection against the remote oid) and already-uploaded chunks are
  deduplicated by the xet storage backend, transferring ~0 bytes.
    N)TYPE_CHECKINGAnyCallable)quote   )	constants)CommitOperationAddCommitOperationDelete_fetch_upload_modes_send_commit_warn_on_overwriting_operations)RepositoryNotFoundError)are_progress_bars_disabledlogging)XetTokenTypeabort_xet_sessionget_xet_sessionxet_connection_info_refresh_urlxet_headers_without_auth)
CommitInfoHfApi   )
   2   K   d   }         i  iX       g      D@g     r@i  @r         ?g      >@currenttotalwidthreturnc                 z    |dk    rd|z  S t          t          | |z  d          |z            }d|z  d||z
  z  z   S )Nr   u   ░      ?u   █)intmin)r#   r$   r%   filleds       d/var/www/app.qaans.com/backend/venv/lib/python3.11/site-packages/huggingface_hub/_upload_pipeline.py_barr-   U   sM    zzu}Wu_c**U233F6>EUV^444    nc                     dD ]A}t          |           dk     r'| dk     r	| d| c S | dk     r	| d| c S | d| c S | dz  } B| ddS )	N)BkBMBGBTBr    
   z.2fr   .1fz.0fPB)abs)r/   units     r,   _format_bytesr;   \   s    -  q66D==2vv'''''''S'''''''##T#####	T	<<<<r.   c                       e Zd ZdZdZddededdfdZdd	Zdd
Z	deddfdZ
deddfdZdeddfdZdee         ddfdZdeddfdZdeddfdZddZddZddZddZdefdZdefdZdefdZdefdZdS ) _LiveDisplayuR  Three-line live progress display on stderr::

        Preparing   ████████████████████  11,100 / 11,100 ✓
        Uploading   ██████████████░░░░░░  580 / 603 files  3.8GB · 19.7MB/s
        Committing  ██████████████████░░  10,800 / 11,100  14 commits

    A small renderer thread redraws the three lines in-place every ~0.5 s on a TTY
    (worker threads only update counters under a lock). When stderr is not a TTY,
    it falls back to a periodic ``logger.info`` summary instead.

    Disabling progress bars (e.g. agent output mode) only turns off the TTY renderer:
    the non-TTY log summaries are gated by the logger verbosity alone, so consumers
    tailing stderr during a long upload still see periodic progress.
       Ttotal_filesenabledr&   Nc                    || _         |ot          j                                        | _        | j        p#t
                              t          j                  | _	        t          j                    | _        d| _        t          j                    | _        d | _        d| _        d| _        d| _        t)                      | _        d| _        d| _        d| _        d| _        d| _        d | _        d S )NFr           )_totalsysstderrisatty_ttyloggerisEnabledForr   INFO_active	threadingLock_lock_drawnEvent_stop_event_thread	_prepared_ignored
_xet_totalset	_xet_done
_committed_nb_commits
_xet_bytes
_speed_ema_prev_bytes
_prev_time)selfr?   r@   s      r,   __init__z_LiveDisplay.__init__z   s    !3
 1 1 3 3	yEF$7$7$E$E^%%
$?,,04 #&55 (,r.   c                 r   | j         sd S | j        rHt          j                            d| j        dd           t          j                                         n$t                              d| j        dd           t          j
        | j        dd          | _        | j                                         d S )NzFound ,z files to upload
z files to uploadzhf-upload-displayTtargetnamedaemon)rK   rG   rD   rE   writerC   flushrH   inforL   Thread_render_looprR   startr^   s    r,   rk   z_LiveDisplay.start   s    | 	F9 	BJGdkGGGGHHHJKK@@@@@AAA 't/@GZcghhhr.   c                     | j         2| j                                         | j                                          | j        r6| j        5  |                                  d d d            d S # 1 swxY w Y   d S d S N)rR   rQ   rV   joinrG   rN   _redrawrl   s    r,   closez_LiveDisplay.close   s    <#  """L9 	                   	 	s   A**A.1A.r/   c                 f    | j         5  | xj        |z  c_        d d d            d S # 1 swxY w Y   d S rn   )rN   rS   r^   r/   s     r,   notify_preparedz_LiveDisplay.notify_prepared   s~    Z 	  	 NNaNN	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	  	    &**c                 f    | j         5  | xj        |z  c_        d d d            d S # 1 swxY w Y   d S rn   )rN   rT   rs   s     r,   notify_ignoredz_LiveDisplay.notify_ignored   s~    Z 	 	MMQMM	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	ru   c                 f    | j         5  | xj        |z  c_        d d d            d S # 1 swxY w Y   d S rn   )rN   rU   rs   s     r,   notify_xet_registeredz"_LiveDisplay.notify_xet_registered   ~    Z 	! 	!OOq OO	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!ru   namesc                 z    | j         5  | j                            |           d d d            d S # 1 swxY w Y   d S rn   )rN   rW   update)r^   r{   s     r,   notify_xet_uploadedz _LiveDisplay.notify_xet_uploaded   s    Z 	) 	)N!!%(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	)s   044c                 f    | j         5  | xj        |z  c_        d d d            d S # 1 swxY w Y   d S rn   )rN   rX   rs   s     r,   notify_skippedz_LiveDisplay.notify_skipped   rz   ru   n_filesc                     | j         5  | xj        |z  c_        | xj        dz  c_        d d d            d S # 1 swxY w Y   d S )Nr   )rN   rX   rY   )r^   r   s     r,   notify_commitz_LiveDisplay.notify_commit   s    Z 	" 	"OOw&OO!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"s   !6::Callable | Nonec                 N      j         sdS ddt          dt          ddf fd}|S )aS  Progress callback for one ``new_upload_commit``.

        The byte counters in ``group_report`` are cumulative *per upload-commit* and several
        upload-commits can be in flight at once (one finalizing, one filling), so each commit
        gets its own closure tracking its own previous value; increments are summed globally.
        Nr   group_reportitem_reportsr&   c                 @   j         5  | j        }xj        t          d|z
            z  c_        ||                                D ]<}|j        dk    r/|j        |j        k    rj                            |j	                   =	 d d d            d S # 1 swxY w Y   d S Nr   )
rN   total_transfer_bytes_completedrZ   maxvaluestotal_bytesbytes_completedrW   add	item_name)r   r   	completeditemprevr^   s       r,   callbackz/_LiveDisplay.new_xet_callback.<locals>.callback   s     ; ;(G	3q)d*:#;#;; (//11 ; ;D'!++0DHX0X0X**4>:::;	; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ; ;s   A<BBB)rK   r   )r^   r   r   s   ` @r,   new_xet_callbackz_LiveDisplay.new_xet_callback   sY     | 	4	;3 	;c 	;d 	; 	; 	; 	; 	; 	; 	; r.   c                    d}| j                             t                    s| j        5  |                                  | j        r|                                  n^t          j                    |z
  t          k    r?t                              |                                            t          j                    }d d d            n# 1 swxY w Y   | j                             t                    d S d S )NrB   )rQ   wait_REFRESH_INTERVALrN   _update_speedrG   rp   time	monotonic_NON_TTY_LOG_INTERVALrH   rh   _summary)r^   last_logs     r,   rj   z_LiveDisplay._render_loop   s   "''(9:: 	0 0 0""$$$9 0LLNNNN^%%04IIIKK000#~//H0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 "''(9:: 	0 	0 	0 	0 	0s   BCCCc                     t          j                    }| j        G|| j        k    r<| j        | j        z
  || j        z
  z  }| j        dk    r|nd|z  d| j        z  z   | _        || _        | j        | _        d S )Nr   g333333?gffffff?)r   r   r]   rZ   r\   r[   )r^   nowrates      r,   r   z_LiveDisplay._update_speed   s~    n?&3+@+@Od&663;PQD&*o&:&:ddd
SSWSbMb@bDO?r.   c                    | j         r(t          j                            d| j         d           t          j                    j        }|                                 | 	                                | 
                                fD ]M}t          |          |dz
  k    r|d |dz
           dz   n|}t          j                            d| d           Nt          j                                         d| _         d S )	Nz[Ar      z...z[K
T)rO   rD   rE   rf   _N_LINESshutilget_terminal_sizecolumns_line_preparing_line_uploading_line_committinglenrg   )r^   r%   line	truncateds       r,   rp   z_LiveDisplay._redraw   s    ; 	7J5T]555666(**2))++T-A-A-C-CTEZEZE\E\] 	7 	7D58YY5J5J[uqy[)E11PTIJ5	5556666
r.   c                     | j         | j        k    rdnd}dt          | j         | j                   d| j         dd| j        d| S )N    ✓ z  Preparing     ra    / )rS   rC   r-   )r^   dones     r,   r   z_LiveDisplay._line_preparing   sW    4;66vvBoT^T[ A AooT^oooZ^Zeooimooor.   c           	      H   | j         dk    r6| j        | j        k    rt          dd          nt          dd          }d| dS t	          | j                  }g }| j        dk    r'|                    t          | j                             | j	        dk    r*|                    t          | j	                   d           |rdd
                    |           nd}| j        | j        k    r|| j         k    rd	nd}dt          || j                    d|d
d| j         d
d| | 	S )Nr   r   z  Uploading   z  -z/sr   u    · r   r   ra   r   z files)rU   rS   rC   r-   r   rW   rZ   appendr;   r[   ro   )r^   barn_donepartsextrar   s         r,   r   z_LiveDisplay._line_uploading   s?   ?a $$+ = =$q!***41::C,C,,,,T^$$?QLLt77888?QLLM$/::>>>???-2:)V[['')))4;666T_;T;TvvZ\tVT_ = =tttttdotttgltnrtttr.   c                     | j         | j        z
  }| j        dk    rd| j         dnd}| j        |cxk    rdk    rn ndnd}dt	          | j        |           d| j        dd	|d| | S )
Nr   r   z commitsr   r   r   z  Committing  ra   r   )rC   rT   rY   rX   r-   )r^   	effectivecommits_strr   s       r,   r   z_LiveDisplay._line_committing	  s    K$-/	9=9IA9M9M54+5555SUI999999999vvrwT$/9==wwwwwW`wwdowquww	
r.   c                     d| j         dd| j        ddt          | j                  dd| j        ddt          | j                   d| j        dd| j         dS )	NzUploading... ra   /z files checked, z uploaded (z transferred), z committed in z
 commit(s))	rS   rC   r   rW   rU   r;   rZ   rX   rY   rl   s    r,   r   z_LiveDisplay._summary  s    MDNN M Mt{N M M4>""tM M'+tM MERSWSbEcEcM MLM M040@M M M	
r.   )Tr&   N)r&   r   )__name__
__module____qualname____doc__r   r)   boolr_   rk   rq   rt   rw   ry   liststrr~   r   r   r   rj   r   rp   r   r   r   r    r.   r,   r=   r=   h   s         H- -C -$ -$ - - - -2	 	 	 	                   !s !t ! ! ! !)c )t ) ) ) )! ! ! ! ! !"S "T " " " "
   2	0 	0 	0 	0+ + + +   p p p p pu u u u u
# 
 
 
 

# 
 
 
 
 
 
r.   r=   c                   R    e Zd ZdZd
dZedefd            ZdededdfdZ	d
d	Z
dS )_CommitPacerzOAdaptive number of files per commit, to stay below server-side commit timeouts.r&   Nc                     t           | _        d S rn   )INITIAL_COMMIT_SIZE_INDEX_indexrl   s    r,   r_   z_CommitPacer.__init__  s    /r.   c                 &    t           | j                 S rn   )COMMIT_SIZE_SCALEr   rl   s    r,   rc   z_CommitPacer.target  s     --r.   durationnb_filesc                     |t           k     r?|| j        k    r4t          | j        dz   t	          t
                    dz
            | _        d S |t           k    rt          | j        dz
  d          | _        d S d S Nr   r   )TARGET_COMMIT_DURATIONrc   r*   r   r   r   r   )r^   r   r   s      r,   record_successz_CommitPacer.record_success#  sr    ,,,T[1H1HdkAos3D/E/E/IJJDKKK...dkAoq11DKKK /.r.   c                 @    t          | j        dz
  d          | _        d S r   )r   r   rl   s    r,   record_failurez_CommitPacer.record_failure)  s    $+/1--r.   r   )r   r   r   r   r_   propertyr)   rc   floatr   r   r   r.   r,   r   r     s        YY0 0 0 0 . . . . X.2u 2 2 2 2 2 2. . . . . .r.   r   c                       e Zd ZdZddZdS )_BatchzSA group of files destined to a single git commit, with their in-flight xet uploads.r&   Nc                 n    g | _         d| _        d | _        g | _        t	          j                    | _        d S r   )opsregular_bytes
xet_commithandlesr   r   
created_atrl   s    r,   r_   z_Batch.__init__0  s2    -/"##=?!%!1!1r.   r   )r   r   r   r   r_   r   r.   r,   r   r   -  s.        ]]2 2 2 2 2 2r.   r   c                      e Zd Zdddededee         dee         deded	z  d
eez  d	z  ded	z  deded	z  dd	fdZddZ	ddZ
dedd	fdZdedd	fdZddZdedd	fdZdee         dd	fdZdee         dd	fdZddZd	S )_UploadPipelineapir   repo_id	repo_typeadd_operationsdelete_operationscommit_messagecommit_descriptionNtokenrevision	create_prparent_commitr&   c       
            || _         || _        || _        || _        || _        || _        || _        || _        |                    |          | _	        |	pt          j        | _        |
| _        || _        t          | j        d          | _        | j        | _        d | _        d | _        d| _        d | _        t-                      | _        t1          j        d          | _        g | _        t9          j                    | _        t?          tA          |          tC                                 | _"        tG          tH          j%        ||| j        |j&                  }|
r|d	z  }tO                      | _(        || j	        tS          | j	                  d
| _*        d | _+        |D ]g}|j,        dk    rZ|-                                5 }|.                                /                                | _+        d d d            n# 1 swxY w Y    d S hd S )N)r   r   safer   r   )maxsize)r?   r@   )
token_typer   r   r   endpointz?create_pr=1)token_refresh_urltoken_refresh_headerscustom_headersz
.gitignore)0r   r   r   r   r   r   r   r   _build_hf_headersheadersr   DEFAULT_REVISIONr   r   r   r   base_revision_quotedcommit_revision_quotedpr_urlpr_revision
nb_commitslast_commit_infor   pacerqueueQueuebatch_queueerrorsrL   rP   abort_eventr=   r   r   displayr   r   WRITEr   r   xet_sessionr   xet_commit_kwargsgitignore_contentpath_in_repoas_filereaddecode)r^   r   r   r   r   r   r   r   r   r   r   r   refresh_urlopfs                  r,   r_   z_UploadPipeline.__init__9  sB    ",!2,"4
,,5,99 >I$>"*
 %*$-b$A$A$A! '+&?#"&'+59!^^
 ).A(>(>(>+-$?,,#N0C0CQkQmQmMmnnn 6#).\
 
 
  	*>)K*,,!,%)\6t|DD"
 "
 .2  	 	B,..ZZ\\ ?Q-.VVXX__->->D*? ? ? ? ? ? ? ? ? ? ? ? ? ? ? /	 	s   ,G::G>	G>	r   c           	         t          g | j        | j                   t          j        | j        dd          }|                                 | j                                         	 |                                  n6# t          $ r) | j
                                         t                        w xY w	 | j
                                        r|                    d           n3| j                            t"                     |                                 | j                                         | j
                                        r2| j        +t(                              d| j         d| j         d           n# | j
                                        r|                    d           n3| j                            t"                     |                                 | j                                         | j
                                        r3| j        -t(                              d| j         d| j         d           w w w xY w| j        r| j        d	         |                                 S )
Nzhf-upload-committerTrb   r6   timeoutzUpload to pull request zF did not complete. To resume into the same PR, re-run with `revision="z^"` (without `create_pr=True`). Re-running with `create_pr=True` would open a new pull request.r   )r   r   r   rL   ri   _committer_looprk   r  _coordinator_loopBaseExceptionr  rV   r   is_setro   r  put	_SENTINELrq   r   rH   warningr   r  _final_commit_info)r^   	committers     r,   runz_UploadPipeline.run  s   '(W$*@(W4CV(WXXX$D,@G\eijjj		""$$$$ 	 	 	  """	 % &&(( ! r**** $$Y///   L   &&(( T-=-ILdk L L8<8HL L L   &&(( ! r**** $$Y///   L   &&(( T-=-ILdk L L8<8HL L L   -I ; 	!+a. &&(((s   (A= <E< =3B00E< <CIc                    dd l }t                      }t          dt          | j                  t
                    D ]}| j                                        r|                     |            d S | j        ||t
          z            }	 t          || j
        | j        | j        | j        | j        j        | j        | j                   n.# t$          $ r!}ddlm} |                    |            d }~ww xY w| j                            t          |                     |D ]-}|j        r>t2                              d|j         d           | j                            d           H|j        dk    r|xj        |j        j         z  c_        n|j!        5 | j"        j#        dd| j        $                                i| j%        |_!        |j        j&        r|j        j'        (                                n|j)        }tU          |j+        tX                    r(|j!        -                    |j+        ||j        	          }	n.|j!        .                    t_          |j+                  |
          }	|j0        1                    ||	f           | j        2                    d           |j3        1                    |           t          |j3                  | j4        j5        k    sL|j        tl          k    s<to          j8                    |j9        z
  tt          k    r;t          |j3                  dk    r#| ;                    |           t                      }/| ;                    |           d S )Nr   )	additionsr   r   r   r   r   r   r  r   )$_CREATE_COMMIT_NO_REPO_ERROR_MESSAGEzSkipping upload for 'z' (ignored by gitignore rules).regularprogress_callback)sha256rd   )r&  r   )<hf_xetr   ranger   r   PREUPLOAD_BATCH_SIZEr  r  _abort_batchr   r   r   r   r   r   r   r   r  r   hf_apir#  append_to_messager  rt   _should_ignorerH   debugr  rw   _upload_moder   upload_infosizer   r
  new_upload_commitr   r  	is_hashedr&  hexCOMPUTE_SHA256
isinstancepath_or_fileobjbytesstart_upload_bytesstart_upload_filer   r   r   ry   r   r  rc   REGULAR_CONTENT_BYTES_BUDGETr   r   r   MAX_COMMIT_INTERVAL_enqueue)
r^   r'  batchrk   chunker#  r  sha_arghandles
             r,   r  z!_UploadPipeline._coordinator_loop  su   1c$"5668LMM 5	% 5	%E&&(( !!%((('8L0L(LME##"n L L!6!X."n&*&<	 	 	 	 	 +   HHHHHH##$HIII	
 L((U444 % %$ LL!i!i!i!ijjjL//222?i//''2>+>>''''/+M4+;+M , ,.2l.K.K.M.M,QUQg, ,(
 >@^=Upbn377999[a[pG!""4e<< m!&!1!D!D.wR_ "E " " "'!1!C!CCHZD[D[dk!C!l!lM(("f666L66q999	  $$$ 	NNdj&777*.JJJ((5+;;>QQQVYZ_ZcVdVdghVhVhMM%((("HHE?%@ 	es   ?C
C2C--C2r>  c                 D   t          |j                  dk    r%| j        dk    rt          | j                  dk    sd S | j                                        s4	 | j                            |d           d S # t          j	        $ r Y Iw xY w| 
                    |           d S )Nr   r(   r  )r   r   r   r   r  r  r  r  r  Fullr*  r^   r>  s     r,   r=  z_UploadPipeline._enqueue  s    uy>>Q1(<(<TE[A\A\_`A`A`F"))++ 	 $$UC$888:   %     s   A6 6BBc                 n    |j         -	 |j                                          d S # t          $ r Y d S w xY wd S rn   )r   abort	ExceptionrE  s     r,   r*  z_UploadPipeline._abort_batch  sU    ' &&(((((    ('s   $ 
22c                    	 	 | j                             d          }n1# t          j        $ r | j                                        rY d S Y Kw xY w|t          u rd S 	 | j                                        s|                     |           n|                     |           n_# t          $ rR}|                     |           | j
                            |           | j                                         Y d }~nd }~ww xY w)NTr"   r  )r  getr  Emptyr  r  r  _process_batchr*  r  r  r   rV   )r^   r>  r@  s      r,   r  z_UploadPipeline._committer_loop  s3   	'(,,S,99;   #**,, FF 	!!''..00 -''....%%e,,,  ' ' '!!%(((""1%%% $$&&&&&&&&'	's)    (AAAB! !
C=+AC88C=c                    |j         |j                                          |j        D ]X\  }}|j        j        s@t
                              |                                j        j	                  |j        _	        d|_
        Y| j                            d |j        D                        g }|j        D ]l}|j        N|j        |j        k    r>t                               d|j         d           | j                            d           W|                    |           mt+          |          dk    s#| j        dk    r/t+          | j                  dk    r|                     |           d S d S d S )NTc                 ~    g | ]:\  }}t          |j        t                    st          |j                  n|j        ;S r   )r6  r7  r8  r   r  ).0r  _s      r,   
<listcomp>z2_UploadPipeline._process_batch.<locals>.<listcomp>  sS       A 4>b>PRW3X3XmC*+++^`^m  r.   zSkipping commit for 'z' (file unchanged).r   r   )r   wait_to_finishr   r0  r3  r8  fromhexresultxet_infor&  _is_uploadedr  r~   r   _remote_oid
_local_oidrH   r.  r  r   r   r   r   r   _commit_with_split)r^   r>  r  rB  ops_to_commits        r,   rL  z_UploadPipeline._process_batch  s    '++---#m ' '
F~/ [,1MM&--//:R:Y,Z,ZBN)"&L,, !&     ) 	% 	%B~)bn.M.MYR_YYYZZZ++A...  $$$$ }!!do&:&:s4CY?Z?Z]^?^?^##M22222 ';&:?^?^r.   r   c                    	 |                      |           d S # t          $ r}| j                                         t	          |          t
          d         k    r t                              dt	          |           d|d           | j        j        }t          dt	          |          |          D ]"}| 
                    ||||z                       #Y d }~d S d }~ww xY w)Nr   z
Commit of z files failed (z). Retrying in smaller chunks.)
_do_commitrH  r  r   r   r   rH   r  rc   r(  rY  )r^   r   r@  rc   rk   s        r,   rY  z"_UploadPipeline._commit_with_split'  s   		EOOC      	E 	E 	EJ%%'''3xx,Q///NNdCdddddeeeZ&Fq#c((F33 E E''EEFN,B(CDDDDE E E E E E	Es    
C!B3CC!c                    | j         r| j        | j                            | j        | j        | j        | j        | j                  }|j	        t          d          |j        | _        |j	        | _        t          |j	        d          | _        t          |          }| j        dk    rt          | j                  |z   }| j        dk    r| j        n| j         d| j        dz    d}t%          j                    }t)          |i || j        pd| j        | j        | j        | j        | j        j        | j        dk    r| j        nd d	
          | _        t%          j                    |z
  }| j                            |t7          |                     | xj        dz  c_        |D ]	}d	|_        
| j                            t7          |                     t>                               dt7          |           d|dd| j        j!                    d S )N)r   titler   descriptionr   zCServer did not return a git reference for the created pull request.r   r   r   z (part r   )T)
operationsfiles_to_copyr   r   r   r   r   r   r   r   retry_on_errorz
Committed z file(s) in r7   zs: )"r   r   r   create_pull_requestr   r   r   r   r   git_reference
ValueErrorurlr   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r   _is_committedr  r   rH   r.  
commit_url)r^   r   prra  r   t0r   r  s           r,   r\  z_UploadPipeline._do_commit3  s(   > 	Kd.6 --)j 3. .  B ' !fggg&DK!/D*/0@r*J*J*JD' $S		
?ad455
BJ $(?a#7#7D@S=r=r\`\kno\o=r=r=r 	 ^ !-!)#6<"nLL0X&0410D0D$,,$!
 !
 !
 >##b(
!!(CHH5551 	$ 	$B#B""3s88,,,k#c((kkkkkI^Iikklllllr.   c                    ddl m} | j        t                              d           | j                            | j        | j        | j	                  }| j        j
        }| j        t          j        k    r| d| j         d} || d| j         d|j         | j        | j        pd|j        | j        j
        	          S | j        dk    r#t                              d
| j         d           | j        H || j        j        | j        j        | j        j        | j        j        | j        | j        j
                  S | j        S )Nr   )r   zPNo files have been modified since last commit. Skipping to prevent empty commit.)r   r   r   r   sz/commit/r   )ri  r   r   oid	_endpointzUpload completed in z	 commits.)ri  r   r   rn  r   ro  )r+  r   r  rH   r  r   	repo_infor   r   r   r   r   REPO_TYPE_MODELshar   r   r   rh   r   ri  rn  )r^   r   rh   
url_prefixs       r,   r  z"_UploadPipeline._final_commit_infoj  sh   &&&&&& ( NNmnnn8%%dldn_c_l%mmD*J~!::: *>>T^>>>
:(KK4<KKKK#2#'#:#@bH(+    ?QKKItIIIJJJ;":0;#4C#'#8#K)-{(+    $$r.   )r&   r   r   )r   r   r   r   r   r	   r
   r   r_   r   r  r   r=  r*  r  rL  rY  r\  r  r   r.   r,   r   r   8  s       GG 	G
 G /0G   56G G  $JG TzD G *G G TzG 
G G G GR) ) ) )B: : : :x
!f 
! 
! 
! 
! 
!& T    ' ' ' '(3F 3t 3 3 3 3>
Ed+=&> 
E4 
E 
E 
E 
E3md#56 3m4 3m 3m 3m 3mn% % % % % %r.   r   F)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   c       
         Z    t          | |||||||||	|
                                          S )zUpload a prepared list of operations through the streamed multi-commit pipeline.

    Requires `hf_xet` to be installed. See module docstring for the architecture.
    )
r   r   r   r   r   r   r   r   r   r   )r   r   )r   r   r   r   r   r   r   r   r   r   r   s              r,   pipelined_uploadru    sG    & %+%-#   
ceer.   )<r   r  r   rD   rL   r   typingr   r   r   urllib.parser   r   r   _commit_apir	   r
   r   r   r   r  r   utilsr   r   
utils._xetr   r   r   r   r   r+  r   r   
get_loggerr   rH   r)  r   r   r   r<  r;  objectr  
_BAR_WIDTHr   r   r   r)   r   r-   r;   r=   r   r   r   r   r   ru  r   r.   r,   <module>r~     s   $   



      / / / / / / / / / /                          , + + + + + 6 6 6 6 6 6 6 6               *))))))))		H	%	%   EDD      1 FHH	 
   5? 5 5% 5 5c 53 5 5 5 5	U 	s 	 	 	 	n
 n
 n
 n
 n
 n
 n
 n
b. . . . . . . .(2 2 2 2 2 2 2 2P% P% P% P% P% P% P% P%v
 &*# $  	  	
 +, 12  d
 : Dj  :      r.   