
    h&                       d dl mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ ddlmZmZmZmZmZ ddlmZmZmZmZ ddl m Z  dd	l!m"Z"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZT 	 d dlUmVZV e+rd dlWZWdZXerddlYmZZZ dZ[ e\e[j                  d            Z^dZ[ e\e[j                  d            Z_dZ[ e\e[j                  d            Z`dZadZb ej                  d      Zd G d dee      Zf G d dee      Zg G d dee      Zhd Zid Zjd  Zky#  Y xY w)!    )print_functionunicode_literalsN)deepcopy)Queue   )ANYWINPY2TYPE_CHECKINGWINDOWSE)LEELOO_DALLASSEESLOGVFSAuthSrv)bos)vf_bmapvf_cmapvf_vmap)Fstab)MParserMTag)*HAVE_SQLITE3SYMTIME
VF_CAREFULDaemonMTHashPebkacProgressPrinterabsrealalltraceatomic_move	db_ex_chkdir_is_emptydjoinfsencgen_filekeygen_filekey_dbggziphidedir	humansizemin_expathmodquotep	rand_nameren_openrmdirs	rmdirs_uprunhookrunihooks2hmss3decs3encsanitize_fnsfsencspackstatdirub64enc
unhumanizevjoinvsplitw8b64decw8b64encwunlink)Path   )SvcHubz[avif,avifs,bmp,gif,heic,heics,heif,heifs,ico,j2p,j2k,jp2,jpeg,jpg,jpx,png,tga,tif,tiff,webp,zm3gp,asf,av1,avc,avi,flv,m4v,mjpeg,mjpg,mkv,mov,mp4,mpeg,mpeg2,mpegts,mpg,mpg2,mts,nut,ogm,ogv,rm,vob,webm,wmvznohash noidx xdev xvol z\cannot receive uploads right now;
server busy with %s.
Please wait; the client will retry...aK  you could try moving the database to another location (preferably an SSD or NVME drive) using either the --hist argument (global option for all volumes), or the hist volflag (just for this volume), or, if you want to keep the thumbnails in the current location and only move the database itself, then use --dbpath or volflag dbpath)
r   rG   r   r   r   r   r   r   r   c                       e Zd Zd Zy)Dbwc                 <    || _         || _        || _        || _        y N)cnnft)selfrL   rM   rN   rO   s        C/jellyfin/media/venv/lib/python3.12/site-packages/copyparty/up2k.py__init__zDbw.__init__h   s        N)__name__
__module____qualname__rR    rS   rQ   rI   rI   g   s    rS   rI   c                       e Zd ZdZd Zy)Mpqezpending files to tag-scanc                 J    || _         || _        || _        || _        || _        y rK   )mtpentagswabspathoth_tags)rP   r[   r\   r]   r^   r_   s         rQ   rR   zMpqe.__init__r   s'      rS   N)rT   rU   rV   __doc__rR   rW   rS   rQ   rY   rY   o   s
    #!rS   rY   c                      e Zd Zd Zd Zd Zd Zd Zd Zd Z	dpdZ
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!	 dpd Z"d! Z#d" Z$d# Z%d$ Z&d% Z'd& Z(d' Z)d( Z*d) Z+d* Z,d+ Z-d, Z.d- Z/d. Z0d/ Z1d0 Z2d1 Z3d2 Z4d3 Z5d4 Z6d5 Z7d6 Z8d7 Z9d8 Z:d9 Z;d: Z<d; Z=d< Z>d= Z?d> Z@d? ZAd@ ZBdA ZCdB ZDdC ZEdD ZFdE ZGdF ZHdG ZIdH ZJdI ZKdJ ZLdqdKZMdL ZN	 	 	 	 	 drdNZOdO ZPdP ZQdQ ZRdR ZSdS ZTdT ZUdU ZV	 dsdVZWdW ZX	 dsdXZYdY ZZdZ Z[d[ Z\d\ Z]d] Z^d^ Z_d_ Z`d` Zada Zbdb Zcdc Zd	 dtddZede Zfdf Zgdg Zhdh Zidi Zjdj Zkdk Zl	 dsdlZmdm Zndn Zodo ZpyM)uUp2kc                    || _         |j                  | _        |j                  | _        |j                  | _        | j                  j
                  | _        | j                  j                  | _        | j                  j                  | _        | j                  j                  | _        | j                  j                  | _
        t        j                  d      | _        d| _        d| _        d| _        d| _        t%        j&                         | _        t%        j&                         | _        d| _        d| _        d | _        d | _        t%        j4                         | _        t9               | _        d| _        t%        j&                         | _        i | _         i | _!        i | _"        i | _#        i | _$        i | _%        i | _&        i | _'        i | _(        i | _)        d | _*        i | _+        i | _,        g | _-        t]               | _/        t]               | _0        t%        j4                         | _1        t%        j&                         | _2        d| _3        d| _4        d| _5        t        j                  d      | _6        d| _7        d| _8        g | _9        i | _:        d | _;        d | _<        d| _=        t}        t        | j                  j                  d      dz        d	z   | _A        t9               | _B        t        r| j                  d
      | _;        | jv                  j                  d       t        t        j                  j                  d      D cg c]  }t}        |       c}      | _<        | jx                  dk  rd| _=        nd}| j                  |d       t        | j                  | j                        | _K        | j                  j                  r| j                  nt        | _O        | j                  j                  dk  rd | _Q        n$t        | j                  j                        | _Q        | j                  j                  r| j                          y y c c}w )Nz^[0-9a-zA-Z_-]{44}$r   F        z(?:^|,)i([0-9]+)T2   g333333?r   z:memory:zcreate table a (b text).)   	   z>could not initialize sqlite3, will use in-memory registry onlyrg      )Uhubasrvargsloglog_funcvfsacctiacctgrpswarksaltsaltrecompiler_hashgidgt0gt1stop	threadingLockmutexreload_mutexreload_flag	reloadingblockedpp	Conditionrescan_condsetneed_rescandb_act	reg_mutexregistryflags	droppable	volnfilesvolsizevolstatevol_actbusy_aps	dupesched	snap_prevmtagr\   mtp_parserspending_tagsr   hashqtagq	tag_eventhashq_mutexn_hashqn_tagq
mpool_usedxiu_ptnxiu_busy
xiu_asleep
fx_backlogcurmem_cur
sqlite_verno_expr_idxintmax	srch_timetimeoutspoolsr   _orzexecutetuplesqlite3sqlite_versionsplitr   fstablog_fk_gen_fkr&   gen_fkhash_mtmthr   no_fastbootdeferred_init)rP   rj   xrO   s       rQ   rR   zUp2k.__init__   s(   XX	HH	99==IINN	YY__
IINN	II&&	jj!67	^^%
%NN,$..0E")
	 !#387
 !7	",,.$>>+zz"56  3tyy22B7#=>Bu99Z0DLLL  !;<#W5K5K5Q5QRU5V$WSV$WXDO'#' PAHHQN4==$))4
&*ii&6&6dllK99q DHdii//0DH99    ! %Xs    Qc                 ^    | j                   j                  ry t        | j                  d       y )Nzup2k-deferred-init)rl   r   r   r   rP   s    rQ   	init_volszUp2k.init_vols   s#    99  t!!#78rS   c                     t        j                          | _        | j                  rd| j                  _        d | _        y y )NT)timerz   r   endr   s    rQ   unppz	Up2k.unpp   s-    99;77DGGKDG rS   c                     |rdnd}| j                   5  | j                  |k  r|| _        d d d        | j                  5  | j                  j                          d d d        y # 1 sw Y   9xY w# 1 sw Y   y xY w)Nri   r   )r   r   r   
notify_all)rP   rescan_all_volsrM   s      rQ   reloadzUp2k.reload   sv     Aa 	%!##$ 	%  	*'')	* 	*	% 	%	* 	*s   A"A."A+.A7c                     | j                   r"t        j                  d       | j                   r"	 | j                  5  | j                  s	 d d d        d| _        y | j                  dk(  }d| _        d d d        | j
                  }| j                  5  | j                         d d d        || j
                  k(  s| j                   r2t        j                  d       || j
                  k(  r%| j                   r2# 1 sw Y   xY w# 1 sw Y   cxY w)N皙?ri   r   F)	r   r   sleepr   r   rz   r~   _reloadr   )rP   ravrz   s      rQ   _reload_thrzUp2k._reload_thr   s    ggJJsO gg"" %''%  &&!+#$ 	%
 ((C "S!"/TWW

3 /TWW % %" "s   C8C8D8DDc                    | j                  dj                  | j                  dz                | j                  j                  j
                  }| j                  5  |j                         D cg c]   \  }}|j                  | j                  vs|" }}}ddd       |rt        |j                               }| j                  |dd       yc c}}w # 1 sw Y   ?xY w)mutex(main) mezreload #{} scheduledr   NTF)rm   formatrx   rk   ro   all_volsr   itemsrealpathr   listkeys_rescan)rP   r   r   kv	scan_volss         rQ   r   zUp2k._reload   s    '..txx!|<=99==))^^ 	&nn.a!**DMM2QI 	
 X]]_-IXy$6	 	s$   C. CCCCCc                 B   | j                   j                  j                  }| j                  |g d      }| j                  rK| j
                  5  | j                  5  | j                          d d d        d d d        | j                          y | j                  s3| j                  j                  dk(  r| j                  j                         S | j                  j                  ry t        | j                   d       |rt        | j"                  d       t        | j$                  d       | j&                  rTt)        t+        d| j                  j,                              D ]'  }t        | j.                  dj1                  |             ) y y y # 1 sw Y   xY w# 1 sw Y   $xY w)NFidxzup2k-snapshotzup2k-hasherzup2k-rescanr   z	tagger-{})rk   ro   r   init_indexesr{   r~   r   _drop_cachesr   r   rl   exitrj   sigtermis_dutr   	_snapshot_hasher_sched_rescanr   ranger   mtag_mt_taggerr   )rP   r   have_e2drM   s       rQ   r   zUp2k.deferred_init  s:   99==))$$Xr5999  $T^^ $!!#$ $ IIKww499>>U288##%%88??t~~/4<</4%%}5yys1dii&7&789 @A4<<););A)>?@  $ $ $ $s$   FF*FF	FFc                 L    | j                   r|dz  }| j                  d||       y )Nz[Kup2k)r   rn   )rP   msgrL   s      rQ   rm   zUp2k.log)  s"    778OCfc1%rS   c           	      `    t        |||||| j                  | j                  j                        S rK   )r'   rm   rl   r   )rP   algrt   fspathfsizeinodes         rQ   r   zUp2k._gen_fk/  s,    vueTXXtyy7G7G
 	
rS   c                 <    || _         | j                  d|z   d       y )Nz#uploads temporarily blocked due to rg   )r   rm   )rP   whys     rQ   _blockzUp2k._block4  s    6<a@rS   c                 j    | j                   'd | _         | j                  s| j                  dd       y y y )Nzuploads are now possibleri   )r   r{   rm   r   s    rQ   _unblockzUp2k._unblock8  s3    <<#DL993Q7  $rS   c                    d}g }| j                   j                   }d}t        rdn| j                  j	                  d      }|rz	 |r| j
                  j                         ng D ](  }	 ||j                  |      j                         d   z  }* |r|r| j                  |      }| j                  j                          n&d}|r"dddt        j                         dfg}t        rg }|j                  d	
       | j                  t        | j                        | j                   | j"                  ||dj%                  | j&                        dj%                  t)        dt        j                         | j&                  z
              d}	t+        j,                  |	d      S #  Y <xY w# | j                  j                          w xY w)Nr   )select count(w) from mt where k = 't:mtp'F      ?r   z(?)r   z"cannot show list (server too busy)Treversez{:.2f}i[&)r   scanningr   r   mtpqupsdbwudbwtz,
: 
separators)rl   
no_up_listr	   r~   acquirer   valuesr   fetchone_active_uploadsreleaser   sortr   boolr   r   r   r   r   minjsondumps)
rP   get_qunamer   r   up_enqgot_lockr   rets
             rQ   	get_statezUp2k.get_state>  s}   II(((75TZZ%7%7%7%D	%05488??,2 CA 7 7 9! <<
 U..u5C

""$D1a.RSTC TWW\\KKOODKK0OO+TYY[4;;-FG
 zz#-887 

""$s$   !F" '%FF" FF" "F>c                 (   g }| j                   j                  j                  |      xs g D ]  }| j                   j                  j                  |      }|s+|j                  }| j
                  j                  |      }|sU|j                         D ]|  }t        |d         }t        |d         }	||	k(  s|s'|d   }
||	z  |d   t        |j                  d|
            t        |
      t        ||d   |d         f}|j                  |       ~  |S )Nneedhashpokesizet0cprelname)ro   areadgetr   r   r   r  lenr   r$   append)rP   r  r  vtopro   ptoptabjobineedihashr  zts               rQ   r  zUp2k._active_uploadsf  s   HHNN&&u-3 	D((##''-C<<D--##D)Czz| CK(CK(E>6{EMKt,-I$FS[9 

2	. 
rS   c                    	 t         r|j                  dd      }|t        |      d  j                  d      }t	        |      \  }}| j
                  5  | j                  |   }|j                         D ]5  }|d   |k(  s|d   |k(  st        j                  |d      c cd d d        S  	 d d d        y# 1 sw Y   yxY w#  Y yxY w)N\/r  r  r   r   z{})
r   replacer  stripr>   r   r   r  r
  r  )rP   r   apvpdnfntab2r"  s           rQ   find_job_by_apzUp2k.find_job_by_ap  s    	ZZc*CIK&&s+BBZFB I}}T*;;= IC6{b(S[B->#zz#-HH	I III I 	sB   AB? +B3=B3B3	B? (B3*B? 3B<8B? <B? ?Cc                    t         s| j                  j                  d      sddiS g }t        |xs ddg      }d}	 | j                  j                         D ]  \  }}| j                  j                  |i       j                  dd      }|s5|d	v r|xs dnd
}	|dv r|nd }
|j                         D ]v  }d|v s|
r|d   |
vs
|	r	|	|d   k7  rt        |d         t        |d   |d   |d         |d   t        |d         t        |d         f}|j                  |       |dz  }|rv   	 | j                  j                          t        r3|D cg c](  }|d   |d   j                  dd      |d   |d   |d   f* }}|j!                  d       |D cg c](  \  }}}}}|dt#        |      z   d|dz  |xs dz  z
  |d* }}}}}}d|iS # | j                  j                          w xY wc c}w c c}}}}}w ) Nri   r   r   r   
*i  u2abortr   ri    r   rg   doneuseraddrt0r  r  r  r  r  r  r   r'  r(  rg      Tr   d   )atr,  pdszf)r	   r   r  r   r   r   r   r  r  r   r$   r  r  r  r   r)  r  r-   )rP   r  ipr  usersetrM   r   r/  cfgr:  r9  r"  zt5r   r>  r,  r@  nnnhret2s                       rQ   get_unfinished_by_userzUp2k.get_unfinished_by_user  s-   dnn,,Q,7q>!,-	%"mm113 
djjnnT2.229a@'*f}
d""%-wT;;= C# S[%< TS[%8 CIc&k3v;FDFCK(CK(C JJsOFA#2 NN""$NQRAaD!A$,,tS11Q41qtDRCR ),
 
 %RR F2J&b3hBG!45	
 
 T{ NN""$ S
s   C(G &G -G/-G4G,c                    t         s| j                  j                  d      syi }	 | j                  j	                         D ]V  \  }}d}d}|j                         D ]3  }d|v r|dz  }	 |t        |d         |d   z  t        |d	         z  z  }5 ||f||<   X 	 | j                  j                          t        j                  |d
      S #  Y uxY w# | j                  j                          w xY w)Nr   r   r6  r   r8  r   r  r  r  r   r   )
r	   r   r  r   r   r  r  r  r
  r  )rP   r  r   r/  nbytesnfilesr"  s          rQ   get_unfinishedzUp2k.get_unfinished  s    dnn,,S,9	%"mm113 -
d;;= 	C} aKF#c&k"2S["@CFDT"TT	 $V,D	-  NN""$zz#-88 NN""$s$   AC )&CC CC C/c                 h    | j                   5  | j                  |      cd d d        S # 1 sw Y   y xY wrK   )r   _get_volsize)rP   r   s     rQ   get_volsizezUp2k.get_volsize  s,    ^^ 	+$$T*	+ 	+ 	+s   (1c                     g }| j                   5  |D ]"  }|j                  | j                  |             $ 	 d d d        |S # 1 sw Y   |S xY wrK   )r   r  rO  )rP   ptopsr  r   s       rQ   get_volsizeszUp2k.get_volsizes  sR    ^^ 	4 4

4,,T234	4 
		4 
s   (AAc                 ,   d| j                   j                  |i       vry| j                  |   }| j                  |   }| j                  |   }t        | j                  j                  |i       j                               D ]  }||d   z  }|dz  } ||fS )Ne2ds)r   r   r  r   )r   r  r   r   r   r   r   r  )rP   r   r   rK  rL  js         rQ   rO  zUp2k._get_volsize  s    b11hhtnc"$dmm''b188:; 	AaiFaKF	 rS   c                 n    | j                   5  | j                  ||||      cd d d        S # 1 sw Y   y xY wrK   )r~   r   rP   r   r   waitfscans        rQ   rescanzUp2k.rescan  s6     ZZ 	B<<)T5A	B 	B 	Bs   +4c                     |s| j                   ry| xj                  dz  c_        t        | j                  dj	                  |r|d   nd      |||| j                  f       y)r   z,cannot initiate; scan is already in progressr   zup2k-rescan-{}r   allr6  )r   rx   r   r   r   rX  s        rQ   r   zUp2k._rescan  sY     AA##IIaL5Iy%2	

 rS   c                    i }t        j                          dz   x}}| j                  st        j                          }t        ||      }|t        j                          z
  }| j                  5  | j                  j	                  |       d d d        | j                  ry | j
                  5  | j                  r9| j                  s-d| _        d| j                  fz  }t        | j                  |       d d d        t        j                          }||k  r|}| j                  r|dz   }
|dz   }| j                  j                  r| j                  j                  s|dz   }n[| j                         }t!        | j#                         |      }	 | j                  j                  rt!        | j%                         |      }	 t!        ||| j+                         z         }| j.                  5  t1        | j2                  j4                  j7                               D ]`  \  }	}
|
j8                  j;                  d      }|s$|	|vr|||	<   ||	   |z   }||k  r| j<                  j?                  |	       t!        ||      }b 	 d d d        | j@                  || j                  j@                  z
  kD  rk| j<                  r_| j@                  | j                  j@                  z   }| j<                  r|}||k  r'|}d}| j)                  |jC                  ||z
               | j.                  5  tE        t1        | j<                              }| j<                  jG                          d d d        r_|dz   }| jI                  | j2                  j4                  |dd      }|r$|D ]  }| j<                  j?                  |        |D ]  }|||<   	 | j                  sy y # 1 sw Y   ExY w# 1 sw Y   xY w# t&        $ r2}t!        ||dz         }d}| j)                  ||fz  d       Y d }~cd }~ww xY w# t&        $ r)}d	t-        |      v r| j)                  d
       Y d }~y  d }~ww xY w# 1 sw Y   xY w# 1 sw Y   xY w)Ng      @Tzup2k-reload-%dr   rg   )#  <   z'could not check for expiring shares: %rzclosed cursorzsched_rescan: lost dbscanz;volume rescan deferred {:.1f} sec, due to database activity
   F)%r   r{   r   r   rY  r   r   r   rx   r   r   r   rl   no_lifetimeshr_check_lifetimesr	  _check_forget_ip_check_shares	Exceptionrm   
_check_xiustrr~   sortedro   r   r   r   r  r   addr   r   r   clearr[  )rP   volagecooldownr   nowrY  zsexrO   r,  volmaxagedeadlineact_timeoutvolserrr   s                    rQ   r   zUp2k._sched_rescan  s   !YY[3..7))))+C'8,GTYY[(D!! ,  %%d+, yy"" 1##DNN%)DN)TXXK7B4++R0	1 ))+CX~"ww7QwH yy$$TYY]]* //1d335w?+yy}}"%d&8&8&:G"DgsT__->'>?  5%dhh&7&7&=&=&?@ 5GB YY]]62F! '%(r
%bzF2H3((,,R0!'84G55 {{S499#3#3338H8H"kkDII,<,<<##!G[()GUAHHQXXgm45 )F4#3#345  &&() 8kk$(("3"3T5%H! 0((,,Q/0  $A #F1I$} ))
, ,1 1: ! +!'384GAAHHQ"Y**+  "c"g-HH45	5 58) )sh   ,O)AO0O% 9P# "BQ9Q%OO"%	P .'PP #	Q,QQQQ"%Q/c                    t        j                          }|dz   }t        | j                  j                  j	                               D ]  \  }}|j
                  d   }|s| j                  j                  |j                        }|sA||dz  z
  }t        d      D ]  }d}	|j                  |	      j                         }
|
s n|
d   d   |z
  }|dkD  rt        |||z         } nbd}	|j                  |	|fz         |j                  }|j                  j                          d	}| j!                  ||||j"                  fz          t        ||d
z         } |S )Nr_  	forget_ipr`  ri   z8select ip, at from up where ip > '' order by +at limit 1r   r   z0update up set ip = '' where ip > '' and at <= %dz*forget-ip(%d) removed %d IPs from db [/%s]i  )r   rk  ro   r   r   r   r   r  r   r   r   fetchallr	  rowcount
connectioncommitrm   vpath)rP   rp  r   r,  rs  rt  r   cutoff_r  hitsremainszirO   s                 rQ   rf  zUp2k._check_forget_ipq  sJ   iik*dhh//5578 	.GBYY{+F((,,s||,C6B;&F1X 6N{{1~..0q'!*v-Q;!'3=9GFA	M*\\%%'@fb#))445#6& '39-G=	.@ rS   c           
          t        j                          }|dz   }t        | j                  j                  j	                               D ]  \  }}|j
                  j                  d      }|s%| j                  j                  |j                        }|sMd}t        j                          |z
  }t        |||z         }d}		 | j                  5  |j                  |	|f      j                         }
d d d        
snz|
D ]t  \  }}|j                  d      s|j                  d      rt        ||      \  }}|d|j                  d      }|r|d|}| j!                  t"        d|g dd	       |d
z  }v |r| j%                  d||fz         |dk  rJd}	| j                  5  |j                  |	      j'                         }
d d d        |
st        |||z   ||
d   z
  z
        } |S # 1 sw Y   xY w# 1 sw Y   6xY w)Nr_  lifetimer   z7select rd, fn from up where at > 0 and at < ? limit 100T//r(  r6  Fr   z%d files graduated in /%srb  z2select at from up where at > 0 order by at limit 1)r   rk  ro   r   r   r   r  r   r   r	  r~   r   r{  
startswithr5   r*  
_handle_rmr   rm   r  )rP   rp  r   r,  rs  r  r   nrmru  r  r  rdr.  fvps                 rQ   re  zUp2k._check_lifetimes  s   iik*dhh//5578 *	IGByy}}Z0H((,,s||,CCyy{X-H'3>2GIAZZ B;;q8+6??ADB " 	FB}}T*bmmD.A!&r2B&("-44S9C)+S1OOM2sBeL1HC	 $ 4Ry@A|DA 1{{1~..01 gsX~tAw'GHU*	IX =B B01 1s   "G( G4(G1	4G=	c                    t        j                          }|dz   }| j                  j                  dz  }||z
  }| j                  j                  j                  | j                  j                  j                  d            }|xr |j                  }t        j                  | j                  j                  d      }|j                         }d}	|r$|j                  |	|f      D 
cg c]  }
|
d   	 c}
ng }|r9|r|j                  sJ t        |      }|j                  D cg c]	  }||v s| }}t        |      }|r| j                  d|       |j                  |	|f      D 
cg c]  }
|
d   	 }}
|rf| j                  d	|       |j!                  d
|D 
cg c]  }
|
f c}
       |j!                  d|D 
cg c]  }
|
f c}
       |j#                          |r!t%        | j&                  j(                  dd       d}	|j                  |	d      j+                         \  }|rCt-        |||z         }|j                  |	|dz
  f      j+                         \  }|rt-        ||      }|j/                          |j/                          | j                  j0                  r+| j                  d||t        j                          z
  fz         |S c c}
w c c}w c c}
w c c}
w c c}
w )Nr_  r`  r(  ri   r   z%select k from sh where t1 and t1 <= ?r   zdisabling expired shares zforgetting expired shares zdelete from sh where k=?zdelete from sf where k=?	sharedrop)FFz#select min(t1) from sh where t1 > ?r   znext shr_chk = %d (%d))r   rl   shr_rtro   nodesr  rd  r*  r   connectshr_dbcursorr   r   r  rm   executemanyr~  r   rj   r   r  r	  closeshr_v)rP   rp  r   rt  lowvnactivedbr   r  r   rmzssrq  r   earliests                   rQ   rg  zUp2k._check_shares  s[   iik*!!B&FlXX^^		 3 3C 89__TYY--q9iik37=CKKC623qad32"(("?b'C!xx529"5B5bHHb:;KKC623qad33HHr;<OO6r8J!!8JKOO6r8J!!8JKIIK488??K@1kk!T*335'8f#45G++a#'4==?KXgx0		

99??HH-'DIIK:O0PPQK 4
 6
 4 9K8Js$   $K 	K*K&K
K
;
K
c                    | j                   ryd}t        | j                  j                  j	                               D ]  \  }}|j
                  }| j                  j                  |      }|s1| j                  5  d}|j                  |      j                         }|s
	 d d d        k	 d d d        g }t        D 	cg c]  }	|	d   	 c}	      D ]M  }
|
t        j                         | j                  |   z
  z
  }|dkD  rt        ||      } n|j                  |
       O |sd| _         t        | j                   d||f        y |S # 1 sw Y   xY wc c}	w )Nri   r_  zselect distinct c from iur   Txiu)r   rk  ro   r   r   r   r   r  r~   r   r{  r   r   r	  r  r   	_run_xius)rP   r  r  rs  rpr   r  cdsrun_cdsr   cddeltas               rQ   ri  zUp2k._check_xiu  s@   ==TXX..4467 	FAsB((,,r"C /kk!n--/	   GC0qad01 #diikDLL,<<=19c5/Cr"#  $t~~usGn=1	4 
)  1s   :%E<E
E	c                 R    |D ]  }| j                  ||        d| _        d| _        y )NFT)_run_xiur   r   )rP   rs  r  r  s       rQ   r  zUp2k._run_xius  s/     	#BMM#r"	# rS   c                 6   |j                   }| j                  |   }| j                  5  d}|j                  |j	                  |            j                         }|s
	 d d d        y d}|j                  |j	                  |      |       |j                  j                          d}g }|D ]6  }|j                  ||      j                         }	|	s&|j                  |	       8 	 d d d        | j                  |   d   }
|
D ]  }| j                  j                  |      }|rt        |j                  d            nd}||k7  rB| j!                  dt#              |fz         t%        | j                   | j&                  j(                  ||        y # 1 sw Y   xY w)Nz-select w,rd,fn from iu where c={} limit 80386z6delete from iu where w=? and +rd=? and +fn=? and +c={}z;select * from up where substr(w,1,16)=? and +rd=? and +fn=?r  r   rC   zxiu: %d# %r)r   r   r~   r   r   r{  r  r}  r~  r  r  r   r   searchr   grouprm   r  r3   rl   hook_v)rP   rs  r  r  r   r  wrfsr   wrfupcmdscmdmccds                 rQ   r  zUp2k._run_xiu$  sh   \\hhrl ZZ 	#?A;;qxx|,557D		# 	# IAOOAHHRL$/NN!!#MAC #[[C(113JJrN#	#, zz"~e$ 	@C##C(A%&#aggaj/ACbyHH]c$i%556TXXtyy//c3?	@/	# 	#s   4F%A(FFFc                     dt        |d         dz  t        |d         xs dz  z
  }t        |d   |d   |d         }d	j                  ||      S )
Nr=  r  g      Y@r  r   r   r  r  z{:5.1f}% {})r  r$   r   )rP   r"  percpaths       rQ   _vis_job_progresszUp2k._vis_job_progressI  sZ    c#f+&.#c&k2B2GaHIS[#f+s6{;##D$//rS   c                     g }|j                         D ]+  \  }}|d   s|j                  | j                  |             - |S )Nr  )r   r  r  )rP   regr  r  r"  s        rQ   _vis_reg_progresszUp2k._vis_reg_progressN  sE    iik 	8FAs6{

411#67	8 
rS   c                     | j                   sd|fS |j                         D ci c]  \  }}|j                  d      r|| }}}t        |      t        |      k(  rd|fS d|fS c c}}w )NFe2tT)r   r   r  r  )rP   r   r   r   r  s        rQ   _expr_idx_filterzUp2k._expr_idx_filterV  sk    %< %I1Q\\%5Hq!tIIs8s5z!%<Sy	 Js
   A%A%c           
      l   |s!| j                   5  | j                  }d d d        t        j                         | _        d}	 |dz  }|dkD  rt        j                  d       | j                   5  || j                  k7  r
	 d d d        y| j
                  r
	 d d d        \t        | j                  | j                        | _        | j                  j                  s| j
                  j                          d d d        	 |r| j                  d|fz         | j                  j                  | _        | j                  j                  | _        | j                  j                  | _        | j                  j                   | _        | j                  j"                  xs. | j                  j$                  xs | j                  j&                  }t)        |j+                               }t        j                         }| j,                  rJd}	|D ],  }
| j/                  |
j0                        \  }}|s$||
_        d}	. |	rd}| j                  |d	       g }| j                   5  | j2                  5  |D ]  }
	 t5        j6                  |
j8                  |
j0                  
       t;        | j<                  | j                  j>                   |
j8                         |r|
jD                  |vrv| jG                  |
j8                  |
j0                        s|jI                  |
       |
jD                  | jB                  vsd| jB                  |
jD                  <    	 d d d        d d d        |}i }d}|D ]  }
d|
j0                  v sd} |rN| jJ                  sBtM        | j<                  | j                        | _%        | jJ                  jN                  sd | _%        tQ        d |D        d       r| jS                  d       | j                  jT                  s|D cg c]  }d|j0                  v s| c}rKd| j                  _*        | j                   5  | j2                  5  | jW                          d d d        d d d        |D ]  }
| jX                  s|| j                  k7  r nt[        |
j0                  j]                  di             }|| j^                  |
j8                  <   d|
j0                  v rd}d|
j0                  v s|rLd| jB                  |
jD                  <   | ja                  |
t)        |j+                                     \  }}|rd||
<   d|
j0                  v rd}nd|
j0                  v rd}nd}|| jB                  |
jD                  <   
 | jc                          |D ]  }
| jX                  r nd|
j0                  vr d}|| jB                  |
jD                  <   | j                  dje                  ||
j8                               | jg                  |
      }|r&| j                  dje                  |      d       d||
<   d|
j0                  v rd}nd}|| jB                  |
jD                  <    d}|D ]  }
| jX                  r nd|
j0                  vr d}|| jB                  |
jD                  <   | j                  dje                  ||
j8                               | ji                  |
      \  }}}|sd}|s|rd||
<   d | jB                  |
jD                  <    |D ]  }
| j                   5  | j2                  5  | jG                  |
j8                  |
j0                        }d d d        d d d        \  }}| j                   5  |jj                  jm                          |jo                  d!       d d d         | jX                  ry|j+                         D ]  }
|
j0                  d"   d#k(  r| j                   5  | j2                  5  | jG                  |
j8                  |
j0                        }d d d        d d d        	 \  }}t4        jp                  js                  |d$z         d%k  r	 	 | j                   5  |jo                  d&       	 |jo                  d'       |jj                  jm                          d d d         | jX                  ryd| j
                  _:        d)}| j                  |je                  tw        |      t        j                         |z
               |rd*}| j                  |d	       | jJ                  rd+nd}|D ]  }
|| jB                  |
jD                  <    | jJ                  rty        | jz                  d,|f       |S | j}                          |S # 1 sw Y   	`xY w# 1 sw Y   xY w# t@        $ rG}d| jB                  |
jD                  <   | j                  d|
j8                  d|d	       Y d }~|d }~ww xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   GxY w# 1 sw Y   ?xY w# 1 sw Y   DxY w#  Y xY w#  Y xY w# 1 sw Y   xY w# t@        $ r,}| j                  d(je                  |      d       Y d }~d }~ww xY w)-Nr   Tr   r   Fzreload #%d runningz6disabling -e2t because your sqlite belongs in a museumrg   rL   vfzOFFLINE (cannot access folder)zcannot access r   z OFFLINE (pending initialization)r  c              3   >   K   | ]  }d |j                   v s|  yw)rU  N)r   ).0zvs     rQ   	<genexpr>z$Up2k.init_indexes.<locals>.<genexpr>  s     9f&89s   indexinge2tsrmtee2drU  zbusy (hashing files)e2vz online (integrity-check pending)e2tszonline (tags pending)online, idlezonline (verifying integrity)z{} [{}]zmodified {} entries in the dbzonline (reading tags)zonline (mtp soon)vacuumdbdacid-wali  P zpragma wal_checkpoint(truncate)r~  zcheckpoint failed: {}z{} volumes in {:.2f} seczJcould not read tags because no backends are available (Mutagen or FFprobe)zonline (running mtp)zup2k-mtp-scan)?r~   rx   r   ry   r   r   r   rm   rl   rj   r   startrk   ro   rp   rq   rr   	idp_h_usrchpwrd  r   r  r   r  r   r   r   makedirsr   r#   rn   
no_scandirrh  r   r  register_vpathr  r   r   usablenextr   re_dhashr   r{   r   r  r\   _build_file_indexr   r   _verify_integrity_build_tags_indexr}  r~  r   r  getsizer   r  r   _run_all_mtpr   )rP   r   r   rZ  rx   nspinr   rw  r;  modifiedrs  r  rA  r   	live_volsrr  need_vac	need_mtagr  enr  vacrO   nmodneeded_mutagennaddr  successr  r   db_paths                                  rQ   r   zUp2k.init_indexes`  s	     hh 99;QJEqy

3 	$$((? 	$ 	$ 77	$ 	$ *$((DII>xxGGMMO	$ HH)SF2399==IINN	YY__
IINN	99&&I$))..IDIIMMHOO%&YY[H $,,SYY71 !CI#H	$ N"	ZZ 	R 	R RLL#))< DII4H4H0H#,,W )!;**3<<C  %99DMM1/QDMM#)),)R	R 	R0 	 	!C		! 		! TYYT]]DII6DI99## 	 9d94@KK
#99t!Kw"((7J"!K!&DII $T^^ $!!#$ $  	)CyyC488OSYY]]5"-.B(*DKK%		!"e+Acii(//T(//:K5LM3$(HSM		!6399$+"'(DMM#))$/	)2 	  	)CyyCII%.A'(DMM#))$HHY%%a67))#.D8??EqI $"+"'(DMM#))$+	)0  	;CyySYY&'A'(DMM#))$HHY%%a67!%!7!7!<D#w!%s $':DMM#))$%	;(  	&C CT^^ C))#,,		BC C FC &%%'H%& &	& 99??$ 	@Cyy6) CT^^ C))#,,		BC C"W88##Gf$45G H

@ZZ ,KK ABH- NN))+,	@4 99(CItyy{R'789^CHHSAH&*ii"^ 	)C'(DMM#))$	) 994$$ov>  IIKG 	$ 	$Z ! /ODMM#)),HHs||RHAHN	R 	R 	R 	RR "L$ $ $ $ZC C C C& &C C C C, ,  @077;Q??@sR  e:+ffAfg4 g''A!fA!g'*g'g4hh:hhhh-'h  h-!,h:i#'i
i*i!i>i1'i)8i1i>:ff	g$	<g	g'g$	$g''g1	,g44g>h	hh h*%h--h7	:i	iii	!i&)i.+i11i;	6i>>	j3!j..j3c                    | j                   j                  j                  |      }|s| j                  d|       yt        j
                  j                  |d      }|| j                  v r	 | j                  |   |fS d}| j                   j                  j                         D ]  \  }}|j                  |k(  s|} | j                  |      \  }}t        |j                  d            }	d}
d}d}d	}t        |j                               }t!               }|j#                  t%                      |j#                  t'                      |j                         D ci c]  \  }}||
 }}}|j                         D ci c]V  \  }}||vsJ|t)        | j*                  ||         k7  r1t-        |      t-        t)        | j*                  ||               k7  r||X }}}t%               j                         D ]^  \  }}||vs||v rt-        ||         t-        t)        | j*                  |            k(  r||= Ad
j                  d ||   D              ||<   ` |d   t/        | j*                  j0                  d      k(  r|j3                  d       	 |d   t/        | j*                  j4                  xs dd      k(  r|j3                  d       dD ]  }	 d||   z  ||<    |j                         D cg c]0  \  }}||vr'|du r|
n|du r|n|j7                  |t-        |            2 }}}|sdg}|rUdj                  t9        |            }|j;                  dd      }| j                  dj7                  ||rdndz   |      d       i }d}g }| j*                  j<                  rdnd}t        j
                  j                  |d      }t>        j
                  jA                  |      rtC        jD                  |d      5 }|jG                         jI                  d      }ddd       tK        jL                        }	 |d    }|d!   }|}|r6d"|tO        tQ        |               vr|jS                         D ]
  }|d#   |d"<    g }|j                         D ]  \  }}||d$<   d%|v }|r|x|d&<   |d'<   nd&|vrg |d&<   d'|vrg |d'<   |rtU        ||d(   |d)         }ntU        ||d(   ||d)   z   d*z         }t>        j
                  jA                  |      r |rtW        jV                         |d+<   i |d,<   | j                  d-|       |	r|jY                  |        |D ]  } || =  tO        d. |jS                         D        d      rCd/j                         }!|jS                         D ]   }d%|v s|!D ]  }|j3                  |d        " |)|j                         D cg c]  \  }}|d&   r| }}}n|D  cg c]	  } | |v s|  }} d0j7                  |t[        |j]                               t[        |xs g             }"|"g| j_                  |      z   }#| j                  d1j                  |#             || j`                  |<   d2| jb                  |<   || j                  |<   |xs g | jd                  |<   | jg                  |d       th        rd3|vsd4|v ry	 t?        jj                  |      rtm        |       	 | jq                  |      }%|d7   }&|&d8k(  rd9}d:}'n|&d;k(  rd<}d:}'n|&d=k(  rd<}d>}'nd<}d?}'	 |%js                  d@|z         ju                         dA   }(|(jw                         |jw                         k7  rdB}"to        |"j7                  ||(            	 |%js                  dD|'z          |%jx                  j{                          | j}                  |%|       |%| j                  |<   dA| j~                  |%<   dA| j                  |%<   |%|fS #  Y yxY wc c}}w c c}}w #  Y LxY w#  Y RxY wc c}}w # 1 sw Y   xY w#  Y xY wc c}}w c c} w # tn        $ r#}$d5}"| j                  |"||$fz  d6       Y d}$~$yd}$~$ww xY w# tn        $ r5}$|'d>k7  r%d:}'dC}"| j                  |"j7                  |'|$             Y d}$~$"d}$~$ww xY w#  dE})| j                  |)|t        j                         t        fz  d6       Y yxY w)Fmutex(main,reg) mezno dbpath for Nzup2k.db?noforgetz[0;32m{}{:.0}z[0;35m{}{:.0}z[0;36m{}:[90m{}zrext_th_d html_head put_name2 mv_re_r mv_re_t rm_re_r rm_re_t srch_re_dots srch_re_nodot zipmax zipmaxn_v zipmaxs_vrE   c              3       K   | ]  }|  y wrK   rW   r  r   s     rQ   r  z&Up2k.register_vpath.<locals>.<genexpr>y  s     !4!!4s   chmod_d   chmod_fz-1)r  r  z%oTFz[90mall-defaultrF   z90mre.compile(z90m(z/{} {}r(  r6  35rf   	up2k.snaprbutf-8r   r   dwrkwarkr   r8  r  r  r  r  .PARTIALr  busyzign deleted file in snap: c              3   4   K   | ]  }d |v sd|v s|  yw)r8  r  NrW   r  s     rQ   r  z&Up2k.register_vpath.<locals>.<genexpr>  s     L1!!QLs   	zhost tnam busy sprs poke t0czloaded snap {} |{}| ({})r2  rd   r  d2dz%failed to initialize volume '/%s': %sr   r  r  deletefullswalwalyolooffnormalzpragma journal_mode=r   z,sqlite failed to set journal_mode {}; got {}zreverting to sync={} because {}zpragma synchronous=z5ERROR: cannot use database at [%s]:
%s
[33mhint: %s
)Dro   dbpathsr  rm   osr  joinr   r   r   r   r   r  r  r   r   r   updater   r   getattrrl   rj  r   r  popr  r   rk  r)  dotpartr   existsr(   GzipFilereaddecoder
  loadsr  iterr  r$   r   r  r  r   r  r   r   r   regdropr   r  r)   rh  _open_db_wdr   r  lowerr}  r~  _verify_db_cacher   r   	traceback
format_excHINT_HISTPATH)*rP   r   r   histpathr  r  r   r   r  n4gftfffvrq  fxfdflk1k2ar  drp	emptylistr  snaprA  rV  reg2r"  r  is_donefpr   zslrO   tarr  r   r  syncamoder   s*                                             rQ   r  zUp2k.register_vpathI  s    88##''-HH412'',,x34== xx~w.. HH%%++- 	DAqzzT!	 ((/5599Z()  & B_Y
		')
		')!xxz*tq!ad** 
1{WTYY1..Fc'$))RU";<< qD
 
 ioo' 	5FB|rRx2b6{c'$))R"899rF!4RV!442	5 i=C		 1 1155FF9	)}DII$5$5$=q AAy! ( 	A11	 

1{ 9RU
"BB1c!fM
 

 &'A&)$B,f5BHHX__UUc%CRH$O	**#ww||Hk288??4 tT* -aFFHOOG,- ::a=D;'J' Cd4T
+;&<<;;= .C"%f+CK. B**, %3"F C-099CK#f+S(&(FS(&(FtS[#f+>BtS['CK2G*2TUB88??2& "&))+CK"$CKHHrCD		!1%4  G L

LdS4::<::< -C}!$ -AGGAt,--
 {%(YY[BTQ&	qBB"%2Qcq22*11$CHHJSYTVXAt--c22BHHTYYr]# 

4 T!d"ybtT2uE1Ue^	||H%!-	P""7+C ,Cf} 	1$:R$?@IIKAN;;=BHHJ.FA#AHHR$788 / KK-45NN!!#!!#u- DHHTN !DLL"#DNN3<}$ +
,	
&- -^ C2"  	7AHHQ%_a(	@  15=!D9AHHQXXdB/0	1 	POCHHSGY%9%9%;]KKQOs   ,_< `>A`	;` `.5`8 `%6
`2 `:#`:/	a 9a  a .8b5 'A#a4 
A1b5 <` ``%`/2`7	a1a,,a14	b2=*b-'b5 -b22b5 52c)c           	         |dz   j                  d      }| j                  j                  D cg c]  }|j                  |      s| }}|j	                          |D cg c]  }|t        |      d   }}|D ]  }| j                  j                  |   j                  }|j                         D 	
ci c]  \  }	}
|	t        v s|	|
 }}	}
|j                  t        t        |j                                             t        j                  dj                  |      j                  dd            j!                         }t#        |d d       j%                  d      }|j'                  d      }	 |j)                         \  }||k7  ra|j'                  d	       |j'                  d
       |j'                  d       |j'                  d|f       |j*                  j-                          y y c c}w c c}w c c}
}	w #  d}Y }xY w)Nr(  r2  r  r)     asciiz#select v from kv where k = 'volcfg'r6  z!delete from kv where k = 'volcfg'delete from dhdelete from cvz"insert into kv values ('volcfg',?))lstripro   r   r  r  r  r   r   VF_AFFECTS_INDEXINGr  rj  rk  hashlibsha1r  encodedigestr;   r  r   r  r}  r~  )rP   r   r  prefixr   vpsseedr,  r  r   r   zbvcfgrL   oldcfgs                  rQ   r  zUp2k._verify_db_cache  s   #+%%c*((++DQq||F/CqDD
*-.Q#f+- ..  	1B""2&,,B#%88:J41a6I1I!Q$JBJKKF288:./0	1
 \\$))D/00)DELLNr#2w&&w/KK=>	

IV T>KK;<KK()KK()KK<tgFNN!!# ' E.
 K	Fs)   G&G&G+0G0G0-G6 6G<c                    d}|j                   }|j                  j                  d      }|j                  j                  d      }t        |j                  j                  d            }d|j                  v }t	        j
                  |      }	|j                  j                  d      r|	j                  nd}
| j                  5  | j                  5  | j                  ||j                        }d d d        \  }}t        |ddt        j                               }t        |j                  j                  d            d   | j                  _        |D cg c]|  }||k7  ru|j"                  j%                  |j"                  d	z         s|j"                  sA|j                   d	z   |j"                  t'        |j"                        d  j)                  d	      z   ~ }}||D cg c]  }t+        |       c}z  }|t-        | j.                  j0                  j3                               z  }|t-        | j.                  j4                  j3                               z  }t6        r |D cg c]  }|j9                  d	d
       }}n|j;                  d       t-        |D ci c]  }|d c}      }| j<                  j>                  r0|j                  j                  d       |xj                   dz  c_        t+        |      }dx}}	 tA        | jB                  | j<                  jD                   |      r-d}| jG                  ||j"                  |fz  d       	 d d d        y|jI                         s-d}| jG                  ||j"                  |fz  d       	 d d d        y| jK                  ||tM        |      ||||||g |	|
t        |j                  j                  d                  \  }}}|s| jO                  |j                  ||      }|j                   r*| jG                  d|j\                  |j                   fz         | j<                  j^                  re|j                  j                  d      ja                         r\|j                  j                  d       | jG                  djS                  |             n |s|r| jc                  |j                  d       |j                  jd                  jg                          |j                  j                  d      sI|j                  j                  d      s.| j<                  jh                  rK| j<                  jj                  s4d}|j                  j                  |      ja                         \  }}|xs d}||dz  z  }|| jl                  |j                  <   || jn                  |j                  <   tq        |j                  j                  d      xs d      }tq        |j                  j                  d      xs d      }djS                  ts        |d      ts        |d      ts        |d      ju                  d       ts        |d      ju                  d       |j                         }| jG                  |       dt        |xs |xs |      fcd d d        S # 1 sw Y   xY wc c}w c c}w c c}w c c}w # tP        $ ri}d}| jG                  |jS                  |tU                     d       tW        | jF                  ||      r| jX                  j[                          Y d }~9d }~ww xY w# 1 sw Y   y xY w)!NFnoidxnohashr  fat32xdevr   select count(w) from upr(  r'  )z/devz/procz/runz/sysr   zdelete from dsz^volume /%s at [%s] is empty; will not be indexed as this could be due to an offline filesystem   )TFz;volume /%s at [%s] will not be indexed due to bad landmarksxvolzfailed to index volume [{}]:
{}r  commit %d new files; %d updateszselect d from dhr1  zforgetting dhashes in {}Tvmaxbvmaxnz!select count(sz), sum(sz) from up   0z+{:>5} / {:>5}  ( {:>5} / {:>5} files) in {}B);r   r   r  r  r   statst_devr~   r   r  rI   r   r  rL   r   r   rM   r  r  r  r3  r   r   ro   histtabr  r  r   r)  extendrl   re_dirszr#   rn   r  rm   check_landmarks
_build_dirr   
_drop_lostrh  r   r+   r"   rj   
log_stacksrN   no_dhashr  _set_tagscanr}  r~  statsnos_volr   r   r<   r*   rstrip)rP   rs  r   do_vactopreirehr  ffatcstdevr  r   r  r  dexclr   r   rtopn_addn_rmrO   r  rr  rq  r  vbrH  rI  s                                 rQ   r  zUp2k._build_file_index8  s   lliimmG$iimmH%399==,-#))#hhsmIIMM&1cjjqZZ b	7 :))#syy9: LCS!Q		,BRTT\\*CDEaHDGGI "8!3!3CIIO!DCII s"QWWS^-=%>%E%Ec%JJD 
 .AWQZ..DD))00233DD))00233D6:;		#t,;; =>t,!A,-Dyy!!-.	3<DED*4993G3G/GNxAHHQ#))T!22A6&Gb	7 b	7H **,UAHHQ#))T!22A6&Ob	7 b	7R #ooIv./q! ??244d;D tt:beeRTT]JKyy!!44<< 23<<>DDLL!12HH7>>sCD$!!"$$-DDOO""$ 		g&99==)IIOODII,=,=8b)224BW1b4i%'RTT"')rtt$"399==#9#@SA"399==#9#@SAAHHb$'eT*b$'..s3eT*11#6LL e5t5v66Eb	7 b	7: :
 / <
 -F  *6#vx0A6TXXr73HH'')	*ub	7 b	7s   \:Z$+A'\:BZ1\:Z6,A/\:Z;4\:
[ 
A\:8A[3[A+[9J!\:$Z.	)\:	\7A\2,\:2\77\::]c                    |r*|j                  |      s| j                  d|d|d       y||
v rd}| j                  ||
d   ||fz  d       ydx}x}}|
|gz   }
g }g }d	}d
x}}| j                  j                  }| j                  j                  }d| j
                  j                  |fz  | j
                  _        |t        |      d  j                  d      }t        r!|j                  dd      j                  d      }|r|dz   nd
}|t        j                  z   }t        | j                  | j                  j                    d	|d      }t#        |      }t%        |D cg c]  }d|d   v s|d    c}      }|D ]  \  } }!| j&                  r y|| z   }"|| z   }#|r#|j)                  |#      r|j+                  |"       Dt-        |!j.                        }$t1        j2                  |!j4                        r	 t7        j0                  |#      }!|!j8                  }%|r|	s|!j.                  dz  rd}t1        j:                  |!j4                        rt=        |#      }&|r^|!j>                  |k7  rOt@        r"t7        j0                  |&      j>                  |k(  s'| j                  d|d|!j>                  d|#d       9|#|v s|&|v r|j+                  |"       T| dk(  r?t6        jB                  jE                  t        jB                  jG                  |#d            r	 | jI                  ||||#|&|||||
|!||      \  }'}(})||'z  }||(z  }||)z  }t1        jL                  |!j4                        s"| j                  d|!j4                  |#fz         |"jO                  d      rtQ        jP                         |$z
  dk  r9|%s| d|v sd| d|v rL||%z  }|j+                  |%|$| f       |%si| jS                         }*|*jU                  dd      d   }+|*|v s|sf|+tV        v r^| j                  d      sM|rG|*|vsC|jS                         |vs1|jY                  |*      |jY                  |jS                               k  r| }|r|+tZ        v s| j                  d      r| } |s|}| j                  j\                  s|t        |      z  }d},	 |j^                  ja                  |,|f      jc                         xs d\  }-}.||-k7  s||.k7  rQ|j^                  ja                  d|f       |j^                  ja                  d |||f       |xj                  dz  c_        g }|je                          | j                  jf                  s1t        |      d!k  r/ti        jj                  tm        |      jo                  d"d#            }/nKti        jj                         }/|D cg c],  }|/jq                  tm        |      jo                  d"d#            . }0}|/jq                  |jo                  d"d#             |/jq                  ts        d$|j.                               tu        |/jw                         d d%       jy                  d&      }1d'}2	 |j^                  ja                  |2||1f      }3|}4|3jc                         r|||fS |r|r	 d)},d|v r|jU                  dd      nd
|f\  }5}6|j^                  ja                  |,|5|6|f      jc                         sR|j^                  ja                  d*|5|6f       |j^                  ja                  d+|5|6|f       |xj                  dz  c_        t%        |D cg c]  }|d   	 c}      }8|D ]C  \  }%}$}9| j&                  r y||9z   }"||9z   }#|r|j)                  |#      nd}:d.}2	 |j^                  ja                  |2||9f      }3t        |3j                               };|;r	| j
                  xj                  dz  c_        |;d   \  }<}=}>}?}@t        |;      dkD  rMd/}d0jG                  |;D cg c]  }t        |       c}      }A| j                  |||"t        |;      |Afz         d}=|rt        |=|$z
        dk(  r|$}=|=|$k(  r|>|%k(  r|:s
|<d   d1k7  s|%s$d2}| j                  |||"|=|$|>|%fz         | j                  |j^                  ||9d       |dz  }|xj                  dz  c_        g };nd
}<d
}?d}@d| j
                  j                  |#fz  | j
                  _        |:s|%st        | j                  |%|$||9      }Bnm|%d3kD  r| j                  d4|#       	 | j                  |#d5j                  | j
                  j                              \  }C}0Cs yt        | j                  |%C      }B|<r	|<Bk7  rd
}?d}@| j                  |j^                  i ||9|$|%d
d
B|Bd
d
|?@       |xj                  dz  c_        |xj                  dz  c_L        |dz  }tQ        jP                         |j                  z
  }D|j                  d8k\  sDdk\  s| j                  d9|j                  |j                  fz         |j^                  j                  j                          dx|_        |_L        tQ        jP                         |_M        F | j                  jf                  s;|j^                  ja                  d:4f       |j^                  ja                  d;|41f       | j&                  ry|D ]  }Ed}Fd<},|Ed(t{        |E      z   fD ]5  }G	 G|Gf}H|j^                  ja                  |,|H      jc                         d   }F n FrZd=}| j                  |F|Efz         d>},|j^                  ja                  |,H       d?},|j^                  ja                  |,|H       ||Fz  }d@},|j^                  ja                  |,H        |r|||fS dA},	 |j^                  ja                  |,|f      }3|3D cg c]%  \  }|j                  d(      rt        |dd        n|' }I}|ID cg c]	  }||8vs| }J}t        |J      }K|JD ]   }9| j                  |j^                  ||9d       " Kr | j                  dBj                  K             |||fS c c}w #  Y 1xY w#  d}| j                  ||#tK               fz  d       Y ZxY w#  Y xY wc c}w #  d(t{        |      z   }4|j^                  ja                  |2|4|1f      }3Y ~xY w# t|        $ r&}7| j                  d,|d|d-|7d       Y d }7~7d }7~7ww xY wc c}w #  |j^                  ja                  |2t        | j                  ||9            }3Y xY wc c}w # t|        $ r"}7| j                  d6|7d7|#       Y d }7~7Fd }7~7ww xY w#  Y xY w#  |j^                  ja                  |,d(t{        |      z   f      }3Y xY wc c}w c c}w )CNzskip xvol: z -> rE  )r   r   r   z;bailing from symlink loop,
  prev: %r
  curr: %r
  from: %rrG   rg   r   Tr6  za%d %sr(  r'  FPARTIAL)rG   r   r   ri   z
skip xdev z->r   z.thr\  zfailed to index subdir %r:
%sr   zskip type-0%o file %rr  r`  rf   z(select sz, nf from ds where rd=? limit 1)rG   rG   zdelete from ds where rd=?zinsert into ds values (?,?,?)i(#  r  r)  s   <d   r0  z#select d from dh where d=? and +h=?r  z.select * from cv where rd=? and dn=? and +fn=?"delete from cv where rd=? and dn=?insert into cv values (?,?,?)zcover z	 failed: z8select w, mt, sz, ip, at from up where rd = ? and fn = ?z(WARN: multiple entries: %r => %r |%d|
%rr2  #z)reindex %r => %r mtime(%s/%s) size(%s/%s)   file: za{}, hash:  @    rG  delete from dh where d = ?zinsert into dh values (?,?)z7select count(w) from up where (rd=? or rd like ?||'/%')z3forgetting %d shadowed autoindexed files in %r > %rz.delete from dh where (d = ? or d like ?||'/%')z.delete from up where (rd=? or rd like ?||'/%')z.delete from ds where (rd=? or rd like ?||'/%')zselect fn from up where rd = ?zforgot {} deleted files)Qr  rm   rl   
th_coversdth_coversd_setr   rM   r   r  r*  r   r)  r  sepr:   rn   r  rk  r   r{   r  r  r   st_mtimerM  S_ISLNKst_moder   st_sizeS_ISDIRr   rN  r   r  isdirr  rS  r+   S_ISREGendswithr   r  rsplitICV_EXTSindexVCV_EXTSno_dirszrL   r   r  rm  rV  r5  r6  rj  r7  r  r9   r;   r8  r  r@   rh  r6   r   r   r{  reprabsdb_rmup2k_wark_from_metadatart   _hashlist_from_filer   up2k_wark_from_hashlistdb_addrN   rO   r}  r~  r?   )LrP   r  r\  rc  cdirrcdirr]  r^  r  r_  seenr`  ra  rF  rO   tfatnfrszunregfilesrB  cvvcvth_cvdth_cvdsr  rdscdirsgglr   partialsinameinfr  r^   lmodr@  rapi1i2i3linameextr  db_szdb_nfzhr  dhashsqlrL   drdcrdcdnrr  
seen_filesr.  rA  in_dbdwdtsdszrB  r>  rep_dbr  hashestdsh_rdrM   sh_erderd_erdr  rm_filesrf  sL                                                                               rQ   rS  zUp2k._build_dir  s     ((-HHdE:A>D=PAHHQ$r(E400!4 cCug~S%%))**$''))T!22#c(*##C(D#&,,S1Bb3h"rvvDMMtyy';';#;T4OAYb>I1,=!>? [	$JE3yyuBemGszz'*R s||$D||CKK(((7+C BTcllQ&6||CKK(g&

c)#(<(<(CHHsCJJPRSTd?cTkLL$E>chhnnRWW\\'55Q&R9!%"JBB 2IC2IC2IC \\#++.0CKK3IIJ ;;z*tyy{T/AB/F$)+x7(-/8; r	b$./"[[]F --Q/3C') "shu?O?OPS?T!088:W4!<</&,,rxxz2JJ" SH_U=M=Mc=R#w[	$z Byy!!3u:C:A!tt||Au5>>@LHu%<3%<DDLL!<reDDDLL!@2sC.QDDAID yy!!5zD \\#e*"3"3GY"GH\\^KPQaRYYs1v}}Wi@AQQIIbii34IIeE3<<01BIIK,-44W=E7C4DDLLr5k2
 zz|C}$"ED03r	299S!,BxStt||AS"~6??ADDLL!ESzRDDLL!@3R.QDDAID .1!A$./
! L	#LBbyyrBbjG,/SZZ(UFLCCDDLLr2h/ &E		Q	',Qx$Cb"u:>CA!YY'?AQ'?@FHHQ#r3u:v!>>?CSt_1C$;3"9&BqESLPR?c2sD#r::;

244R+q	"dggii%99DGGKR.tyy"dBK#HH745 $ 8 8		!:!IFA #.tyy"fEbDj KKb"b$BD$BPRTVWDDAIDEEQJE1HCrtt#Bttt|rRx:beeRTT]JK&&(  ruyy{YL	#\ yy!!DDLL5v>DDLL6eE99  	%EAIA $%"89 %v.GQ099;A>A	 Iae_,-DQ(DQ(q@ADDLLG$1	%4 S=  -	8Q&A JKK!1<<#5121<KK#;!q
':A;;8} 	(BJJrttRQ'	( HH.55d;<C}m ? R98AHHQ'68!44a8\ R4Xb\)DDLLsEl3  ERR@!DDE /CDDLLeDLL"b&AB (@B ! HHG<=R0	8Q!4 67AK;s   r!*r!3r&0r.Bs 1s7 s$ 1Bt u
%u(v
8v/1v;)w 
*w4:	w9w9&r+.#ss$.t	u!uu3v	v8v33v8;w +w1c           
         g }d}d}| j                   sJ t        |j                  d            d   }|j                  d      }|D ]  \  }	|dz  }|	j                  d      rt	        |	dd        }
n|	}
t        ||
      }d||z
  |fz  | j                   _        	 t        j                  j                  |      rr	 |j                  |	        |rd}|D ]$  }
|t        |j                  ||
f            d   z  }& | j                  d	j                  t        |      |             |D ](  }
|j                  d
|
f       |j                  d|
f       * d}|j                  j                         }|D cg c](  }|j                  |dz         s|t        |      dz   d  * }}d}|D ]  }
|
dt!        |
      z   fD ]  }	 |j                  |||dz   f      } n d}t#               }|D ]d  \  }	}t%        |	|      \  }
}||
k7  r+|
}	 t#        t        j&                  t        ||
                  }||vsJd}|j                  ||	|f       |dz  }f  |r | j                  dj                  |             |j                  j+                          d}d}d}|j                  d      D ]O  \  }}}t-        ||      }|j                  |||f      j/                         r6|j                  ||||f       |dz  }Q |r | j                  dj                  |             |j                  j+                          |j1                          ||z   S #  Y xY wc c}w #  Y xY w#  |j)                          Y DxY w)Nr   z!select count(distinct rd) from upz+select distinct rd from up order by rd descr   r  ri   zb%d %sz$select count(w) from up where rd = ?z$forgetting {} deleted dirs, {} filesrs  zdelete from up where rd = ?r(  zBselect rd, fn from up where (rd = ? or rd like ?||'%') order by rdz///&delete from up where rd = ? and fn = ?z$forgetting {} shadowed deleted filesz,select 1 from up where rd=? and fn=? limit 1z,delete from cv where rd=? and dn=? and +fn=?zselect * from cvzforgetting {} deleted covers)r   r  r   r  r?   r$   r   r  r  r|  r  rm   r   r  r}  r  r@   r   r5   listdirrm  r~  r=   r  r  )rP   r   r\  rc  r  rf  ncheckedndirsrL   r  r  r^   r  n_rm2c2r   erdr  cdcdfnr.  n_rm3qur  urds                            rQ   rT  zUp2k._drop_lost  s   www S[[!DEFqIKKEF 	FSMH~~d#c!"g&CnG"eh&6%@@DGGK77==) *
 IIcN	" 6A 7S[[RE23A667 HH;BB3r7DQR B82%@9B5AB
 ^^""$+/Ka1<<c	3J#c(Q,.!KKP 	BD8B</0 AS3Y'78A C5C SsCB"9C$!"**U3^"<= S=@AJJq3*-QJE	2 HH;BB5IJ
 ;:KK(:; 	LCbS/C::b3),557

1sCn-
		 HH3::5AB


e|C" L$		s0   M /M	M9M>(M MMM*c                 >   |j                   }| j                  sJ | j                  |   }|j                  j	                  d      }|j                  j	                  d      }d|j                  v }d|j                  v }| j
                  j                  D cg c]b  }||j                  k7  rQ|j                  |j                  dz         s|j                  s'|t        |j                        d j                  d      d }	}g }
g }|	D ]&  }|
j                  d       |j                  ||g       ( t        |      }dj                  |
      }|rd	|z   }g }g }| j                  5  d
}d
}d|z   }|j!                  ||      D ]  \  }||z  }|dz  } | j#                  |d|z   |d
      \  }}ddd       t%        j&                  d      5 }|D ]  }| j(                  r
 ddd       y|dd j+                  d      j-                  dd      }|j/                  d      \  }}}| j                  5  d}	 |j!                  ||||f      j1                         \  }}	 ddd       dz  }z  }|j                  d      s|j                  d      rt3        ||      \  }}n|}|}t5        |||      } |r|j7                  |       r|r|j7                  |       nd}!dj9                  ||dz  dz        }"|"| z   | j                  _        	 t=        j>                  |       }#tA        jB                  |#jD                        rt=        j@                  |       n|#}$tK        |#jL                        }&|$jN                  }'|!s|'stQ        | jR                  |'|&||      }(nR|'dkD  r| jI                  d|        	 | jU                  | |"      \  })}|)s
 ddd       ytW        | jR                  |'|)      }(||(k(  r|$|#k7  r9|!s|'s5tK        |$jL                        }*tQ        | jR                  |'|*||      }+||+k(  rV|j                  |||(|'|&f       d },| jI                  |,| |||(|'|&fz  d        	 ddd       |rw|s|rsd| jX                  _-        t]        | jX                  j^                         d!},|,|j                  t        |      t        |      fz  },| jI                  |,d       tG        |,      |r|s|sy
| j                  5  d"}|D ](  \  }}}}}|j!                  |||tK        |      ||f       * |rd#}|ja                  ||       |jb                  je                          ddd       t        |      t        |      z   S c c}w # 1 sw Y   xY w#  Y ddd       xY w# 1 sw Y   ;xY w# tF        $ r4}%| jI                  d| d       |j                  |||f       Y d}%~%d}%~%ww xY w# tF        $ r"}%| jI                  d|%d|        Y d}%~%6d}%~%ww xY w# 1 sw Y   xY w# 1 sw Y   xY w)$z0expensive; blocks database access until finishedr@  rA  e2vue2vpr(  Nz$up.rd != ? and not up.rd like ?||'%' and z where r   zselect sz from upr   zselect w, rd, fn from upr  modefileobjrG   r   r2   z2select mt, sz from up where rd=? and fn=? and +w=?r  Fzv{}, {:.0f}+   zmissing file: rg      ro  rp  rq  z?hash mismatch: %r
  db: %s (%d byte, %d)
  fs: %s (%d byte, %d)z@in volume /%s:  %s files missing, %s files have incorrect hashesz1update up set w=?, sz=?, mt=? where rd=? and fn=?z+delete from up where rd=? and fn=? and +w=?)3r   r   r   r   r  ro   r   r  r  r  r3  r  rP  r   r  r~   r   _spool_warksr(   r  r{   r  r)  r   r  r5   r$   r  r   r   r   lstatrM  rx  ry  rh  rm   r   rw  rz  r  rt   r  r  rj   retcoder   r   r  r}  r~  )-rP   rs  r   r   r]  r^  r  r  rb  rc  qexapexar  pexqexrewarkf404b_leftn_leftr  r@  tfr  gfr<  rq  r]   r  r  mtr  r.  r^   rA  pfstlstrr  mt2sz2w2r  mt2bw2brO   s-                                                rQ   r  zUp2k._verify_integrityE  s   ||wwwhhtniimmG$iimmH%"" XX&&
CII~1<<		C#@		 c#))n&&s+
 

  	(EKK>?KK'	( dll4 c/CZZ 	UFF#c)AQ, "! %%c+E+KSRSTEB	U ]]b1 I	DR HD99I	D I	D
 W^^G,44ZF hhz23ZZ !LA!!$Qc1!>!G!G!IB! !">>$'3>>$+?"3_FBBBb"-3::g.03G,#**66D=43GH 7l))G,C.2ll3;;.G'*SB #,,'jj0Cb"MB--w!89!$($<$<Wb$I	
 "!qI	D I	Dt 1CHB7 9&r{{+D1$))S$BOCCx sCS#67Wgq"b"c3??CQHDI	DV Vt DHH488##$RASYYD	3v;77AHHQNA,F4ZZ 		$CA%+ 9!B2rA2s2wB789 A4(NN!!#		$ 6{SY&&}
&	U 	U*! ! ! ! !8 ! HH7<a@KKc1.  % !B!@A !gI	D I	Dl		$ 		$s   A'U)AU.X"AX'V+&U;B*X=AVAX!W6XBX*A X.U8;V	=V>
XVVX	W!)W
XWX	X!W>8X>XXXXc                    |j                   }| j                  5  | j                  5  | j                  ||j                        }d d d        d d d        r| j
                  sJ | j                  |   }| j                  j                  sX| j                  5  |j                  d      }|j                         s!ddt        | j                        fcd d d        S 	 d d d        | j                  |      }| j                  5  | j                  |d       |j                  j!                          d d d        |S # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   uxY w# 1 sw Y   |S xY w)N$select k from kv where k = 'tagscan'r   F)r   r~   r   r  r   r   r   rl   rV  r   r  r  r   _build_tags_index_2rW  r}  r~  )rP   rs  r   r  r   rL   r  s          rQ   r  zUp2k._build_tags_index  s,   ||ZZ 	7 	7%%dCII6C	7 	7 twwhhtnyy!! 1KK FGzz|adii01 1#1
 &&t,ZZ 	$c5)NN!!#	$ 
%	7 	7 	7 	71 1	$ 
s:   E D4E 9E=-E4D=	9E  E	EE"c                 p   | j                  d       | j                  j                  j                         D ]~  }| j	                  |j
                  |j                        }|s,|\  }}| j                  |d       |j                  d       |j                  d       |j                  j                           y)r  z*dropping caches for a full filesystem scanTr1  r2  N)rm   ro   r   r  r  r   r   rW  r   r}  r~  )rP   rs  r  r   r  s        rQ   r   zUp2k._drop_caches  s    =>88$$++- 		$C%%cllCII>CFCc4(KK()KK()NN!!#		$rS   c                     | j                   j                  ry|j                  d      }t        |j	                               |k(  ry|r|j                  d       y|j                  d       y)NFr  #insert into kv values ('tagscan',1)z"delete from kv where k = 'tagscan'T)rl   rV  r   r  r  )rP   r   r  rL   s       rQ   rW  zUp2k._set_tagscan  s^    99KK>?

%KK=>  KK<=rS   c                    | j                   |   }| j                  |   }| j                  |   }d}d}d|v rj| j                  5  |j	                  d      j                         d   }|r1| j                  dj                  |             |j	                  d       d d d        d|v r| j                  5  d}|j                  j                         }d}	d}
|j	                  d	      D ]A  \  }|j	                  |	|f      j                         r'|j	                  |
|d d
 f       |dz  }C |j                          |r'd}| j                  |j                  |             ||z  }d d d        | j                  5  |j                  j                          d d d        d|v sd|v rd|dfS d|v r| j                  sd|dfS d}| j                  5  | j                  |ddd      \  }}d d d        |s| j                         d|dfS |dk(  ryt        j                   d      5 }| j#                  |||||      }d d d        | j                  |       ||dfS # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   PxY w)Nr   r  zselect count(w) from mtz*discarding {} media tags for a full rescanzdelete from mtr  z)select w from up where substr(w,1,16) = ?zdelete from mt where w = ?zselect w from mt   r   z)discarded media tags for {} deleted filesd2tr  Tr  Fz select w from up order by rd, fnrW   rG   )rG   rG   Tr  r  )r\   r   r   r~   r   r  rm   r   r}  r  r  r~  r   r  _unspoolr(   r  _e2ts_q)rP   r   r\   r   r   re  rf  rM   r  up_qrm_qr]   rO   nqr  r  s                   rQ   r  zUp2k._build_tags_index_2  s   T"

4 hhtne 2{{#<=FFHKHHIPPQUVWKK 01	2 E> ^^**,B3KK(:; DQ::dQD1::<

4!CR&3Q
 
CAHHQXXa[)AID  ZZ 	$NN!!#	$ E>Ue^dD=  U?99$~%B **;RB
 b!$}$Rx#D"5 @RS$?@ MM"dD  o2 2  	$ 	$ @ @sE   AI0AI	AI3I(I4J II%(I14I= J	c           
         | j                   r| j                  sJ | j                  |   }d }| j                  j                  r)| j                  j
                  dkD  r| j                         }d}d}	t        j                         }
|D ]  }| j                  r y|d d j                  d      }|d d }| j                  5  	 d}|j                  |||f      j                         \  }}}}|j                  d      s|j                  d      rt        ||      \  }}d|v rBd	}|j                  ||f      j                         r
	 d d d        d
}|j                  ||f       d d d        t        |      }d||fz  | j                   _        |s| j#                  ||||      }nWr|d}ni }|j%                  t'        i ||||             | j                  5  t)        | j+                  |            }d d d        |z  }|	|z  }	|dz  }t        j                         |
z
  }|	dk\  s|| j,                  dz  k\  s| j/                  dj1                  |	             | j                  5  |j2                  j5                          d d d        t        j                         }
d}	 |rC| j7                  |       | j                  5  |t)        | j+                  |            z  }d d d        | j                  5  |j2                  j5                          d d d        |S #  Y d d d        xY w# 1 sw Y   xY w# 1 sw Y   DxY w# 1 sw Y   xY w# 1 sw Y   pxY w# 1 sw Y   |S xY w)Nr   r   rG   r0  r  z=select rd, fn, ip, at from up where substr(w,1,16)=? and +w=?r  r[   z1select 1 from mt where w=? and +k='t:mtp' limit 1z%insert into mt values (?,'t:mtp','a')zc%d %s)up_ipup_atrr  ri   zcommit {} new tags)r   r   r   	prefer_mtrl   r   _start_mpoolr   r{   r  r~   r   r  r  r5   r$   r   _tagscan_fileputrY   r  _flush_mpoolr   rm   r   r}  r~  _stop_mpool)rP   qfr  r   r   r\   r   mpoolre  n_buf
last_writebwr]   w16r  r  r.  rB  r>  r^   n_tagsr_   r  s                          rQ   r  zUp2k._e2ts_qD  s.    ww499$$

4 99499#4#4q#8%%'EYY[
 3	Byy3Bw'ACR&C +WA%([[S!H%=%F%F%HNBB
 ==&"--*="2r]FBE>KA{{1sf-668 + +  @AKKC6*#+& D"b)G"b']2DGGK++CGRL)+b9H!H		$r61gx@AZZ 9 !2!23!78F9 VOEVOE!GBz)B}dllQ&6 6-44U;<ZZ ,NN))+, "YY[
g3	j U# 5T..s3445 ZZ 	$NN!!#	$ e+ + + +<9 9, ,5 5	$ sa   <L&>)L'AL&	L&L3$M ?M1ML#L&#L&&L0	3L=	 M		MM"c                    d}|j                   j                         }t        j                  ddd      }t	        j
                  d|      5 }|j                  ||      D ]  }	| j                  r|dfc cd	d	d	       S |d
k(  r*d}|j                  ||	d   d	d f      j                         rMdj                  |	      j                  dd      }
|j                  |
dz   j                  d             |d
z  } 	 d	d	d	       |j                          |j                  d       | j                  j!                  |       ||fS # 1 sw Y   IxY w)r   r   i   zw+bzcpp-tq-)r9  wbr  rG   Nr   z,select 1 from mt where w=? and +k != 't:mtp'r  r  r2  r  r  )r}  r  tempfileSpooledTemporaryFiler(   r  r   r{   r  r  r)  writer7  r  seekr   rl  )rP   r   r  paramsfltrM   r  r  r  rowrq  s              rQ   r  zUp2k._spool_warks  s*    ^^""$**?E)T]]b1 	R{{1f- 99r6M	 	
 !8FAzz!c!fSbk^4==? __S)11$
C"t)++G45Q	 	


1u#	 	s   'E =A:E  E	c                     	 | j                   j                  |       	 |j                          y #  Y y xY w# t        $ r+}| j	                  dj                  |      d       Y d }~y d }~ww xY w)Nzfailed to delete spool: {}rg   )r   remover  rh  rm   r   )rP   r  rr  s      rQ   r  zUp2k._unspool  sb    	KKr"	AHHJ		  	AHH188<a@@	As   / 6 3	A*!A%%A*c                     g }| j                   D ])  } | j                  |g|  |j                  |d          + g | _         |S )Nr   )r   	_tag_filer  )rP   wcurr  r   s       rQ   r  zUp2k._flush_mpool  sN    "" 	ADNN4$!$JJqt	 
rS   c           	         t        j                          }| j                  j                         D ]  \  }}d|v rD|| j                  vr$d}| j	                  |j                  |      d       n| j                  ||       d}| j                  j                  j                         D ]  }|j                  |k(  s|j                  }  d| j                  j                  |d      v sd| j                  |<    t        j                          |z
  }d}	| j	                  |	j                  |t        |d	                   | j                          | j                   j"                  d
k(  r| j$                  j'                          y y )Nr[   z&skipping mtp for unavailable volume {}r   r2  zrunning mtpr6  r  zmtp finished in {:.2f} sec ({})Tr   )r   r   r   r\   rm   r   _run_one_mtpro   r   r  r   r  r   r  r4   r   rl   r   rj   r   )
rP   rx   r;  r   r   rO   r  rs  r  r   s
             rQ   r  zUp2k._run_all_mtp  s2   YY[::++- 	5KD%~t{{*@AHHQXXd^Q/%%dC0Dxx((//1 %<<4'99D%  1 1$ ;;&4d#	5 YY[2/Bb$01		99>>U"HH #rS   c                 ,   || j                   k7  ry | j                  |   }i }| j                  |   d   D ]3  }	 t        |      }|D ]   }||j
                  v s|||j                  <   " 5 | j                  j                  r;d}| j	                  |j                  |t        |j                                     d       || j                  |<   d}| j                  5  | j                  |   }	|	j                  j!                         }	|	j                  j!                         }
|	j#                  |      j%                         d   }d d d        | j'                         }|j(                  dz  }t+        j*                         }}d}i }i }	 d	}| j                  5  || j                   k7  r	 d d d        nd
}	j#                  ||f      j-                         }|D cg c]  }t/        |d          }}g }|D ]@  }||v r	d}|	j#                  ||f      j%                         \  }}}}t1        ||      \  }}t3        |||      }d}|	j#                  ||f      j-                         }|D ci c]  }|d   d
 }}d}| j5                  |||      }|sd	||<   |dz  }t7        d |j9                         D        d       rBd}|	j#                  ||f      j-                         }|D  !ci c]  \  } }!t/        |       |! }"} }!ni }"|r
||"d<   ||"d<   |j;                  t=        |t?               |||"             d	||<   C 	 d d d        | j                  5  | jA                  
      }#|#D ]  }d	||<   d}|jC                  |       |dz  }! |D ]  }d}	j#                  ||f        i }d d d        sn|r0| jD                  5  | jD                  jG                  d       d d d        s[	 t+        j*                         }$|$|z
  ||z
  z  |z  }%tI        |%d      \  }&}%tI        |%d      \  }'}%|}|$}d}(| j                  5  |(j                  ||tK        |&      tK        |'            }(| j	                  |(d       d d d        |D ]  })|dz  }|jM                  |)        | j                  5  	j                  jO                          d d d        =| jQ                  |       | j                  5  | jA                  
      }#|#D ]  }d}	j#                  ||f        	j                  jO                          |r4| j	                  dj                  ||      d       |	j#                  d       |
jS                          |	jS                          d d d        y #  | j	                  d|z   d       Y  y xY w# 1 sw Y   dxY wc c}w c c}w c c}!} w # 1 sw Y   xY w# 1 sw Y   2xY w# 1 sw Y   	xY w#  d}&d}'Y xY w# 1 sw Y   xY w# 1 sw Y   JxY w# 1 sw Y   y xY w)Nr[   z+invalid argument (could not find program): r   zparsers for {}: [0m{}90r   r   rg   Tz*select w from mt where k = 't:mtp' limit ?z<select rd, fn, ip, at from up where substr(w,1,16)=? limit 1zselect k from mt where w = ?Fc              3   :   K   | ]  }|j                   s|  y wrK   )prir  s     rQ   r  z$Up2k._run_one_mtp.<locals>.<genexpr>$	  s     B1AEEQBs   zselect k, v from mt where w = ?r  r  z+delete from mt where w = ? and +k = 't:mtp'g?i  r`  z&mtp: {} done, {} left, eta {}h {:02d}mrE  r  zmtp: scanned {} files in {}r  )*rx   r\   r   r   rm   tagstagrl   mtag_vvr   r   r   r   r~   r   r}  r  r   r  r  maxsizer   r{  rj  r5   r$   _get_parsersr  r  r  rY   r   r  r
  r   rY  divmodr   r  r~  r  r  )*rP   r   rx   r\   parsersparserr  rO   r  r   r  r  r  batch_szt_prevn_prevn_done	to_deletein_progressdid_nothingzqr   warksjobsr]   r  r.  rB  r>  r^   havezq2r   r   r_   r8  rp  shr  r   rV  s*                                             rQ   r  zUp2k._run_one_mtp  s   $((?T"jj&u- 		1F 
  1&++%*0GFJJ'1		1 99+AHHQXXdD$894@!(7ZZ 	2((4.C..'')C>>((*D[[^,,.q1F		2 !!#==1$	K (*$((?(* (* A[[XK099;,./qQqT//  *AK' VA%([[QD%9%B%B%DNBB"2r]FB#D"b1G6AQ-668B0231qtQw3D3"'K"//dGDG"'+	!! B(8BDI=!kk!aT2;;=:=#>$!QCFAI#>#>#%,.),.)KKWceQ JK%)KNA *(*T  ((.  A#'IaL"'KOOA&aKF	  # )AEAKKA4() 	 ^^ -NN'',- 	iikFlv76Aa1a}1
 ;C #jjQQ@"#  !		!  (%%'(q v 	ZZ 	$$T*D %AAt$% NN!!#6==fdKqQH%JJLIIK	 	qFOQRS	2 	2* 0 4 $??(* (*T $- -# #( (	 	s   VA&V V=+'V=V-'A:V=!V2.A-V=V7
1A V=AW
4WAW$ /:W0W=BX
V V*-V==W
WW!$W-0W:=X
Xc           
         	 | j                   |   }| j
                  |   }i }|j                         D ]M  \  }}d|v sd|v rd|v sd|v rI|j                  dk(  r| j                  j                  r#d}	| j                  |	j	                  |      d       `|j                  dk(  r:| j                  j                  r#d}	| j                  |	j	                  |      d       |j                  rd	}
|j                  D ](  }|j                         j                  d
|z         s&d}
 n |
sY| j                  j                  rAd}	| j                  |	j	                  ||j                  |j                  d
      d         d       I|||<   P |j                         D ci c]  \  }}|j                  s||vs|| }}}|S #  | j                  j                  r!| j                  dj	                  |      d       i cY S xY wc c}}w )Nzno mtp defined for {}r  acz.aqrM   z%skip mtp {}; want no-audio, got audioyz%skip mtp {}; want audio, got no-audioFrf   Tz%skip mtp {}; want file-ext {}, got {}rG   )r   rl   r  rm   r   r\   r   audior  r  r~  r  force)rP   r   r+  r^   all_parsersr\   r  r   r   rO   matchr  s               rQ   r  zUp2k._get_parsersp	  s   	**40K T"%%' 	DAqv~&4<5D=ww#~99,, GA HHQXXa[$7 WW^yy((C!d3uu55 C}}//c	: $
 yy((C!QUUGNN34G4K!LdSGAJ;	> %,MMOPDAqqww!4-1a4PPO	yy  077=tDIF Qs   F9 G9/G99;G6c                 <   t        d| j                  j                        }| j                  s<d| _        | j	                  dj                  || j                  j                               t        |      }t        |      D ]  }t        | j                  d|f        |S )Nr   Tzusing {}x {}z
up2k-mpool)r   rl   r   r   rm   r   r   backendr   r   r   _tag_thr)rP   nwr  r  s       rQ   r  zUp2k._start_mpool	  s     DII%%&"DOHH^**2tyy/@/@ABrr 	:A4==,9	: rS   c           
          |sy t        |j                        D ](  }|j                  t        i t	               ddi              * |j                          y )Nr6  )r   r  r  rY   r   r  )rP   r  r  s      rQ   r  zUp2k._stop_mpool	  sF    u}}% 	3AIId2sub"b12	3 	

rS   c                    | j                   sJ 	 |j                         }|j                  s|j                          y 	 t	        j
                  |j                        }|j                  s| j                  j                  rBd}| j                  |j                  | j                   j                  |j                        d       |j                  r%| j                   j                  |j                        ni }n| j                  j                  rOd}| j                  |j                  t        |j                  j                               |j                        d       | j                   j!                  |j                  |j                  |j"                        }|j%                         D cg c]  \  }}dj                  ||       }}}|r:| j                  dj                  dj'                  |      |j                               | j(                  5  | j*                  j-                  |j.                  |j                  |j                  |f       d d d        |j                  r/| j6                  5  | j6                  j9                          d d d        |j                          c c}}w # 1 sw Y   \xY w#  t1        j2                         }	| j5                  |j                  xs | j                   j                  |j                  |	       Y xY w# 1 sw Y   xY w# |j                  r<| j6                  5  | j6                  j9                          d d d        w # 1 sw Y   w xY ww xY w)Nztag-thr: {}({})r  z[36m{} [33m{}z{}[0m [{}]rF   )r   r  r]   	task_doner   rM  r^   r[   rl   r  rm   r   r7  rz  r   r   get_binr_   r   r  r~   r   r  r\   r  r  _log_tag_errr   r   )
rP   r  qer  rO   r  r   r   vtagsrr  s
             rQ   r8  zUp2k._tag_thr	  sp   yyyB444XXbjj)vvyy((-$))*;*;RZZ!H$O8:

499==4Dyy((-$rvv{{}*=rzz!JDQ99,,RVVRZZMDIMAEA/66q!<E  !1!8!8%"**!UVZZ R%%,,biirzz4-PQR 66 41134 KKMI (R RO))+!!"&&"=DII,=,=rzz2N4 4 66 41134 4 4 sh   E+J5 )J#AJ5 >J)J5 .L#J5 )J2.J5 5AL
L LM%4M	M%M!M%c                 r    |d|d|}| j                  |j                         d|v rd       y d       y )Nz failed to read tags from z:
z<Signals.SIGr   rg   r  )rm   r3  )rP   r   r^   rr  r   s        rQ   r>  zUp2k._log_tag_err	  s2    7=wKn&;CCrS   c                    	 t        j                  |      }t        j                  |j                        sy	 |j                  r| j
                  j                  |      ni }|r
||d<   ||d<   | j                  5  | j                  |||||      cddd       S #  Y yxY w# t        $ r}	| j                  d||	       Y d}	~	yd}	~	ww xY w# 1 sw Y   yxY w)zwill mutex(main)r   r6  Nr  r  )r   rM  r}  ry  rz  r   r  rh  r>  r~   r  )
rP   	write_curr\   r  r^   rB  r>  r  r  rr  s
             rQ   r  zUp2k._tagscan_file	  s    	'"B ||BJJ'	-/ZZ499==)RD
 DMDMZZ 	J>>)VT7DI	J 	J!	  	b'2.		J 	Js.   B )B 9CB	C(C  CCc           	      H   t         j                  j                  |      sy|r.|j                         D ci c]  \  }}||v s|| }}}|sddi}|sy|j	                         D ]j  }dj                  dj                  dgt        |      z              }|dd gt        |j	                               z   }	|j                  |t        |	             l d}
|j                         D ]$  \  }}d}|j                  ||dd ||f       |
d	z  }
& | j                  |d
       |
S c c}}w )r   r   r   z#delete from mt where w = ? and ({})z or z+k = ?Nr  zinsert into mt values (?,?,?)r   T)r   r  isfiler   r   r   r  r  r   r   r   rW  )rP   rC  r\   r  r^   r  r   r   r  rl   r  s              rQ   r  zUp2k._tag_file
  s*    xxw'%)ZZ\ATQQ&[AqDADAQx 	.A5<<XJT23A "I;diik!22Dat-	. JJL 	DAq/Aa$s)Q!231HC	
 	)T*
- Bs   DDc                 D    | j                  dj                  |             y )NzST: {})rm   r   )rP   r   s     rQ   _tracezUp2k._trace'
  s    %&rS   c                     g }| j                   j                  st        | j                  d||g       	 | j	                  |      |j                  d       S # |j                  d       w xY w)Nopendb_watchdogr   )rj   r   r   _open_db_timeout_open_dbr  )rP   r  oks      rQ   r  zUp2k._open_db_wd*
  sQ    xx4((*;gr]K	==)IIaLBIIaLs   A A(c                 ~    t        d      D ]  }t        j                  d       |s y  d}| j                  ||       y )Nr`  r   zpWARNING:

  initializing an up2k database is taking longer than one minute; something has probably gone wrong:

)r   r   r   _log_sqlite_incompat)rP   r  rL  r  rO   s        rQ   rJ  zUp2k._open_db_timeout4
  s>    r 	AJJqM	
 C!!'1-rS   c                    |xs d}t        j                  |j                  dd            j                         }t	        |d d       j                  d      }t        j                  j                  t        j                  d|d      }d}|d	z  }|||t        |fz  z  }| j                  |d
       	 t               }t        |d      5 }	|	j                  |j                  dd             d d d        | j"                  j$                  rd}t!        |d|d|      y # 1 sw Y   5xY w# t         $ r"}
| j                  d|d|
d
       Y d }
~
^d }
~
ww xY w)Nr6  r  r)  rh   r0  zstack-z.txtz`  the filesystem at %s may not support locking, or is otherwise incompatible with sqlite

  %s

zs  PS: if you think this is a bug and wish to report it, please include your configuration + the following file: %s
rg   r  zwarning: failed to write r   zH------------------------------------------------------------------------r2  )r5  sha512r7  r8  r;   r  r  r  r  r   rD  r  rm   r    openr	  rh  rl   r  )rP   r  r;  txtr8  	stackname	stackpathrO   stkrA  rr  s              rQ   rN  zUp2k._log_sqlite_incompat>
  s2   hBw	 BCJJLF2AJ'..w7	GGLL)(EF	r	  D  	DqG]I666a	M*Ci& 8!

7I678
 99;;AAsA677 8 8 	MHH)RH!LL	Ms0   2D* "D*D* D'#D* *	E3EEc                 f    t        j                  || j                  d      j                         }|S )NF)r   check_same_thread)r   r  r   r  )rP   r  rL   s      rQ   r   z	Up2k._orzT
  s,    OOT\\U

&( 	
 rS   c                    t         j                  j                  |      }| j                  |      }| j	                  |      }|s|| j                  ||      S |dk(  r*	 d}| j                  ||||      }| j                  |       d}|t        k(  r| j                  |       | j                  |       | j                  |       | j                  ||       | j                  |       	 t        |j!                  d            d   }| j                  dj#                  ||      d	       |S |xs dt        kD  r!d}t'        |j#                  |t                    d}|rd}| j                  ||||      }|j(                  }|j+                          |j+                          | j-                  |       | j                  |d       S #  | j                  dd       Y OxY w#  | j                  d
t%               z          Y xY w)Nr<  z creating backup before upgrade: rC   zWARN: failed to upgrade from v4rg   rD  r   z	  {} |{}|r  z(WARN: could not list files; DB corrupt?
zCdatabase is version {}, this copyparty only supports versions <= {}z&creating new DB (old is bad); backup: z.creating new DB (too old to upgrade); backup: )r   r  r  r   	_read_ver_try_create_db
_backup_db_upgrade_v4rm   DB_VER_add_dhash_tab_add_xiu_tab_add_cv_tab_add_idx_up_vp_add_ds_tabr  r   r   r+   rh  r}  r  
_delete_db)	rP   r  existedr   verrO   rL  r   r  s	            rQ   rK  zUp2k._open_db[
  s   ((//'*ii nnS!3;&&w44!8?6oogsC;  % &=$c"S!W-S!Qckk*CDEaH++GV<dC
 H1UAAHHS&1226BCoogsC5^^		

 ""7D11=?:A>QDvxOPs   )F? 'A G ?GG9c                 X    dD ]  }	 t        j                  ||z           y #  |s Y &xY w)N)r6  z-shmr  z-journal)r   unlink)rP   r  sufs      rQ   rc  zUp2k._delete_db
  s8    3 	C

7S=)	 s   ")c                    dj                  |t        t        j                               |      }| j                  ||z          	 t	        j
                  |      }|5  |j                  j                  |       d d d        ||j                          S # 1 sw Y   xY w#  d}| j                  |t               z          Y nxY w	 j                          n# j                          w xY w|j                  }|j                          |j                          t        j                  t        |      t        |             | j                  |      S )Nz{}.bak.{:x}.v{}z5native sqlite3 backup failed; using fallback method:
)r   r   r   rm   r   r  r}  backupr  r+   shutilcopy2r%   r   )	rP   r  r   re  r   bakr  rO   r  s	            rQ   r[  zUp2k._backup_db
  s      &&wDIIK0@#Fs		%B *%%b)*
 HHJ* *	#HAHHQ\""HHJBHHJ^^		

U7^U3Z0yy!!s0   B B7	B BB  B?=C C&c                     dD ]H  }	 |j                  dj                  |            }|j                         }|s7t        |d   d         c S  y #  Y PxY w)N)kikvz!select v from {} where k = 'sver'r   )r   r   r{  r   )rP   r   r!  rL   rowss        rQ   rY  zUp2k._read_ver
  sa     	'CKK D K KC PQ ::<D471:&	' s    AAc                 f    	 | j                  ||      S #  	 | j                  |        #  Y  xY wxY wrK   )
_create_dbrc  )rP   r  r   s      rQ   rZ  zUp2k._try_create_db
  sA    	??7C00	( s    0)0-0c                    |s| j                  |      }d}| j                  rd}ddddd|dd	d
dddj                  t              fD ]  }|j	                  |        | j                  |       | j                  |       | j                  |       | j                  |       | j                  dj                  |             |S )a   
        collision in 2^(n/2) files where n = bits (6 bits/ch)
          10*6/2 = 2^30 =       1'073'741'824, 24.1mb idx  1<<(3*10)
          12*6/2 = 2^36 =      68'719'476'736, 24.8mb idx
          16*6/2 = 2^48 = 281'474'976'710'656, 26.1mb idx
        z'create index up_w on up(substr(w,1,16))zcreate index up_w on up(w)zKcreate table up (w text, mt int, sz int, rd text, fn text, ip text, at int) create index up_vp on up(rd, fn)zcreate index up_fn on up(fn)create index up_ip on up(ip)create index up_at on up(at)z'create table mt (w text, k text, v int)zcreate index mt_w on mt(w)zcreate index mt_k on mt(k)zcreate index mt_v on mt(v)zcreate table kv (k text, v int)z"insert into kv values ('sver', {})zcreated DB at {})
r   r   r   r]  r   r^  r_  r`  rb  rm   )rP   r  r   r   r  s        rQ   rs  zUp2k._create_db
  s     ))G$C8/C [/+++6))).188@
 	C KK	  	C ##**734
rS   c                 h    dD ]  }|j                  |        |j                  j                          y )N)z!alter table up add column ip textz alter table up add column at intrv  z update kv set v=5 where k='sver')r   r}  r~  rP   r   r  s      rQ   r\  zUp2k._upgrade_v4
  s3    
 	C KK	 	rS   c                     	 |j                  d      j                          y #  Y nxY wdD ]  }|j                  |        |j                  j                          y )Nzselect d, h from dh limit 1)z create table dh (d text, h text)zcreate index dh_d on dh(d)r  r   r  r}  r~  ry  s      rQ   r^  zUp2k._add_dhash_tab
  s[    	KK56??A	
 	C
 KK	 	   " &c                     	 |j                  d      j                          y #  Y nxY w	 |j                  d       n#  Y nxY wdD ]  }|j                  |        |j                  j                          y )Nz#select c, w, rd, fn from iu limit 1zdrop table iu)z1create table iu (c int, w text, rd text, fn text)zcreate index iu_c on iu(c)zcreate index iu_w on iu(w)r{  ry  s      rQ   r_  zUp2k._add_xiu_tab  sv    	KK=>GGI		KK(	
 	C
 KK	 	s   " &< A c                     	 |j                  d      j                          y #  Y nxY wdD ]  }|j                  |        	 |j                  d       n#  Y nxY w|j                  j                          y )Nz!select rd, dn, fn from cv limit 1)z+create table cv (rd text, dn text, fn text)zcreate index cv_i on cv(rd, dn)r1  r{  ry  s      rQ   r`  zUp2k._add_cv_tab  su    	KK;<EEG	
 	C KK			KK()	s   " &A Ac           	         	 |j                  d       dD ]-  }| j                  d|d|d d        |j                  |       / | j                  d|d       |j                  j                          |j                  d       y #  Y y xY w)Nzdrop index up_rd)ru  rw  zupgrading db [z]: r/  z]: writing to disk...r  )r   rm   r}  r~  )rP   r   r  r  s       rQ   ra  zUp2k._add_idx_up_vp.  s    	KK*+
 	C HHSbBCKK	 	GEFH	s   B B
c                     	 |j                  d      j                          y #  Y nxY wdD ]  }|j                  |        |j                  j                          y )Nzselect rd, sz from ds limit 1)z)create table ds (rd text, sz int, nf int)zcreate index ds_rd on ds(rd)r{  ry  s      rQ   rb  zUp2k._add_ds_tab@  s[    	KK78AAC	
 	C KK		 	r|  c                 z    | j                   5  | j                   j                          d d d        y # 1 sw Y   y xY wrK   )r   r   r   s    rQ   wake_rescannerzUp2k.wake_rescannerP  s2     	*'')	* 	* 	*s   1:c                    || _         | j                  s| j                  rt        dt        dz        d}	 | j
                  j                  d      r)d}| j                  5  | j                  |      }d d d        n"t        dt        | j                  xs dfz        |r| j
                  j                          	 | j                  r| j                          S # 1 sw Y   DxY w# t        $ r_ t        s | j
                  5  | j                  5  | j                  |      }d d d        n# 1 sw Y   nxY wd d d        n# 1 sw Y   nxY wY w xY w# |r| j
                  j                          w w xY w)Ni  )z	fs-reloadFrb  r   Tz	[unknown])r   r   r   r   SBUSYr~   r  r   _handle_jsonr   	TypeErrorr	   r  r   do_fx_backlog)rP   cjr   r  r  s        rQ   handle_jsonzUp2k.handle_jsonT  sF    !t~~en455	%zz!!"!-^^ 0++B/C0 0 S%4<<+F;*H"HII 

""$?? 
!0 0  	, ,T^^ ,''+, , , , , ,	, 

""$ sk   *C$  C2+C$ C!C$ $E D=D(	D=(D1-D=4	E=E	E	E EE E.c                 $  T |dkD  rt        dd      |d   T| j                  T|d         sT| j                  vrt        dd      t        j                         x|d<   x}x| _        | j
                  T<   | j                  |      x}}d }t        T|d	         }t        ||d
         }	 t        j                  |      j                  }	| j                  j                  |      dk7  }
	 | j                  j                  T      }| j                  T   }| j                  j                  |d      }t!        |j"                  j                  d            }t!        |j"                  j                  d            }|j"                  j                  d      xs |j                  d      }g }|j"                  j                  d      xs d}|dk  xs |}|rT|fgng }|j"                  j                  d      r8|| j                  j%                         D cg c]  \  }}|Tk7  s||f c}}z  }|r%t'        | j(                  |d   |d   |d	   |d
         }|d   }|dk  xs |dkD  }|s|j"                  j                  dd      dk(  r<t+        t        j                               |d<   |r|rd}| j-                  ||fz  d       g }|D ]  \  T}| j                  j                  }t/        Tfd|j1                         D        |      }||k(  r|d   n|j2                  }| j4                  rd}|g} n	d }|d d |g} |j7                  |t9        |             }!|!D ]  \  }"}#}$}%}&}'}(|%j;                  d!      s|&j;                  d!      rt=        |%|&      \  }%}&t        T|%|&      })|r|)|k7  rR	 t        j                  |)      }*t        j>                  |*j@                        r
tC               |*jD                  |$k7  rGd"}|jG                  ||*jD                  |$|*jH                  |#|)      }| j-                  |       tC               	 |&|%|T|
|$|#|d#   |d$   |'|(d%},d&D ]  }||v s||   |,|<    d' tO        tQ        |%d(z   |d	   d)z               D        }-|*j                  |	k(  rd*nd|%|d	   k(  rd+n
t/        |-      z   |&|d
   k(  rd,ndz   }.|jM                  |.tS        |       |,||%|&f         d }tU        |d-      D ]  }/|/d.   }0t        |0d   |0d	   |0d
         }1|s||1k(  rd}|0} n| j-                  d/|1       | jW                  |1      \  }2}*tY        | j(                  |*jD                  |2      }3||3k7  r/d0}| j-                  ||3||1|0fz         |jM                  |/d1d         d}|0} n |r||v r||= g x|d2<   |d3<   d,|d4<   i |d5<   |rd }!|D ]w  \  }}%}&d6}| j-                  |d7t[        t[        |j2                  |%      |&      z   z         | j]                  ||%|&|d          |!r|!|k7  r|!j^                  ja                          |}!y |!j^                  ja                          |}d T|s}||v rxd}4d }*||   }0d8D 5cg c]  }5|5|0v s|0|5    }6}5|6D ]@  }7t        |0d   |0d	   |7      }4	 t        j                  |4      }*|*jD                  dkD  sd4|0v r nB t        |d   |d	   |d
         }t        |0d   |0d	   |0d
         }1| jb                  jd                  s|s|*rd4|0vrn|*jD                  |0d   k7  r=d:}|||*jD                  |0d   |*jH                  |0d   |4|0fz  }| j-                  |       ||= ny||1k7  rt|srd4||   v rk| j-                  d/|1       | jW                  |1      \  }2}"tY        | j(                  |*jD                  |2      }3||3k7  rd;}| j-                  ||3||1|0fz         ||= |s||v r)|xs ||   }|d   |d   k7  s|d	   |d	   k7  s|d
   |d
   k7  rt        |d   |d	   |d
         }8t        |d   |d	   |d
         }9t        |d   |d	   |d
         }:|:jg                  d<d7      }:d4|vry| j-                  d=|8d>|9       d?};|;d7ti        |:      z   d@z   z  };|r>|d	   |d
   |d   f}/	 |/| jj                  |8   vr| jj                  |8   jM                  |/       t        dA|;      dB|j"                  v r9| j-                  dC|8d>|9       dD};|;d7ti        |:      z   d@z   z  };t        dE|;      |8dFz   }<| jb                  jl                  r3to        jp                  dG|<      }=|=r|=js                         \  }>}?|>dHz   |?z   }<|8| jt                  v s|<| jt                  v s||v rd4||   vrt        dAdI      tw        |      }||dJ<   ||dK<   |j                  dL      xs ||dL<   dM}@|@jy                         D ]  }|j                  |      xs d||<    dND ]  }||v s||   ||<    t        |d   |d	         }|r t{        ||d
   |j"                  dO         |d
<   t        |d   |d	   |d
         }9|j"                  j                  dP      }A|Art        |d   |d	   |d
         }Bt}        | j,                  d | dQA|9|B|d#   |d$   | j                  j                  |d   |d$         |d   |d   |dR   |dL   d      }C|Cs#dS|9}| j-                  |d,       t        dT|      Cj                  dU      rt        | j                  |9BCdU         }5|5r|j2                  |d	   |d
   f}D|5\  }}"|d
<   \  }}Et        j                  j                  ||d
         }9|j"                  |d<   |j                  |d<   |j2                  |d<   |E|d	<   t        |d
   d      |d
<   |j2                  |d	   |d
   f}F|D|Fk7  r3|d2   |d2<   | j-                  dV|fz  d       | j                  ||d,z         S | j                  |||      |d
<   t        |d   |d	   |d
         }9| jb                  jd                  s8|j"                  }G	 | j"                  |d      }G| j                  |8|9|G|d   dW       |r}| jb                  jd                  sgdX}@|@jy                         D 5cg c]  }5||5   	 }H}5 | j                  ||j"                  g|H  |j^                  ja                          n||v r|dR   |dR<   |st        |d   |d	         }I|r t{        I|d
   |j"                  dO         |d
<   |j                  r|j                  j                  |dR   |d	   |d   |d   I| j                  j                  |dY      \  }J|d	<   t        j                  |J|j"                  Z       |j                  j                  |dR          |j                  j                  |dR   |d          ||||
tw        |d2         g i d[}d\}@|@jy                         D ]
  }||   ||<    dND ]  }||v s||   ||<    t               }K|d2   D ],  }|Kvs|d3   jM                  |       Kj                  |       . 	 | j                  |||      }L|LrLS 	 d]jG                  |d   |d	         j                  d7      }M|Mrd^jG                  M      nd7}M|d
   |M|d   |d   |j                  d_|
      |d3   ||d`}L|Ld2   sda|j"                  v r| jb                  jd                  s|d$   |j                  j                  v s|d$   |j                  j                  v rdb|j"                  v rd.nd,}Nt        t        |d   |d	   |d
               }Ot        rdnt        j                  O      j                  }P| j                  N| jb                  j                  O|d   |P      }Q|Qd |j"                  da    Lda<   Ld2   s5|r2|j                  dc      r t+        |d         t+        |d         k7  r| jb                  jd                  s|d$   |j                  j                  v r|d$   |j                  j                  v rdd}R	 |j7                  R|d   |d d |d	   |d
   f       |j^                  ja                          t        |d   |d	   |d
         }Ot+        t        j                               t+        |d         f}St        j                  |O|Sde       | j-                  df|O|d   |d   fz         LS LS #  d}	Y nxY wc c}}w # tB        $ r.}+|rtJ        }*n|jM                  ||%|&f       Y d }+~+Y d }+~+9d }+~+ww xY wc c}5w #  | jb                  jd                  s|sd9}| j-                  ||4fz         ||= Y  
xY w#  |/g| jj                  |8<   Y xY w#  t        j                  j                  |9      rt        | j,                  |9G       |s Y xY wc c}5w #  | j                  |d      j                  |dJ   d         xY w# tB        $ r }+| j-                  dg|+d1       Y d }+~+LS d }+~+ww xY w)hNr    ztoo many xbu relocs, giving upr   r=  i  zlocation unavailabler  r  r  r   ngTr  r  noclonerand	safededupre   rb  xlinkr  r  l   *UU up_tsr6  fuz6ignoring impossible last-modified time from client: %srE  c              3   B   K   | ]  }|j                   k(  s|  y wrK   )r   )r  r   r   s     rQ   r  z$Up2k._handle_json.<locals>.<genexpr>  s     L1t9KQLs   zselect * from up where w = ?z0select * from up where substr(w,1,16)=? and +w=?r  zScandidate ignored (db/fs desync): {}, size fs={} db={}, mtime fs={} db={}, file: {}hostr9  )r  r  r  r   sprsr  r  r  r9  r:  r>  )lifec              3   <   K   | ]  \  }\  }}||k7  r|d z     yw)r   NrW   )r  rM   c1r  s       rQ   r  z$Up2k._handle_json.<locals>.<genexpr>  s-      'AxB 8	 As   r2  i9  i  r   r   ri   zasserting contents of z;will not dedup (fs index desync): fs=%s, db=%s, file: %r
%srg   r  r  r8  r  z forgetting desynced db entry: %rr(  )r  tnamz'forgetting deleted partial upload at %rzVwill not dedup (fs index desync): %s, size fs=%d db=%d, mtime fs=%d db=%d, file: %r
%sz<will not dedup (fs index desync): fs=%s, idx=%s, file: %r
%sr'  zunfinished:
  
  zUpartial upload exists at a different location; please resume uploading here instead:
rF   i  nodupezdupe-reject:
  z&upload rejected, file already exists:
i  r  z(.*[\\/])(.*)rf   z(source file busy; please try again laterr  r  r>  z,vtop ptop prel name lmod host user addr poke)r  r)  nrandxbuzxbu.up2k.duper:  %upload blocked by xbu server config:   reloczxbu reloc1:%d...)r  r  z9prel name lmod size ptop vtop wark dwrk host user addr atzup2k._get_volsizer  )r  r  r;  r  r  r  r  z1vtop ptop prel name size lmod host user addr pokez{}/{}z/{}/r  )r  purlr  r  r  r  r  r  fkfkaumodz=update up set mt=? where substr(w,1,16)=? and +rd=? and +fn=?Fztouched %r from %d to %dzumod failed, )cr   r  r   r   r   r   	_get_warkr$   r   rM  rN  r   r  r   ro   r   r  r   r   r  rt   r   rm   r  r  r  r   r   r   r  r5   rx  ry  rh  rz  r   rw  NULLSTATr  	enumeratezipr  rk  r  r  r=   r  r}  r~  rl   r9  r)  r-   r   r  ru   r5  groupsr   r   r   r.   r2   	get_permsr,   r  r  r  r   r7   r  _untaken_symlinkr  rA   r  limr]  rj   brokerr  nupbupr   rl  _new_uploadr
  r*  axsureadupgetr   r   st_inor   fk_saltuwriteudelutime)UrP   r  depthrp  r  dwarkr"  pdirinc_apra  r  jcurr  ro   r  r  r  lost
safe_dedupdata_okrw  r   r   r  bad_mtrO   altsr   allvcvfsr  r  argvr  r  dtimedsizedp_dirdp_fnrB  r>  dp_absr  rr  rV  zigscoreduperjorig_aphashes2wark2r  r   namesr.  srcdstvsrcrx  psrcr  zs1zs2rq  r  r,  hrud1remud2dvfr#  ap1ap2lutr  r  r   r+  inor  r  timesr   sU                                                                                       @rQ   r  zUp2k._handle_jsons  s   2:>??&z""4F44==(S"899>BiikI6
ISI4;d);~~b))uT2f:&tRZ(	((4.''C zz~~d#t+88<<%D--%C((##BvJ/Csyy}}Z01C399==34G99==(:BFF6NDD{39rJ 2o,G%)T4L>rDyy}}W%DHHNN,<JDAqT	!QJJ.IIr&z2f:r&z2f: FB1W/ZFw3t; -6
&PAHHQ"Y*D! FL	cxx((LLcR%(D[r&zdjj##7A!7DKA!#2J.D[[E$K0>@ 9L:AueVUB((.%2B2B42H(-fe(<"47F6V#3 % XXf-<<

3"++-::. uA ! %rzz5"++uf!A !HHQK"++- / !& & $ $ $ % % "6
 "6
 " A & )7#%a5AaD)
+4r&zD/@A,C "$c!1q#)RZ#74T#YH %F 31< 
 KKT
AsFE JKs9LFLP CtT2 !W6
BvJ6
Cf/"GCHH7DE"&":":7"CKGR3DIIrzz7SE~ZeUGR%@!@ADH- "GC%( 3;D	,..Fc&kF F*. &C:AHHQ#eCIIv.F(N"NOPJJsFE2f:>bCi,,.B $$&CD43;Y(8D1AGADD B FRZ<D XXd^::>Vr\! .:	 r&z2f:r&zB6
BvJ6
C99<<3bF"4DZZ2f:-qA

6
6
 A HHQKD	w&w6SY;NHH7DE!%!9!9'!BJGQ3DIIrzz7SEu}[eT7B%?!?@Idck&SYK2f:-6{bj06{bj0  FS[#f+FC6
BvJ6
CC Vc&k3v;GD<<c2DS(c3!GHvsVD\1C77 $&vJ6
BvJ#GD=#'t~~c/B#B$(NN3$7$>$>t$D %S#..!SYY.sC!HIGsVD\1C77$S#..  #Z/99,, ")94 @A +,88:S'*Sy3  4==0#t}}4 $c$i0G"( #%O#  'sm&*F&+F$&FF4L$7CD	K!# 5A%'VVAY_"CF5!4 /A Bw)+AA/  %RZF<*3 $bj#))G2D+CK $CKVc&kJ!iimmE2!&s6{CKV!MB!( $ $ $ / # # " #F #F $ 2 23v;F L #F #F #F #D	 ""B" $&QT$V $A&,S!n 4!vvg$+DHHc2r'{$K#$+.99c&k3v;*OCGH$DD!S[*3*,'',,tS[*IC25))CK25,,CK25))CK25CK2=c&k22NCK+.99c&k3v;*OC'*cz68jF(,1Cuh1NPQ(R/3/@/@eai/P(P&*mmD#s&CF#CKVc&kJ#yy||$'IIC*&*jjV&= $c3"V*QU V tyy||!\B13 <AQ <A <'DKKSYY;;NN113S["$V*CKBvJ6
3!*36
CIIg<N!OBvJ77&)ggkk6
6
6
6
+	'OCF LL3GGKK6
+GGKK6
BvJ7 !! $RZ0 I #AUCF# - 'ABw!#AA' eF #A|F**1-
#
**3U;C"
  >>#f+s6{;AA#FD*.6==&CD FFF-F	C KCII%		Z377==0BvJ#''--4O CII-a1U3v;FS[IJ!asxx|':':[[dii&7&7S[#N041D	 KFF6N6
Os3v;'77		vJ#''..0vJ#'',,.U
;KKbj%*c&k3v;%WXNN))+s6{CKVEB -s2f:?EIIb%/HH72s6{BvJ:WWX J3Jw	C( Kd % %!)B KKfe(<=$ %p E#yy||C IA HHQ$[1 #D	n=7;fs 3@*#&88??3#7$+DHHc3$?'*$) (+
 !=tMM#f+.223v;E^ ! ;HH"6::J;s   *AD0 AD:AD:$BAE 1	AE:;AE:(AE?$/AF: "+AG 9AH AH A;B1AI D0AD7E 	AE7E	AE2E2AE7E?4AF7F:AGG;AHH(AI I	AI,IAI'I'AI,c           	         |d   }|d   }| j                   j                  r|S t        ||      }|j                  d      xr t        j
                  j                  |      }|rfdt        |d         j                         v rHt	        j                  |      j                  }|d   }	|	|k  rd}
| j                  |
||	z
  ||	|fz         d}|d   }| j                  j                  |      xs i }|r| j                  d	|       d }t	        j                  |      }	 t        |d
   |      }t        |j                  d            }| j                  ||      \  }}}}}}| j!                  ||||d|j"                  |       |r|j&                  j)                          	 t+        | j                  ||       | j                   j,                  r|j/                  dd      }n%| j0                  j2                  j5                  |      }t7        |d|d||fz  |      \  }}|j9                          |S # t$        $ r}| j                  d|       Y d }~d }~ww xY w# |r|j&                  j)                          w w xY w)Nr  r:  r)  r  r  z?will not overwrite; server %d sec newer than client; %d > %d %rFr   zreplacing existing file at r  r  Tzskipping replace-relink: :rf   r  -%.6f-%sfdirsuffixr  )rl   r9  r$   r  r   r  r  rj  r  rM  rw  rm   r   r=   r  _find_from_vpath_forget_filerz  rh  r}  r~  rA   plain_ipr)  rj   iphashr-  r/   r  )rP   r  r"  tsfnamerB  r)  owmtsmtcrO   r   r  r   r  vrelr  r  r  rr  diprA  r  s                          rQ   r  zUp2k._untaken@  s0   F[99<<L4WWY7CHHOOB$7$#c)n-3355((2,''Cf+CSyUcCic26676{ZZ^^D!'RHH<=C"B	,S[%0RVVG_-(,(=(=dD(I%T1aA!!$c4rzz5Q NN))+DHHb"%99**S#&C((//##B'CS	)
3 	
	
+  @">??@ NN))+ s+   A#H* *	I3II II I3Nc	                    ||k(  s|r||k(  rd}	t        |	|||fz        |r+d}	|r|	dz  }	| j                  |	j                  |||             | j                  j                  ry d}
	 d|v rt        d      |s|j                  d      st        d      |}|}t        j                  t        j                  j                  |            j                  }t        j                  t        j                  j                  |            j                  }|dk(  s|dk(  rt        t        j                  d	      ||k(  rt        rd
nd}t!        j"                  ||      }t!        j"                  ||      }d}t%        ||      D ]  \  }}||k7  r n|dz  } |dkD  r-||d  }t'        ||d        dz
  }d|z  dj)                  |      z   }t        r$|j+                  dd      }|j+                  dd      }|r6t        j                  j-                  |      rt/        | j                  ||       	 d|v r3t        j0                  t3        t5        |            t3        |             d}
|
st8        rct;        |      j=                  |       t        j                  j-                  |      sR	 t/        | j                  ||       d}	t        |	||fz        t        j>                  t3        |      t3        |             d}
|rI|
rtH        r@tK        tM        jL                               tK        |      f}t        jN                  ||d       y y y # t         $ r7}| j                  dt7        |      z          d|v rt        d      Y d }~d }~ww xY w#  Y xY w# t         $ r}tA        |      dk7  r| j                  dt7        |      z          t        j                  jC                  |      r|}nI|r"t        j                  jC                  |      r|}n%d}	| j                  |	d       t        |	|||fz        tE        jF                  t3        |      t3        |             Y d }~cd }~ww xY w)Nz8symlinking a file to itself?? orig(%s) fsrc(%s) link(%s)z1linking dupe:
  point-to: {0!r}
  link-loc: {1!r}z
  data-src: {2!r}Freflinkdedupzdedup is disabled in configr   zfilesystem does not have st_devz[\\/]r(  r   z../r'  hardlinkTzcannot hardlink: hardlinkonlyz symlink-fallback disabled in cfgz0the created symlink [%s] did not resolve to [%s]zcannot link; creating copy: z>BUG: no valid sources to link from! orig(%r) fsrc(%r) link(%r))(rh  rm   r   rl   r9  r  r   rM  r  r  dirnamerN  OSErrorerrnoENOSYSr   ru   r   r  r  r  r)  r  rA   linkr%   r   r  r   rB   
symlink_tosymlinkrj  rE  rk  rl  r   r   r   r  )rP   r  r  r   verboser  r  fsrcis_mvrO   linkedlsrcldstfs1fs2splnsrcndstncr#  br*  hopsrr  csrcr  s                             rQ   r  zUp2k._symlinkt  s    #:$43;JAAdC 0011EA**HHQXXc3-.99<<E	2E!	**7!3 =>>DD((277??3/077C((277??3/077Cax3!8ell,MNN")hsxxS)xxS)dO DAqAv!GB 6rs)CtBCy>A-D 4<#((3-7D||C.||C.chhooc*#u-H&GGE'#,/s<!F J))$/88??3/!#DHHc59 O'T4L(899JJuT{E$K8 7%s4y1EIIc5%( $+4A  H,tBx78!U*#$FGG +H!   	22w)#7$r(BCxxs##((//$/TAS$$4 455LLteCj11	2s]   'F>N &7L> AN N 6<N >	M>,M93N 9M>>N NN 	QB>QQc                 :   | j                   5  | j                  5  t        j                         x| _        | j                  |<   | j
                  |   j                  |      }|sndj                  | j
                  |   j                         D cg c]  }| c}      }| j                  dj                  ||             t        ddt        z         d|vrt        j                         |d<   t        |      dkD  rt        |d         dk  rg }t               }|d   D ])  }	|	|vs|j                  |	       |j!                  |	       + 	 |j#                  |d	         }
|d	   g}|dd  D ]A  }|
dz  }
||
   }	|	j%                  |      sd}t        d|||	fz        |j                  |	       C |}|D ]  }	|	|d   vrTdj                  |	      }|dj                  |d         z  }| j                  |       d}|	|d   vrd}t        d||	fz        |	|d   v sft        |d         }|d   j#                  |	      }d}t        d|j                  ||	|||d                t'        |d         }g }g }|D ]m  }	t)        |d         D cg c]  \  }}||	k(  s| }
}}|
st        dd|	      |
D cg c]  }||z  	 }}|j                  |       |j                  |
       o t+        ||dd        D ]3  \  }}|d	   |d	   z
  |z
  }|sd}t        d|||d	   |d	   |d   fz         t-        |d   |d   |d         }|d   s[t.        j0                  j3                  |      }|d	   d	   |kD  r1d}|j                  |d   |d	   d	   |d	   d	   |      }t        d|      d|d   	<   d d d        d d d        t        j                         d<   ||d   |d   |d   fS c c}w #  t        dd
|d	   d      xY wc c}}w c c}w # 1 sw Y   bxY w# 1 sw Y   fxY w)NrF   zunknown wark [{}], known: {}  zunknown warkr  r   ,   r  r   zunknown chunk0 []zAnext sibling chunk does not start with expected prefix [%s]: [%s]r  zchash = {} , need:
r2  z"already got that (%s) but thanks??zunknown chunk wtf: %sr  z<that chunk is already being written to:
  {}
  {} {}/{}
  {}r  r  zunknown chunk zTonly sibling chunks can be stitched; gap of %d bytes between offsets %d and %d in %sr   r  r  r  zplease upload sequentially using one thread;
server filesystem does not support sparse files.
  file: {}
  chunk: {}
  cofs: {}
  flen: {}r  r  )r~   r   r   r   r   r   r  r  r   rm   r   r   r   r  r   r  rl  r  r  up2k_chunksizer  r  r$   r   r  r  )rP   r   r  chashesr"  r   knownuniqr  chashnchunkexpandedr9  rO   r   rG  r   	chunksizecoffsetsnchunksrM   r   ofsofs1ofs2gapr  cur_szs                               rQ   handle_chunkszUp2k.handle_chunks  s    ZZ M	# M	#/3yy{:DK$,,t,--%))$/CT]]4-@-E-E-G!H!!HI7>>tUKLS.7":;;C!YY[E
7|aC
Ob$8e [ 'EC'E*'M!ZZ
3F $AJ<%abk +FaKF LE ++F3_$S!vuo*=>>OOE*+ #  SF+077>C499S[11CHHSM<ACK/3 a5(l33CK'S[)Bf+++E2CYA ahhtUCS[&QRRS  's6{3IHG  '(1#f+(>M1!u*!MM E&CDD.45y1}55$v&' "(HQRL9 P
dAwa(I5nA a3Qa#f+*N&NOO	P Vc&k3v;?Dv;))$/A;q>F* jAVgajmXa[^VTA a.(!"CK[M	# M	#^ iikF	8T3v;FSQW[XX[ "I"M wqz&KLL@ N 6uM	# M	# M	# M	#s   PA3P	OBP$$P	O"B8PA/PO:O:P/P 
;APBP$PP"O77PP	
PPc                    | j                   j                  d      sy| j                  j                  d      s| j                   j                          y	 | j	                  ||||      | j                  j                          | j                   j                          S # | j                  j                          | j                   j                          w xY w)NF)rG   r6  )r~   r  r   r  _confirm_chunks)rP   r   r  r  s       rQ   fast_confirm_chunkszUp2k.fast_confirm_chunks.  s     zz!!%(~~%%e,JJ 	!''dGWENN""$JJ  NN""$JJ s   B 6Cc                     | j                   5  | j                  5  | j                  ||||      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wrK   )r~   r   r&  )rP   r   r  writtenlockeds        rQ   confirm_chunkszUp2k.confirm_chunks<  s]     ZZ 	E 	E''dGVD	E 	E 	E 	E 	E 	E 	Es!   AA	AA
	AAc                 N   	 t        j                          x| _        | j                  |<   	 | j                  |   |   }t	        |d   |d         }t	        ||d         }t	        ||d         }|D ]  }
|d   j                  |
d         	 |D ]  }
|d	   j                  |
        	 t        |d	         }|dkD  r||fS | j                  j                  r| j                  ||       ||fS # t
        $ r}	dd|	dfcY d }	~	S d }	~	ww xY w# t
        $ r}	dd

d|	fcY d }	~	S d }	~	ww xY w)Nr   r  r  r  zconfirm_chunk, wark()r  r  zconfirm_chunk, chash(z) r   )r   r   r   r   r$   rh  r
  r  r  rl   r9  r  )rP   r   r  r)  r*  r"  r  r  r  rr  r  r  s               rQ   r&  zUp2k._confirm_chunksB  s=    /3yy{:DK$,,t,=mmD)$/S[#f+6D#f+.D#f+.   -Ft,-G$ .EK&&u-. c&k"CQwCxyy||T4(Cx)  =<<<=  GE2FFFGs<   AC) 
D )	D2C?9D?D	D$	DD$D$c                     || _         | j                  5  | j                  5  | j                  ||       d d d        d d d        | j                  r| j                          y y # 1 sw Y   /xY w# 1 sw Y   3xY wrK   )r   r~   r   _finish_uploadr   r  )rP   r   r  r   s       rQ   finish_uploadzUp2k.finish_uploadb  sj     ZZ 	, 	,d+	, 	, ??  	, 	, 	, 	,s!   A.A"A."A+	'A..A7c                    	 | j                   |   |   }t        |d   |d         }t        ||d         }t        ||d         }|d	   r5| j                  t	               d       d
}t        d|||d	   t        fz        |j                  d      xs t        j                         }	| j                  |   }
t        | j                  |||
       t        t        j                               t        |d         f}d}| j                  ||t        j                  j                  |      |fz         	 t        j                  ||       d}|j!                         D cg c]  }||   	 }}d}	 |d   }| j"                  j$                  |d      }|j                  d   }|r8|dkD  r3||k  r.|	||z
  z  }	d}d}| j                  |j'                  |	||             |j)                  |	        | j*                  |
g| r| j                   |   |= nWdj!                         D ]  }||=  |j-                  dd       t        |d         |d<   g |d<   d|d<   | j/                  ||       |r/| j0                  5  | j0                  j3                          ddd       | j4                  j-                  |g       }|sy| j6                  j                  |      }|D ]r  \  }}}t        |||      }t8        j                  j;                  |      r4| j=                  ||| j                  |   |       |sY | j>                  ||
|||g|dd   t |r|j@                  jC                          yy# t        $ r4}| j                  t	               d       t        dd|t              d}~ww xY w#  | j                  d|d|d       Y xxY wc c}w #  Y xY w# 1 sw Y   FxY w)r  r   r  r  r  r   r  zfinish_upload, wark, Nr  z+finish_upload %s with remaining chunks %s%sr>  r  z+no more chunks, setting times %s (%d) on %rzfailed to utime (z, r.  z6prel name lmod size ptop vtop wark dwrk host user addrFr  r  r  Tz(using client lifetime; at={:.0f} ({}-{})zhost tnam busy sprs poker  r;  r  r8  r  rg   )"r   r$   rh  rm   r+   r   r   r  r   r   r!   r   r   r  r  r  r   ro   r   r   r  idx_warkr
  r  r   r   r   r   r  r  r  r  r}  r~  )rP   r   r  r"  r  r  r  rr  rO   uptvflagsr  rq  r   z2wake_srr  ro   vltr   dupesr   r  r.  r  d2s                             rQ   r0  zUp2k._finish_uploadj  s   	K--%d+CVc&k2Dc&k*Cc&k*C
 v;HHVXq!=Aa4Vg">>??ggdm*tyy{D!DHHc3/TYY[!3s6{#349eSXX--c2C889	@IIc5! F hhj)c!f))
	f+C((##CK0C))J'CsQw39sSy >#sC01 			#4==%"%d#D)/557 FGGE4 CICICKCKLLt$!! .  ++-. ""3+hhll4 ! 	@LBDtR$Bww~~b!MM#r4::d#3$M?CR?12?	@ NN!!# G  	KHHVXq!RIJJ	K&	@HH3>? *	. .sC   AM .N N?)A*O -O	N(/NNN<O	Oc                    | j                   |   }|r|j                  |       t        |      | j                  j                  k  ryt        |      t        | j                  j                  dz        z
  }d}| j                  |j                  |t        |      |             |d| D ]!  }| j                  |   j                  |d       # ||d | j                   |<   y)r  Nri   z3up2k-registry [{}] has {} droppables; discarding {})
r   r  r  rl   reg_capr   rm   r   r   r
  )rP   r   r  oldsrM   rO   r   s          rQ   r  zUp2k.regdrop  s    ~~d#KKt9		)))IDII--122A$D	1-.bq 	-AMM$##At,	-#ABxtrS   c                    | j                   j                  |      }|syt        j                         x| _        | j                  |<   	 | j                  ||||||||||	|
||||       |j                  j                          d| j                  |   v r7| j                  j                  ||	|||||f       | xj                  dz  c_        y# t        $ r2}| j                  |i       }t        | j                  ||d           d }~ww xY w)NFr   r  T)r   r  r   r   r   r  r}  r~  rh  r  r"   rm   r   r   r  r   )rP   r6  r  r.  r  r@  r   r  r  r  r  usrrB  r>  skip_xaur   rr  r   s                     rQ   r4  zUp2k.idx_wark  s   " hhll4 +/99;6dll4(	KK" NN!!# DJJt$$IIMM4BB;<KK1K  	##D"-AdhhAaD)	s   9C
 
	D-D  Dc           	         d}	 |j                  |||f      }|j                  r/| j                  |xx   |z  cc<   | j
                  |xx   dz  cc<   y y #  |j                  |t        | j                  ||            }Y gxY w)Nr  r   )r   r6   r   r|  r   r   )rP   r  r  r.  r@  r  rs          rQ   r  z
Up2k.db_rm  sz    6	=

3R)A ::LL"NN2!# 	=

3dllB ;<As   A )A?c                    | j                  ||||       |sd}nd|v rdn|}d}|
t        |      ||||t        |xs d      f}	 |j                  ||       | j
                  |xx   |z  cc<   | j                  |xx   dz  cc<   |rdn|j                  d      }t        |||      }|rt        | j                  d	| d
||t        |||      ||| j                  j                  t        |||      |      ||||xs t        j                         d      }|sVd}| j                  |d       t        | j                  ||       | j                  |   j!                  |	d	       t#        d|      |j                  d      }|rt%               }|D ]J  }| j&                  j)                  |      }|j+                  |rt        |j-                  d            nd       L d}|D ]  }	 |j                  |||
d	d ||f        | j.                  r6d| _        | j0                  5  | j0                  j3                          d	d	d	       |r"|r|j5                         | j6                  j8                  v rd|v r|j;                  dd      nd|f\  }}	 d}|j                  |||f      j=                         d   }|j5                         } | | j6                  j8                  v r^| j6                  j>                  jA                  |       }!| j6                  j>                  jA                  |j5                               }"|"|!k  }#nd}#|#r*	 |j                  d||f       |j                  d|||f       d|vrW	 d}d}$	 |j                  |||f      jB                  s|j                  |$||f       |sy	d|v r|j;                  dd      d   nd}Qy	#  t        | j                  ||      \  }}|
t        |      ||||t        |xs d      f}|j                  ||       Y xY w#  t        | j                  ||      \  }}|j                  |||
d	d ||f       Y mxY w# 1 sw Y   xY w#  d}#Y *xY w#  Y xY w#  Y y	xY w)r   r6  no_db_ipz1.1.1.1z%insert into up values (?,?,?,?,?,?,?)r   r   FxauNzxau.up2kz#upload blocked by xau server configr  r  rC   zinsert into iu values (?,?,?,?)r  r(  z%select fn from cv where rd=? and dn=?Trk  rl  nodirszz)update ds set nf=nf+1, sz=sz+? where rd=?zinsert into ds values(?,?,1))"r  r   r   r6   r   r   r   r  r$   r2   rm   ro   r  r   rA   r   r
  r   r   r   r  rl  r  r   r   r   r  rl   ru  r  r  rt  r  r|  )%rP   r  r6  r  r.  r  r@  r   r  r  r  r  r@  rB  r>  rA  db_ipr  r   rF  r  r  rO   r  r  r  r  r  r  r  r  db_cvdb_lcvidx_dbidx_fnadd_cvq2s%                                        rQ   r  zUp2k.db_add  sP   & 	

2r2r"E ",v!5I2E5CGRRBG!=	JJsA 	RBraeVZZ%6D"b!dB#""5r2#6<!diikB" 9A#v.d#''d3S!n$jj5C 5LL'',1AGGAJ!45 2A <<JJq2uSbz2r":;< "'%% 2$$//12 "tyy'?'?? -02Iryya(B8HC;

1sCj1::<Q?TYY555!YY1177?F!YY1177
CF#f_F!F JJCc3ZPJJ>c2O F"
?3::a"b2;;

2Bx0143*1-B 	 #a	4<<R0FBBRUCaLAAJJsAX<"4<<R8FBJJq2uSbz2r":;2 2"sP   N; PQ	B/Q 6)Q  $9Q( Q( ;AP5Q	QQ Q%(Q,c           	      v   d}i }i }	|D ]i  }
|r.|d   dk  r&| j                  dj                  |d         d        n9| j                  |||
|||      \  }}}||z  }|D ]  }d||<   	 |D ]  }d|	|<   	 k |	D ci c]
  }||vs|d }	}t        |      }t        |	      }dj                  ||||z         S c c}w )Nr   hit delete limit of {} filesr   rg   z$deleted {} files (and {}/{} folders))rm   r   r  r  )rP   r  rB  vpathsr  rm_upunpostn_filesrL  r  r,  r#  r  rL   r   iokings                    rQ   	handle_rmzUp2k.handle_rm  s      
	Bs1v{7>>s1vFJooeRS%HGAq!qLG 1 1
	 .q!2+ad.."g"g5<<Wc3QT9UU	 /s   9	B6B6c                    t        j                          | _        d}|s?g dg} | j                  j                  ||g|d    \  }	}
|	j	                  |
      \  }}n#ddgg} | j                  j                  ||g|d    \  }	}
|	j	                  |
      \  }}|j
                  }| j                  5  | j                  5  | j                  j                  |i       j                  dd      }|dv r|xs d	nd}|d
v r|xs d	dfnd }|r| j                  j                  |i       ni }|j                         D ]  \  }}|r||d   k7  s	|r|d   |vrt        |d   |d         }t        r|j                  dd      }||k(  sJ|d   |k7  rd}t        ||d   |fz        |j                  t!        |d   |d               } n |r|}t        j                          }n=| j"                  j$                  sd}t'        d|      | j)                  ||      \  }}}}}}d d d        d d d        d}s|dz  }nA||k7  r|dz  }n6t        j                          | j"                  j$                  z
  k  r|dz  }nd}|rt'        d|      |j
                  }|j                  |d      }| j                  | j*                  |<   t,        j.                  j1                  |      \  }}	 t3        j4                  |      }t7        j8                  |j:                        }| j"                  j<                   }|r&|	j?                  d|
g ||d|d      }|rt'        dd      t7        j@                  |j:                        st7        jB                  |j:                        r*tE        |      d   } tE        |      d   }!|| |!||dfgg i fg}n%| jG                  d|j:                  |fz         dg g fS |j                  j                  d      }"|j                  j                  d       }#d}$|D ]  \  }%}&}}}'}(})|'D *cg c]  }*|*d   	 c}*D ]  }|r;|dxx   dz  cc<   |d   dk  r&| jG                  d!jI                  |d         d"        `t        ||      }+t3        j4                  |+      x}},t7        j@                  |j:                        r	 t3        j6                  |+      }|&d|jK                  d      }-|%jL                  d|-jK                  d      }| jG                  d#|d$|+       |s |%j                  |-|g|d    }|"r}tO        | jF                  d | d|"|+|d|| j                  jQ                  ||      |,jR                  |jT                  |t        j                          d      sd%}| jG                  ||+fz  d       ~|$dz  }$| j                  5  | j                  5  d }.	 |%j
                  }tW        |%j                  j                  d&            }/| j)                  ||-      \  }.}}}}}| jY                  ||-|.|d|jT                  |/       |.r|.jZ                  j]                          	 d d d        d d d        t_        | jF                  |+|%j                         |r#t_        | jF                  ||%j                         d}|#stO        | jF                  d | d |#|+|d|| j                  jQ                  ||      |,jR                  |jT                  |t        j                          d         |rta        | jb                  |d|d      \  }0}1ng x}0}1|r-te        t,        j.                  jg                  |      |      \  }2}3ng x}2}3|$|0|2z   |1|3z   fS # 1 sw Y   ixY w# 1 sw Y   nxY w#  t'        dd      xY wc c}*w #  Y xY w# |.r|.jZ                  j]                          w w xY w# 1 sw Y   xY w# 1 sw Y   xY w)'Nr6  )TFFTr   FTr4  r   r5  r2  r7  r3  r:  r9  r  r  r'  r(  r   z!job.ptop [%s] != vol.ptop [%s] ??r  z/the unpost feature is disabled in server configr  zyou cannot delete this: zfile not foundznot uploaded by (You)zuploaded too long agoz)file not found on disk (already deleted?)ri   zcannot unpost folderszrm: skip type-0%o file %rxbdxadrP  rg   zrm r  z'delete blocked by xbd server config: %rr  )4r   r   ro   r  get_dbvr   r~   r   r   r   r   r$   r   r)  rh  	canonicalr=   rl   rS  r   r  r   r  r  r   r   r  rM  r{  ry  r  walkrx  r}  r>   rm   r   r*  r  r2   r  rw  rz  r  r  r}  r~  rA   r0   rn   r1   r  )4rP   r  rB  r  r  rR  rS  partialpermsetsvn0rem0r  r  r   abrt_cfgr:  r9  r  r  r"  jremrO   r  datr  atopadirr.  r  is_dirscandirr  voldir	vpath_dirrY  rZ  rT  dbvvremr  r  vdr   r^   r  volpathr   r  rL  r  ok2ng2s4                                                       rQ   r  zUp2k._handle_rm  s    iik23H$UE@HQK@ICkk$'GB H$UE@HQK@ICkk$'GB;;D LT^^ L::>>$377	1E'/6'9
dr19V1C$-5=dmm''b12!$ ID#V!4VD!8  Vc&k:D#||D#6s{v;$. CA"+AVd0C,C"DD"$,,uS[#f+/N"O C))+C99++M$S!n,+/+@+@s+K(Aq!QS7L L: +A%%,,tyy{TYY%5%555,,S!n${{||C'![[T77==&b	K4B\\"**-F ii*** T2uh7DIAS"9::\\"**%bjj)AC[^Fua(IfiQy"bABAHH0BJJ3EEFb"9hhll5!hhll5!12 P	-Cq$r2%*+qt+ OFaKF1vz!?!F!Fs1v!NPQRb/99W--S<<

+ XXg. '+B/66s;$'IIw7>>sC%9:=!=A"**5%8

		" FgZ3 1ZZ 4 4C	4"|| $SYY]]7%; <040E0EdG0T-T1aA)) '3dBJJ NN1134 4 '3995DHHgsyy9 G**5%8

		OP	d DMM7D$BFBLB !6=HCNC#S"s(**EL L L Lb	KIJJ* ,T NN113 4 4 4 4s   2^?C ]> B]>^4^ $^)7^._%_	A*^6=_	_%>^	^^^&.^36__	_"_%%_/c           
      H   ||k(  s|j                  |dz         rt        dd      | j                  j                  ||dd      \  }}| j                  j                  ||dd      \  }}|j	                  |      \  }	}
|j                  |d      }t               }t        j                         x| _        | j                  |	j                  <   t        j                  |      }t        j                  |j                        st        j                  |j                        rP| j                   5  	 | j#                  |||||      }|D ]  }|j$                  j'                           	 |cd d d        S t        j(                  |j                        st        dd|j                  fz        ddgg}| j*                  j,                   }||j.                  j0                  v rdnd}|j3                  d	|g ||||d      }| j                   5  	 |D ]	  \  }}}
}}}}|D ]  }t        j                         x| _        | j                  |j                  <   dj5                  d
 |j6                  ||d   fD              }|j                  |dz         s2| j9                  t;               d       d}t        d|||t<        fz        ||t?        |      d  z   }| j#                  |||||        |D ]  }|j$                  j'                           |jA                           	 |D ]  }|j$                  j'                           	 d d d        y# |D ]  }|j$                  j'                           w xY w# 1 sw Y   xY w# |D ]  }|j$                  j'                           w xY w# 1 sw Y   yxY w)Nr(  r  z%cp: cannot copy parent into subfolderTFzcannot copy type-0%o filer   ri   r6  c              3   &   K   | ]	  }|s|  y wrK   rW   r  s     rQ   r  z!Up2k.handle_cp.<locals>.<genexpr>z       'Qaq'Q   r   zcp: bug at %r, top %r%sr  r   )!r  r   ro   r  r[  r\  r   r   r   r   r   r   rM  r}  ry  rx  r~   _cp_filer}  r~  r{  rl   r  r  udotr]  r  r  rm   r+   r   r  rm  )rP   r  rB  svpdvpsvnsremdvndremsvn_dbvr  sabscursr  r  r   r_  rh  dotsr  rk  rl  re  r  r  rm  r.  svpfrO   dvpfs                                 rQ   	handle_cpzUp2k.handle_cpS  sB   #:c	2EFFHHLLeT59	THHLLeUD9	T[[&
}}T5)7;yy{Bdll7#3#34XXd^<<

#t||BJJ'? .--r3TBC! .++-.   ||BJJ'9RZZMIJJ5M?ii*** SWW\\)qq HHRr5(D'4HZZ 	**9: !5Cq$r2# 	CCG99;Ndll3<<&@"xx'QCIItRU3K'QQ#sSy9 HHVXq1 9A"(a4g2F.F"GG"T#c(*%55eRtTB	C " .++-.JJL!   *ALL'')*%	** M " .++-.	 N  *ALL'')*%	** sI   )M%+L? #M%NDM2!N?#M""M%%M/2#NNN!c                 t   | j                   j                  ||dd      \  }}|j                  |      \  }}	| j                   j                  ||dd      \  }
}|
j                  |      \  }
}|j                  |d      }|
j                  |      }t	        |      \  }}t
        j                  j                  |      rt        dd      t        j                  |      x}}t        j                  |j                        rd}	 t        j                  |      }nBt        j                  |j                        s!| j                  d|j                  |fz         yd}|j                  }|j                   }|j"                  j                  d      }|
j"                  j                  d      }|rrt%        | j                  d	| d|||d|| j                   j'                  ||      |||t)        j(                         d      s#d
|}| j                  |d       t        d|      t        j*                  t,        j                  j/                  |      |
j"                         | j1                  |j2                  |	      \  }}}}}}| j4                  j                  |
j2                        }|ry|r||k7  r| j7                  |||       |j9                  |       |r`| j;                  |i |||||
j2                  |
j<                  ||dd|xs d|xs d       |j9                  |       n| j                  d|       	 |r||k7  rt        j>                  |      }t,        j                  jA                  t,        j                  j/                  |      |      }t
        j                  jC                  |      }| jE                  |||
j"                  |       n| jE                  |||
j"                  |       |rOt%        | j                  d	| d|||d|| j                   j'                  ||      |||t)        j(                         d       y#  Y xY w# tF        $ rl}|jH                  tH        jJ                  k7  r | j                  d|jL                  d|d|       tO        |      tO        |      }!} t,        j                  jQ                  |       }	 tS        jT                  | |!       n#  	 tW        | j                  ||
j"                         n#  Y nxY w|s 	 t-        j>                  |       }"t-        jX                  |"|!       n'#  tW        | j                  ||
j"                          xY wY nxY w|rH	 t[        t)        j(                               t[        |      f}#t        j\                  ||#d       n#  Y nxY wY d	}~d	}~ww xY w) mutex(main) me;  will mutex(reg)TFr  zcp2: target file existszskipping type-0%o file %rr6  xbcxacNz#copy blocked by xbr server config: r     r  r   not found in src db: r3  zusing plain copy ():
  r  r   )/ro   r  r[  r\  r>   r   r  r  r   r  rM  rx  ry  r}  rm   rw  rz  r   r2   r  r   r  r  r  r  r   r   
_copy_tagsrl  r  r  readlinkr  r^   r  r  r  EXDEVstrerrorr%   islinkrk  rl  rA   r  r   r  )$rP   r  rB  rw  rx  r  ry  rz  r}  srem_dbvr{  r|  r~  dabsr  r  r  r  is_linkftimer   r  r  rO   r  r]   ftime_fsize_r>  r  dlinkrr  b1b2r<  r  s$                                       rQ   ru  zUp2k._cp_file  s    HHLLeT59	TKK-HHLLeUD9	TKK%	T}}T5)}}T"$<S88??4 78899T?"S<<$GXXd^ bjj)HH0BJJ3EEFG

iimmE"iimmE"""3.		" @CDAS!n$RWW__T*syy9(,(=(=h)
%Avvr2 XX\\#,,'bBhB*HHRLLLIIH"G!  HH#78*	29 T*RWW__T%:EB((/eT3995AdD#))%@F ""3.		$ I^  	xx5;;&HHR[[$PTUV4[%+BggnnR(GR$DHHdCII6RBJJr2&DHHdCII6 '
  -s5z:EIIdE51?	s    P9 0B7Q 9P>
V7A7V2SV2U!S?>U?TU
+T65U6$UUV2%?V%$V2%V)'V22V7c           
      @
   ||k(  s|j                  |dz         rt        dd      | j                  j                  ||ddd      \  }}|j	                  |      \  }}|j                  |d      }	t               }
t        j                         x| _        | j                  |j                  <   |st        dd      t        j                  |	      }t        j                  |j                        st        j                   |j                        rP| j"                  5  	 | j%                  |||||
      }|
D ]  }|j&                  j)                           	 |cd d d        S t        j*                  |j                        st        dd|j                  fz        g dg}| j,                  j.                   }|j1                  d	|g ||d
|d      }|D ]  \  }}}}}}}||k7  st        dd       |j1                  d	|g ||d
|d      }| j"                  5  	 |D ]  \  }}}}}}}||k7  rt        dd      |D ]  }t        j                         x| _        | j                  |j                  <   dj3                  d |j4                  ||d   fD              }|j                  |dz         s2| j7                  t9               d       d}t        d|||t:        fz        ||t=        |      d  z   }| j%                  |||||
        |
D ]  }|j&                  j)                           |
j?                           	 |
D ]  }|j&                  j)                           	 d d d        tA        | jB                  |d|	d      \  }}||fD ]4  }tE        |      D ]"  }|j                  |	      s2| j7                  t9               d       d}t        d|||	t:        fz        |t=        |	      d  jG                  tH        jJ                  d      jM                  d      } tO        ||       }!	 | j                  j                  |!|dd      \  }"}#|"j                  |#      }$t        jP                  |$|"jR                  d          d|"jR                  v r1t        jT                  |$|"jR                  d   |"jR                  d          % 7 y# |
D ]  }|j&                  j)                           w xY w# 1 sw Y   xY w# |
D ]  }|j&                  j)                           w xY w# 1 sw Y   xY w#  Y xY w)Nr(  r  z%mv: cannot move parent into subfolderTFzmv: cannot move a mountpointzcannot move type-0%o file)TFTr6  ri   z(mv: source folder contains other volumesc              3   &   K   | ]	  }|s|  y wrK   rW   r  s     rQ   r  z!Up2k.handle_mv.<locals>.<genexpr>W  rs  rt  r   r   zmv: bug at %r, top %r%sr  zmv_d: bug at %r, top %r%sr  chownuidrx   r   )+r  r   ro   r  r[  r\  r   r   r   r   r   r   r  rM  r}  ry  rx  r~   _mv_filer}  r~  r{  rl   r  r]  r  r  rm   r+   r   r  rm  r0   rn   reversedr)  r  rv  r3  r=   mkdirr   r  )%rP   r  rB  rw  rx  ry  rz  jailjail_remr~  r  r  r  r   r_  rh  r  rk  rl  r  re  r  r  rm  r.  r  rO   r  rm_okrm_ngr*  r+  r  r,  r{  r|  daps%                                        rQ   	handle_mvzUp2k.handle_mv(  s   #:c	2EFFHHLLeT5$?	TT*h}}T5)48IIK?dll4==1<==YYt_<<

#t||BJJ'? .--r3TBC! .++-.   ||BJJ'9RZZMIJJ'(ii*** HHRr5(AwE12 	N-Cq$r2d{S"LMM	N
 HHRr5(AwEZZ 	**9: !5Cq$r2d{$S*TUU# 	CCG99;Ndll3<<&@"xx'QCIItRU3K'QQ#sSy9 HHVXq1 9A"(a4g2F.F"GG"T#c(*%55eRtTB	C " .++-.JJL%!(  *ALL'')*-	*2 dmmWdD!Du5> 	Csm }}T*HHVXq)3A a2tW*=&=>>To--bffc:AA#F3_ $Rt DIC---CIIc399Y#78#))+		#syy'759IJ	$ A " .++-.	 Z  *ALL'')*-	* 	*TsV   SR2.#STD S%/!TBT2#SSS"%#TTTTc                 4   | j                   j                  ||ddd      \  }}|j                  |      \  }}| j                   j                  ||dd      \  }}	|j                  |	      \  }}	|j                  |d      }
|j                  |	      }t	        |	      \  }}|j                  d      d   }|j                  d      d   }|j                  d      s|j                  d      rI| j                  j                  rt        dd      | j                  j                  r||k7  rt        dd      t        j                  j                  |      rt        dd	      dx}}t        j                  |
      x}}t        j                   |j"                        r7d}	 t        j                  |
      }t        j$                  |j"                        }|j&                  }|j(                  }|j*                  j                  d
      }|j*                  j                  d      }|rrt-        | j.                  d| d
||
|d|| j                   j1                  ||      |||t3        j2                         d      s#d|}| j/                  |d       t        d|      |j4                  |j4                  k7  }t        j6                  t8        j                  j;                  |      |j*                         |rPt=        |
      }d}| j/                  ||
||fz         t        j                  j?                  |
d      }tA        | j.                  |
|j*                         | jC                  |||j*                  d|       | jD                  jG                  |jH                         | jD                  jG                  |jH                         | jJ                  5  | jJ                  jM                          ddd       |rOt-        | j.                  d| d|||d|| j                   j1                  ||      |||t3        j2                         d       y| jO                  |j4                  |      \  }}}}}} | jP                  j                  |j4                        }!d}"|r|!r|!|k7  r| jS                  ||!|       tU        |j*                  j                  d            }#| jV                  5  | jY                  |j4                  |||||xs ||#      }"ddd       |s | j[                  ||j4                  ||||#      }"|jG                  |       |!r`| j]                  |!i |||||j4                  |jH                  ||dd|xs d| xs d       |jG                  |!       n| j/                  d|       	 |r|"rt_        t`        jb                  d      |r||k7  rt        jd                  |
      }$t8        j                  jg                  t8        j                  j;                  |
      |$      }$t        j                  ji                  |$      }$| jC                  |$||j*                  |d       tA        | j.                  |
|j*                         n"tk        | j.                  |
||j*                         |rOt-        | j.                  d| d|||d|| j                   j1                  ||      |||t3        j2                         d       y#  Y xY w# 1 sw Y   
xY w# 1 sw Y   xY w# t^        $ r}%|%j`                  t`        jb                  k7  r | j/                  d|%jl                  d|
d|       to        |
      to        |      }'}&t8        j                  jq                  |&      }	 ts        jt                  |&|'       n#  	 tA        | j.                  ||j*                         n#  Y nxY w|s 	 t9        jd                  |&      }(t9        jv                  |(|'       n'#  tA        | j.                  ||j*                          xY wY nxY w|rH	 ty        t3        j2                               ty        |      f})t        jz                  ||)d       n#  Y nxY wtA        | j.                  |
|j*                         Y d}%~%d}%~%ww xY w)r  TFr(  rG   rf   r  z,moving dotfiles is disabled in server configz.renaming dotfiles is disabled in server configzmv2: target file existsxbrxarNr6  z#move blocked by xbr server config: r   r  r  z'moving symlink from %r to %r, target %rr3  zxar.lnr   r  r   r  zsrc is symlink)r  r  zusing copy+delete (r  r  zxar.mv)>ro   r  r[  r\  r>   r   r  rl   	no_dot_mvr   
no_dot_renr   r  r  r  rM  rx  ry  r{  rw  rz  r   r2   rm   r  r   r   r  r  r  r   getmtimerA   r  r   rl  r  r   r   r  r   r  r  r   r  _relinkr  r  r  r  r  r  r^   r!   r  r%   r  rk  rl  r  r   r  )*rP   r  rB  rw  rx  r  ry  rz  r{  r|  r~  r  r  r  n1n2r  
is_dirlinkr  r  r  r   r  r  rO   is_xvoldlabsr  r  r]   r  r  r>  r  	has_dupesr  r  rr  r  r  r<  r  s*                                             rQ   r  zUp2k._mv_file}  s    HHLLeT5$?	TKK%	THHLLeUD9	TKK%	T}}T5)}}T"$<SYYs^BYYs^B==s!3yy""S"PQQ%%"(S"RSS88??4 788$$*99T?"S<<$GXXd^!\\"**5
 

iimmE"iimmE"""3.		" @CDAS!n$,,#,,.RWW__T*syy9DME9AHHQ$e,,-""4/BDHHdCII.MM%syy%bMA   +  +!! .  ++-. HHHH&&sE2IIK$ (,(=(=cllD(Q%Avvr2XX\\#,,'	bBhB*w/0E  --LL$Aw%	
  LLCLL$b%P	HHRLLLIIH"G!  HH#780	/9ekk+;<<29 T*RWW__T%:EB((/eT3995M$		2DHHdD#))<J ""3.		$ gX. .D \  !	/xx5;;&HHbkk4QUVW4[%+BggnnR(GR$DHHdCII6RBJJr2&DHHdCII6 '
  -s5z:EIIdE51DHHdCII..C!	/s   4[ [&&[3C:\  [#&[03[= 
b
A7b^b`!^>=`>_ `	+_54`5$``b$?a$#b$a(&&bbc                     |dd }|j                  d|f      j                         ry|j                  d|f      D ]  \  }}}|j                  d|||f        y)z,copy all tags for wark from src-db to dst-dbNr  z"select * from mt where w=? limit 1zselect * from mt where w=?zinsert into mt values(?,?,?))r   r  )rP   r  cdstr  r]   r  r   r   s           rQ   r  zUp2k._copy_tagsR  sf     "I<<<qdCLLN||$@1$G 	DGAq!LL7!QC	DrS   c           	      8   | j                   j                  |      }|syt        |      \  }}d}	 |j                  |||f      }|j                         }|r|\  }	}
}}}||	|
|||fS |d d d dd fS #  |j                  |t	        | j
                  ||            }Y UxY w)N)NNNNr6  Nz<select w, mt, sz, ip, at from up where rd=? and fn=? limit 1r6  )r   r  r>   r   r6   r   r  )rP   r   rl  r   r  r.  r  rL   hitr  r  r   rB  r>  s                 rQ   r  zUp2k._find_from_vpath^  s     hhll4 3BJ	<ABx(A jjl),&D%BeUB22D$b$..	<AuT\\2r:;As   A. .)Bc                    t        |      \  }d}	| j                  d|       |rl|rj| j                  dj                  |             |r| j                  |||d||      rd}	d}|rd}
|j	                  |
|dd f       | j                  |||       | j                  j                  |      }|rt        |      d	   |xs$ t        fd
|j                         D        d      }|r|j                  |      nd}|r|d   r4d}| j                  |      }| j                  |j                  ||             t        ||      }t        | j                  j                  |g             }|rd}| j                  ||fz         |sJ ||= |	S )z`
        mutex(main,reg) me
        forgets file in db, fixes symlinks, does not delete
        Fzforgetting zfound {} in dbr6  Tzdelete from mt where w=?Nr  r   c              3   h   K   | ])  \  }}|d    |j                  d      fv r|d   k(  r| + yw)r  r  r  N)r  )r  r   r1  sfnvdirs      rQ   r  z$Up2k._forget_file.<locals>.<genexpr>  sA      1qy!%%-88QvY$=N s   /2r  z!forgetting partial upload {} ({})z'...and forgetting %d links in dupesched)r>   rm   r   r  r   r  r   r  r  r   r  r$   r  r   r
  )rP   r   rl  r   r  	drop_tagsr@  r  srdr  r  r  r"  rO   pr  r  r  r  s                    @@rQ   r  zUp2k._forget_filey  s    $<S	D*+CHH%,,T23<<dD"c5A $I %I.ASb	|,JJsCb)mm%$<?D 4 #		
 D $(#''$-TCv;;A..s3AHHQXXdA./D$'++C45AAHHQ"Y'tIrS   c           
      	   g }t        ||      }| j                  rd}	|f}
n	d}	|dd |f}
| j                  j                         D ]  \  }}|s|r||k7  r|j	                  |	|
      D ]  \  }}|j                  d      s|j                  d      rt        ||      \  }}t        ||      j                  d      }||k7  s||k7  s]|j                  ||g       | j                  d|d|d	|         |sy
i }i }|D ]<  \  }}t        ||      }	 t        j                  j                  |      r|n|}||f||<   > |s	|s|rt        t        |j!                                     d
   }|j#                  |      \  }}| j                  d|d|       t        j                  j%                  |d      }| j&                  j)                  |      xs i }t+        | j                  |||       t        j,                  |t/        t1        j0                               t/        |      fd       | j3                  |||dd       ||f||<   |}|s%t        t        |j!                                     d
   }|j                         D ]  \  }}d}	 d}|}	 t5        d      D ]  }t        j6                  |      }t8        j                  j;                  t8        j                  j=                  |      |      }t        j                  j?                  |      }t        j                  j                  |      s n"||k(  sd} | j                  | |fz  d       d} tC        |      }||k7  rd} | j                  | ||fz  d       |}d}t        j                  j                  |      rtA        d|      ||k7  r	||k7  r|s9| j                  d|d|       | j&                  j)                  |d
         xs i }	 t        j                  j%                  |d      }tE        | j                  ||       | j3                  |||d|xs d
|d        tG        |      tG        |      z   S #  | j                  d|       Y {xY w# t@        $ r%}!| j                  d|!d|!d       d}Y d}!~!Td}!~!ww xY w# t@        $ r$}!d} | j                  | |!|!fz  d       Y d}!~!d}!~!ww xY w#  Y xY w)z
        update symlinks from file at svn/srem to dabs (rename),
        or to first remaining full if no dabs (delete)
        z!select rd, fn from up where w = ?z5select rd, fn from up where substr(w,1,16)=? and +w=?Nr  r  r(  zfound z dupe: rF   r   zrelink: not found: z	linkswap r  FT)r  rd   (   z$relink because level %d would break:rE  zrelink because walk failed: z; rg   z8relink because computed != actual destination:
  %r
  %rzbroken symlink: z2relink because symlink verification failed: %s; %rz
relinking z to )r  r  r  )$r$   r   r   r   r   r  r5   r=   r*  r  rm   r   r  r  r   rk  r   r
  r  r   r  r!   r  r   r   r  r   r  r  r  r  r^   rh  r   rA   r  )"rP   r  sptoprz  r  vcurr  r:  r~  r  r  r   r   r  r.  dvremr  linksr,  r+  rb  slabsr  r  r   alinkpartsr  faultyr  rM   rq  rO   rr  s"                                     rQ   r  zUp2k._relink  su    UD!4A7DHA"It$D) 
	KID#SSD[++a. KB==&"--*="2r]FBb"++C05=DEMLL$/HHtT5IJK
	K  	:HD"tRB: XX__R0Edr
"		: DU

-.q1E		%(ID#HHT59:""5%0BJJNN4(.BE$u5IIec$))+.B8%@MM%ue4M@+DKDtyy{+,Q/D!KKM 6	LE5D"*""2Y 
* \\$/!ww||BGGOOD,A2F"xx//5"xxt4!4< FA HHQ!Xq1%)F
* U^2:TAHHQ$^Q/D!F88??4(#E$CDDD=TT\& HHUD9:JJNN58,2Exx((6%/ MMeUE	D  i6	p 4y3u:%%[:B89J ! "HHRLaP!F""  *Hb"Xq))*sb   *Q	R,B%Q;4Q;A&R,27SQ8;	R)R$R,$R))R,,	S5SSS c                    t        |d         dkD  st        |d         dkD  rt        dd      |d   D ]8  }| j                  j                  |      rt        ddj	                  |             	 t        |d         |d<   |d   rt        | j                  |d	   |d         }|S t        | j                  |d	   |d   |d
   |d         }|S #  t        t        j                               |d<   Y oxY w)Nr  r  r     r  z'name or numchunks not according to specz.at least one hash is not according to spec: {}r  r  r  )
r  r   rw   r5  r   r   r   r  rt   r  )rP   r  r   r  s       rQ   r  zUp2k._get_wark0  s    r&z?T!SF_z%AGHHF 	A;;$$Q'IPPQRS 		*RZBvJ f:*499bj"V*MD 	 +		2f:r&z2f:r&zD 	*TYY[)BvJs   0C "C0c           	         t        j                  |      }|j                  }t        |      }g }dj	                  |      }t        t        |      d| j                  j                        5 }| j                  rC|dk\  r>| j                  j                  |||| j                  ||      }	|	D 
cg c]  }
|
d   	 }}
d}|dkD  r| j                  rg |fcd d d        S | j                  r%|dz  }|t        |      z   |z   | j                  _        t        j                          }t#        ||      }||z  }|dkD  rg|j%                  t#        |d            }|s%t'        dt        |j)                               z         |j+                  |       |t-        |      z  }|dkD  rg|j/                         d d }|j1                  t3        |      j5                  d	             |dkD  rd d d        ||fS c c}
w # 1 sw Y   ||fS xY w)
Nz MB, {}r  r  r   rn  i   zEOF at !   r0  )r   rM  rz  r  r   rQ  r%   rl   iobufr   r  r   r{   rj  r   r5  rP  r	  r  rh  tellr  r  r8  r  r;   r  )rP   r  r9  r  fszcszr  r  rA  tltr   mbhashobjr  bufr8  s                   rQ   r  zUp2k._hashlist_from_fileI  s    XXd^jjS!!!$'%+tTYY__5 	<xxC:-hhmmAsC&&I%()qt))'99r6	< 	< 77-B"(3r7"2V";DGGK!..*#sms
Ag&&S)!45C'	CM(ABBNN3'3s8OC Ag !)#2.

76?11':;+ '	<: Bw5 *	<: Bws-   *A G/*G*6G/CG/AG/*G//G;c                 
   t        |d   |d         }|d   sT	 t        j                  t        ||d               }t        j                  |j                        r|j
                  |d<   i S 	 | j                  |d      }|j                  d      }t        ||d         }t        |d   |d   |d         }	|rRt        | j                  d | d|||	|d	   |d
   | j                  j                  |	|d
         |d   |d   |d   |d   d      }
|
s#d|	}| j                  |d       t        d|      |
j                  d      rt        | j                  ||	|
d         }|r|j                  |d   |d   f}|\  }}|d<   \  }}|j                  x|d<   }|j                  |d<   |j                  |d<   ||d<   t!        |d   d      |d<   |j                  |d   |d   f}||k7  r+| j                  d|fz  d       | j#                  ||dz         S | j%                  |||d         |d<   || j&                  |d      |d   <   |d   dz   }| j(                  j*                  rd|z   }| j(                  j,                  r"||d<   |d   s| j&                  |d      |d   = i S | j(                  j.                  r|d   j1                  dd      }n(| j2                  j4                  j7                  |d         }t9        |d|d|d   |fz  |      \  }|d<   	 t        ||d         }|d   }|d   }d}t:        rQ|rO| j(                  j<                  r9| j(                  j<                  d z  d z  |k  r	 t?        j@                  d!d"d#|g       t:        s|r|d'kD  r| jB                  j                  |      }|d(k(  rnd)|v rd*}| j                  ||d   fz         d&}d}nzd"|v rd+}| j                  ||d   fz         d&}nYd&}|jE                  d,       |jG                  d-       |jI                          	 t        j                  |      jJ                  }|d.k  }|rdd/}|rd(nd0}| j                  |jM                  || jB                  j                  |      |             | jB                  jO                  ||       ||d<   |d   r'|r%|jE                  |dz
         |jG                  d-       |jQ                          |d   s| jS                  |d   |d          i S #  Y xY w#  | j                  d$|d%       d&}d}Y xY w#  d}Y xY w# |jQ                          w xY w)1Nr   r  r  r  r  r  r  zxbu.up2kr  r9  r:  r;  r6  r  r   r  r  r=  zxbu reloc2:%d...rE  r  r  rf   r  r  r  r  r  r  r  Fr  fsutilsparsesetflagzcould not sparse rg   Trn  rL  nosparsezMvolflag 'nosparse' is preventing creation of sparse files for uploads to [%s]zHvolflag 'sparse' is forcing creation of sparse files for uploads to [%s]i    erJ  z&sparse files {} on {} filesystem at {}r  )*r$   r   rM  r}  ry  rz  r   r  r2   rm   ro   r  r   r,   r  r   r7   r  r  r   rl   r  r9  r  r)  rj   r  r-  r/   r   r  sp
check_callr   r
  r	  flush	st_blocksr   relabelr  r0  )rP   r"  ro   r  r  r  r  r  ap_chkvp_chkr  rO   r   r  r  r  r  r  r  rA  r^   r  r@  r  fsnblknvs                              rQ   r  zUp2k._new_uploadp  sM   S[#f+.6{hhuT3v;78<<,"%++CKI - ZZF$ffUmtS[)s6{CKV=FF""63v;7FFFD	B" AGIAS!n$vvgDHHffbkB99c&k3v;?C784D!S[*3'*yy0CK""%,,CK"%))CK"%CK"-c&k2">CK99c&k3v;?Ccz!3uh!>B#00eai@@mmD#s4y9F25c&k"3v;/6{Z'99:D99<<CKv;MM#f+.s6{;I99f+%%c3/C((//##CK0C!TC 00
3v;5	D#f+.Gv;DVBGII$$II$$t+d2b8!MM8Xy'"JK drK'7ZZ^^D):2%gAHHQ#f+/0"G D^bAHHQ#f+/0"G"GFF?+GGDMGGI%"xx0::#d{ <!Tt"djjnnT&:DAB

""4,"F6{trAvGGI6{FS[9	b!HHg?C"G D.%$ GGIsK   AT AT8 !T :B&T8 !$T/ BT8 TT,)T8 /T53T8 8U
c                     | j                   j                  }|r| j                   j                  ry 	 t        j                  |       | j
                  rd}n&| j                   j                  }| j                          K)NrC   )rl   snap_wrino_snapr   r   r   do_snapshot)rP   slps     rQ   r   zUp2k._snapshot  s]    ii  dii''JJsOwwii((  " rS   c                     | j                   5  | j                  5  | j                  j                         D ]  \  }}| j	                  ||        	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY wrK   )r~   r   r   r   	_snap_reg)rP   r   r  s      rQ   r  zUp2k.do_snapshot  sg    ZZ 	' 	'----/ '3q#&'	' 	' 	' 	' 	' 	's"   A-5A!A-!A*	&A--A6c                    t        j                          }| j                  j                  j                  |      }|sy | j                  j
                  dz  }|j                         D cg c]  }|d   s	||d   z
  |k\  s| }}| j                  j                  rg }nU|j                         D cg c]<  }|d   r5t        j                  j                  t        |d   |d   |d               s|> }}|s|rd}	|	j                  t        |      t        |      |      }	|j                  |       |D cg c]  }| j                  |       }
}| j!                  dj#                  |	g|
z                |D ]  }||d	   = 	 t        |d   |d   |d         }t        j                  j%                  |      d
k(  rt        j&                  |       	 t        |d         t        |d         k(  r+t        |d   |d   |d         }t        j&                  |        | j                  j                  s| j                  j(                  ry t*        j                  j#                  |d      }|sa|| j,                  vs| j,                  |   Cd | j,                  |<   t        j                  j                  |      rt        j&                  |       y t/        |r t1        d |j3                         D              nd
      }t        |      |f}|| j,                  j                  |      k(  ry t        j4                  |      rt7        |       dj                  |t+        j8                               }| j:                  |   |d}t=        j>                  |dd      jA                  d      }|jC                  dd      }|jC                  dd      }tE        jF                  |d      5 }|jI                  |       d d d        tK        | j                   ||tL               | j!                  d|t        |      t        j                          |z
  fz         || j,                  |<   y c c}w c c}w c c}w #  Y xY w#  Y xY w# 1 sw Y   xY w)Nr`  r  r  r   r  r  z/dropping {} abandoned, {} deleted uploads in {}r2  r  r   r  r  r  c              3   >   K   | ]  \  }}d |v r|d   n|d     yw)r8  r;  r  NrW   )r  r  r   s      rQ   r  z!Up2k._snap_reg.<locals>.<genexpr>B  s'     N$!Q6Q;$AfI5Ns   z{}.{})r   r   Tr   )	sort_keysr   r  s   "need": [],
rS   s   "hash": [],
r  zsnap: %s |%d| %.2fs)'r   ro   r  r  rl   	snap_dropr  r9  r   r  r  r$   r   r  rP  r  rm   r  r  rg  r  r  r   floatr   r   r  r)   getpidr   r
  r  r7  r)  r(   r  r	  r!   r   )rP   r   r  rp  r  idropr   r  r  rO   visr"  r  newestetagpath2bodyrV  rA  s                      rQ   r  zUp2k._snap_reg  s   iik88##''-		##b(PA6sQvY%7OaPP99<<D V9ai6AfI(NO D  AAR#d)T2AIIdO6894))!,9C9HHTYYsSy)* F$ Vc&k3v;GDxx''-2

4(3v;'3s6{+;;$S[#f+s6{K

4($ 99<<499,,ww||Hk24>>)T^^D-A-M'+t$88??4(JJt$ N#))+NN

 C&!4>>%%d++<<!HtRYY[1!^^D1sCJJttFMMgVII&,II&,]]5$' 	1GGAJ	 	DHHeT:6&$C$))+:K)LLM#tO Q
 :>	 	sE   (
Q3Q?Q0AQ4Q>AQ AQ(Q0 Q%(Q-0Q9c                 J   | j                   5  | xj                  dz  c_        d d d        | j                  sJ 	 | j                   5  | xj                  dz  c_        d d d        | j                  j	                         \  }}}}}}}d| j
                  |   vr_t        |||      }	 |r| j                  j	                  |      ni }	t        |	      }
| j                  ||	|      }| j                  j                  rQdj                  |t        |j                               |
| j                  j                  |	      }| j                  |       |r6||	d<   ||	d<   |	j!                  | j                  j#                  |||	             | j                   5  | j(                  |   }|s| j                  dd       	 d d d        | j*                  j	                  |      }|s| j                  d	d
       	 d d d        | j-                  |||||	       |j.                  j1                          d d d        | j                  d||
t        |	      |
z
  fz         # 1 sw Y   4xY w# 1 sw Y   xY w# t$        $ r}| j'                  d||       Y d }~Xd }~ww xY w# 1 sw Y   rxY w)Nr   r  zparsers({}): {}
{} {} tags: {}r  r  r6  zno cursor to write tags with??r  zno entags okay.jpgrg   ztagged %r (%d+%d))r~   r   r   r   r  r   r$   r  r  rl   r  r   r   r   r7  rm   r  r=  rh  r>  r   r\   r  r}  r~  )rP   r   r  r  r.  r@  rB  r>  r^   r  ntags1r  rO   rr  r   r\   s                   rQ   r   zUp2k._tagger[  sX   ZZ 	KK1K	 yyy !q ! .2YY]]_*D$BBDJJt,, D"b)G13tyy}}W-T++D$@99$$9@@d7<<>2FDII<M<MtA HHQK$&DM$&DMKK		 1 1'7D IJ
  (hhtnHH=HC	( ( .HH1QH7( ( sFD'4@%%'( HH(GVSY=O+PPQW 		 	
! !.  !!"gr2( (sH   II"&CI/ &J>1J9/JI"I,/	J8JJJ"c                    | j                   5  | xj                  dz  c_        d d d        	 | j                   5  | xj                  dz  c_        d d d        | j                  j                         }t	        |      dk7  rt        d      	 | j                  |      s| j                  ry }# 1 sw Y   xY w# 1 sw Y   hxY w# t
        $ r"}| j                  d|d|d       Y d }~?d }~ww xY w)Nr   rh   zinvalid hash taskzfailed to hash r   )	r   r   r   r  r  rh  _hash_tr{   rm   )rP   taskrr  s      rQ   r   zUp2k._hasher  s     	LLAL	 !! "!" ::>>#D4yA~ 344B||D)dii 	 	" "  BD"=qAABs.   B)B5
C )B25B>	C,
C''C,c                 8   |\	  }}}}}}}}	}
| j                   5  | j                  5  | j                  ||      s	 d d d        d d d        y	 d d d        d d d        t        |||      }| j	                  d|       t        j                  |      }|j                  s7t        | j                  |j                  t        |j                        ||      }n8| j                  |      \  }}|syt        | j                  |j                  |      }| j                   5  | j                  5  | j                  | j                  |   |||j                  |j                  ||||d|	|||
       d d d        d d d        |rIt!        j                          |z
  dkD  r/| j"                  5  | j"                  j%                          d d d        | j&                  r| j)                          y# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   XxY w)NTzhashing Fr6     )r~   r   r  r$   rm   r   rM  rz  r  rt   r   rw  r  r  r4  r   r   r   r   r   r  )rP   r  r   r  r   r  r.  rB  r>  r@  rA  r^   r  r  r  r  s                   rQ   r  zUp2k._hash_t  s    <@8dE2r2r3ZZ 	 	&&tU3	 	 	3	 	 b"%*+hhw{{*		3;;CLL(92rD 009IFA*499ckk6JDZZ 	 	MM

4 	 	$ $))+"R'!! .  ++-. ?? W	 	 	 	$	 	 	 	&. .sX   G+GG+G+H&A G8&HHG(	#G++G58H	=HHHc
           	         d|vry | j                   dkD  red}
| j                  |
| j                   fz  d       t        | j                   dz        D ](  }t        j                  d       | j                   dk  s( n |||||||||	f	}| j
                  5  | j                  j                  |       | xj                   dz  c_         d d d        y # 1 sw Y   y xY w)Nr  r  z%d files in hashq; taking a naprE  r   r   )r   rm   r   r   r   r   r   r  )rP   r   r  r   r  r.  rB  r>  r@  rA  rO   r  r%  s                rQ   	hash_filezUp2k.hash_file  s     <<$1AHHQ$,,(!,4<<4/0 

3<<$&
 D%RRh? 	JJNN2LLAL	 	 	s   1CCc                     | j                   5  | j                  5  | j                  }g | _        d d d        d d d        D ]  \  }}}| j                  |||        y # 1 sw Y   0xY w# 1 sw Y   4xY wrK   )r~   r   r   hook_fx)rP   todoactr  req_vps        rQ   r  zUp2k.do_fx_backlog  sp    ZZ 	! 	!??D DO	! 	!  $ 	*OCVLLb&)	*	! 	! 	! 	!s!   A)AA)A&	"A))A2c                    |D cg c]
  }|dk7  s	| }}|rd}t        |||fz        |j                  d      xs g D ])  }|j                  d      rt        |dd        \  }}d|z   }nt        |      \  }}t	        | j
                  d|||d      }	|	sd}| j                  ||||fz         q|	\  }
}}\  }}t        ||      }|st        d      |d	k(  rp|t        |j                        d  j                  d      }| j                  |j                  |j                  |j                  ||dt        j                         dd
	       |dk(  s| j                  t         d|g dd       , y c c}w )Nr,  z'got unsupported key in %s from hook: %sr(  r   r6  )r,  r.  z-hook_fx(%s): failed to resolve %r based on %rzhook_fx: blank vp from pathmodr   TdelF)rh  r  r  r>   r,   ro   rm   r=   r  r  r*  r  r   r   r   r  r   )rP   r  r  r  r   badrO   r  r.  r   r+  r  r  r  r,  s                  rQ   r  zUp2k.hook_fx  so   *QT	q**9AAc
N++66$<%2 	IC ~~c" QR/RCi +R"fS.CDACc3//0$%!BB	Sr2B @AAe|BHH(..s3KK288RRbRV e|r2r5%H7	I +s
   
E-E-c                    d| _         | j                  rd| j                  _         t        | j                        D ]  }| j	                  |        | j
                  j                  s!| j                  d       | j                          t        j                         }| j                  r=t        j                  d       t        j                         |z
  dk\  rn| j                  r=t        | j                        D ]  }| j	                  |        | j                  j                         D ]?  }|j                  }	 |j                          |j!                          |j!                          A | j"                  r@| j"                  j                  }| j"                  j!                          |j!                          i | _        y #  Y zxY w)NTzwriting snapshotr   r   )r{   r   r   r   r  rl   r  rm   r  r   r   r   r   r  r}  	interruptr  r   r   )rP   r   r;  r   r  s        rQ   shutdownzUp2k.shutdown  sT   	88 DHHM dkk" 	AMM!	 yy  HH'(YY[ggJJsOyy{R1$ gg dkk" 	AMM!	 88??$ 	CB IIKHHJ	 <<((BLL HHJs   9F??G)r   r  )TFr   NF)F)r6  )qrT   rU   rV   rR   r   r   r   r   r   r   rm   r   r   r   r  r  r0  rI  rM  rP  rS  rO  r[  r   r   rf  re  rg  ri  r  r  r  r  r  r   r  r  r  rS  rT  r  r  r   rW  r  r  r  r  r  r  r  r  r  r  r8  r>  r  r  rG  r  rJ  rN  r   rK  rc  r[  rY  rZ  rs  r\  r^  r_  r`  ra  rb  r  r  r  r  r  r$  r'  r+  r&  r1  r0  r  r4  r  r  rW  r  r  ru  r  r  r  r  r  r  r  r  r  r   r  r  r   r   r  r  r  r  r
  rW   rS   rQ   rb   rb      s   U!n9*"7@6&

A8&9P6"1f94+ Bb$H#J/b3jB#@J0
 67gRPd$:l7\EN
VpI'V,$?!BOb<	A2N`,\&PDJ@$L'.8,+2Z".

%N	  " . *$  *>KZ2r b)HTYl!E@!K$Z(> 2h	$6 !zxV<q+f7rZxSjSj
D/69vz&x4  "%NGR#'
N$`0RdB&0x :*!IF'rS   rb   c                     d}d}	 dD ]:  }t        j                  | dz  |z        }|dk  s
|dk\  r	|dk  r|c S ||z  }||z  }< @)Nrn  r  r5  g      ?   r  rr  )mathceil)filesizer  stepsizemulr  s        rQ   r  r  C  sh    IH
 	Cii3 :;G#~)/?"?GtO  !IOH	 rS   c                     | t        |      g|z   }dj                  |      }t        j                  |j	                  d            j                         dd }t        |      j                  d      S )zDserver-reproducible file identifier, independent of name or locationr2  r  Nr  r0  )rj  r  r5  rP  r7  r8  r;   r  )rt   r  r  r  vstrr  s         rQ   r  r  P  s_    CM"V+F99VD>>$++g./668"=D4=((rS   c                     t        d| ||||fz        }t        t        j                  |      j	                               }d|j                  d      d d S )Nz%s
%d
%d
%s
%srm  r0  r  )r8   r;   r5  rP  r8  r  )rt   r@  lastmodr  r.  r  s         rQ   r  r  Y  sR    
%wB(CC
DC
'..%,,.
/CZZ(*CR00rS   )l
__future__r   r   r  r5  r
  r  r  ru   rk  rM  
subprocessr  r  r|   r   r  copyr   queuer   rR   r   r	   r
   r   r   authsrvr   r   r   r   r   rD  r   r   r   r  r   r   r   r   utilr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   pathlibrB   r   r]  svchubrD   zsgr   r   r  r  r4  r  r  stat_resultr  objectrI   rY   rb   r  r  r  rW   rS   rQ   <module>r!     sQ   7     	 	          < < 9 9  * *  + + + + + + + + + + +Z	 	
csyy~usyy~#))C.)  	i ^ 2>>:;& !6 !&~R6 ~RBf
)1_h	s   E< <F 