a

�9pe�p�@s8
dd
lm
Z

ddlmZ
ddlmZ
ddlmZ
ddlmZ
ddlmZmZm	Z	
ddl
mZmZm
Z
mZ
dd	lmZ
dd
lmZ
ddlmZmZmZ
ddlZddlZddlZddlZddlZddlZdd
lmZ
ejdkZgd�
Z dd�Z!dd�Z"dd�Z#dd�Z$dd�Z%dd�Z&d#dd�
Z'ddifdd �
Z(d!d"�Z)dS)$�)
�absolute_import�
)
�S3)
�Config)
�S3Uri)
�FileDict)�dateS3toUnix�dateRFC822toUnix�s3path)�	unicodise�deunicodise�
deunicodise_s�replace_nonprintables)
�ParameterError)
�	HashCache)�debug�info�warningN)
�S_ISDIR)�r
)�fetch_local_list�fetch_remote_list�compare_filelistsc
cs�zt�
t|�
�
}
Wnty&


Yd
S0gg}}|
D]F}t|�
}tj�ttj�||��
�
rrt||�s||�	|�

q6|�	|�

q6|||fV
|D]6}tj�||�}tj�
t|�
�
s�t|�
D]
}|V
q�q�d
S)z_
    Reimplementation of python's os.walk to nicely support unicode in input as in output.
    N)�os�listdirr�	Exceptionr�path�isdir�join�handle_exclude_include_walk_dir�append�islink�_os_walk_unicode)�top�names�dirs�nondirs�name�new_path�
x�r*�0/usr/lib/python3.9/site-packages/S3/FileLists.pyr"'s"














r"cs�t�}d
}t
j�||
d�}td|�

d}|jD]@�t�f
dd�|D�
�
sLq0��|�
r0d}td|j��


qrq0|r�|j	D]R�t�f
d	d�|D�
�
s�q|td
|�j
f�

��|�
r|d}td|j��


q�q||r�td|�

ntd
|�

|S)a


    Should this root/dirname directory be excluded? (otherwise included by default)
    Exclude dir matches in the current directory
    This prevents us from recursing down trees we know we want to ignore
    return True for excluding, and False for including
    )z/)$z/)\Zz\/$z	\/\Z(?ms)��CHECK: '%s'Fc
3s|]}
�j�
|
�
V
qdS�
N��pattern�endswith��.0Zdp�
�
rr*r+�	<genexpr>O�z2handle_exclude_include_walk_dir.<locals>.<genexpr>T�EXCL-MATCH: '%s'c
3s|]}
�j�
|
�
V
qdSr.r/r2r4r*r+r6Yr7zINCL-TEST: '%s' ~ %s�INCL-MATCH: '%s'�
EXCLUDE: '%s'�
PASS: '%s')rrrrr�exclude�any�search�
debug_exclude�includer0�
debug_include)�root�dirname�cfgZdirectory_patterns�
d�excludedr*r4r+r@s2




















rc

cs�tj
�t|�
�
sJ�
t|g
�
}
t|�
D]�\}}}ttj
�t|�
�
�
}|D]h}tj
�||�}ttj
�t|�
�
�
}tj
�	t|�
�
rH||ks�|�
|tj
j�
r�td
|�

qH|
�
|�

qHq&|
D]$}	t|	�
D]\}}}|||fV
q�q�dS)z�
    Walk filesystem, following symbolic links (but without recursion), on python2.4 and later

    If a symlink directory loop is detected, emit a warning and skip.
    E.g.: dir1/dir2/sym-dir -> ../dir2
    z+Skipping recursively symlinked directory %sN)rrrr�setr"r�realpathrr!�
startswith�sepr�add)
rZwalkdirs�dirpath�dirnames�	filenamesZreal_dirpathrC�currentZreal_currentZwalkdirr*r*r+�_fswalk_follow_symlinkshs 









�


rPc
cs$t|�
D]\}
}}|
||fV
qd
S)z\
    Directory tree generator

    path (str) is the root of the directory tree to walk
    N)
r")rrLrMrNr*r*r+�_fswalk_no_symlinks�s
rQc
Cs�td
�

t
�}
tdd�
}|��D]�}td|�

d}|
jD](}|�|�
r:d}td|
j|�


qdq:|r�|
jD](}|�|�
rnd}td|
j|�


q�qn|r�td|�

||||<||=q q td	|�

q ||fS)
NzApplying --exclude/--includeF�
Zignore_caser-Tr8r9r:r;)	rrr�keysr<r>r?r@rA)�src_listrD�exclude_list�filerFr5r*r*r+�filter_exclude_include�s0

























rWc
Csb
d
d�}i}|jD�
]}z�d}|dkr>t
jtj��ddd�}nhzt
jt|�
dd�}WnPty�
}
z8td||j	f�

WYd}~W|r|�
�
qWYd}~n
d}~00|D]Z}t|�
��}t
j�t
j�|
|��
}tt
j�t|�
�
�
}tt
j�t|�
�
�
}	||||	�
q�W|�
r(|�
�
q|�
r&|�
�
0qg}
t|�
D]&}||}|��
|
�|g|f�

�
q6|
S)	NcSs&|
|v
r|g
||
<n||
�|�

dSr.)
r )rE�key�valuer*r*r+�_append�s

z(_get_filelist_from_file.<locals>._append�
-r5F)�mode�closefd)
r\zJ--files-from input file %s could not be opened for reading (%s), skipping.)�
files_from�io�open�sys�stdin�filenor�IOErrorr�strerror�closer�striprr�normpathrrC�basename�sorted�sortr )
rD�
local_pathrZ�filelist�fname�
f�
e�linerCri�resultrX�valuesr*r*r+�_get_filelist_from_file�s>









	
�






�





rtFcsR
��
fd
d�}�
�fdd�}�
f
dd�}t��
t
���
jrptj�t�
j�
�
rptj�t�
j�
�
dkrp���
j�

nt	d�

g}t
d	d
�
}d	}	t|�
tt
tfv
r�|g
}|dkr��
j}|D]D}
t|
�
}|jdks�td|
�
�
|��r�|s�td
|
�
�
|�|�

q�|D]}|||�|�\}}	q�t|�
dk�
r(d	}	t|�
\}}
||�
}|�|�
||	|
|fS)Nc
s�
t|�
}
d
}t
d|
�

d
}|D�
]t}|d7}|dd
krJt
d||
f�

|dkrTq ||}|d}|d}zt�t�t|�
�
�
}Wn@ty�
}	
z(|	jtjkr�WYd}	~	q n�WYd}	~	n
d}	~	00|r�d
}
n|j	}
||�
|
|j|j|j
|j|j|d	��

||j	7}|�
rq d
�
jvr ��|j|j
|j|j	�}|du�
r~z|�|�
}Wnt�
yb


Yq Yn0��|j|j
|j|j	|�
|�||j|j
||j	�
q |S)Nr
zYRunning stat() and reading/calculating MD5 values on %d files, this may take some time...ri�z[%d/%d]r[�	full_name�is_dir)�size�mtime�dev�inode�uid�gid�sr�md5)�lenrr�stat_result�statr�OSError�errno�ENOENT�st_size�update�st_mtime�st_dev�st_ino�st_uid�st_gid�sync_checksr~�get_md5rdrKZrecord_hardlink)�loc_listZlen_loc_list�
total_sizeZcounter�
relative_fileZ
loc_list_itemrurvr}rprwr~)�cacherDr*r+�_fetch_local_list_info�sX























�












z0fetch_local_list.<locals>._fetch_local_list_infoc	sDtd
�

|
�
�dkrfzt��}t��}WntyB


d}d}Yn0ddd||dddd�|d<|dfS|
��r�|
�
�}|
��}�
r�t�j	�
r�t
�|�}d}	q܈jr�t|�
}nt
|�
}d}	n"d}|
��}|g|
�
�g
fg
}d}	|D�
]X\}
}}|
�||d	�}
|�
s|dff
}n|df|dff}|D�
]\}}|D�
]}tj�|
|�}|�
s|tj�t|�
�
�
s|tj�t|�
�
�
r.td
|�

�
q.tj�t|�
�
�
r��j�
s�td|�

�
q.tj�|
|�}tjjdk�
r�d�|�tjj�
�
}�jd
k�
r�t|�
}|�d�
�r|dd�}|�r$|�r$|ddk�r$|d7}||d�||<�
q.�
q q�||	fS)Nz Compiling list of local files...r[r
���F)rurwrxr{r|ryrzrvTr,rz"Skipping over non regular file: %szSkipping over symbolic link: %s�
/Znormalz./�)rurv)rrir�geteuid�getegidrrrrr^rt�follow_symlinksrPrQrC�replacer�isfiler�existsrr!rJ�splitZurlencoding_moderrI)r�Z	local_urir��	with_dirsr{r|Z
local_baserlrm�single_filerBr%�filesZrel_rootZ
iter_elements�elementsrvrorur�)rD�is_srcr*r+�_get_filelist_local

sz














�





































�z-fetch_local_list.<locals>._get_filelist_localcs��jr~t
�j�
d
kr~|��
tr*|
��}n|
��}|D]2}|�dd
�}|sLq6|�|d||d|d�
q6|�	�
|�
�j�

dS)Nr
rzryrxrw)�
cache_filerr^Zmark_all_for_purge�PY3rsZ
itervalues�getZunmark_for_purge�purgeZsave)r��
local_listZlocal_list_val_iterZf_inforz�
rDr*r+�_maintain_cacheU
s









�
z)fetch_local_list.<locals>._maintain_cacher
z6Cache file not found or empty, creating/populating it.FrRrVz.Expecting filename or directory instead of: %sz)Use --recursive to upload a directory: %sr)rrr�rrr�r
�getsize�loadrr�type�list�tuplerG�	recursiverrrr rrW)�argsr�r�r�r�r�r�Z
local_urisr�r��arg�uriZlist_for_urirUr�r*)r�rDr�r+r�s<;K
.


















rcs�
f
d
d��d��fdd�	}t��
g}t
dd�
}t|�
tttfv
rH|g
}|dkrV�
j}|D],}t|�
}|jdks|td	|�
�
|�	|�

qZd
}	|r�|D]D}||dd�\}
}|	|7}	|
D]"}|
|||<|�
||
�|�
�
q�q��
n |D�
]}|��}
t
jd|
d
d�}t|�
dk�
rl|\}}d|v}|t|�
|d�\}
}|	|7}	|
D]*}tj�|
|d|
��
r>|
|||<�
q>q�t�|���
}|�
s�td|���
�
|�
o�|ddk}||��|��|d�}|
�
rȈ||�
|||<|�d�
}|�
r�|�
||�
|	|�dd
�7}	q�t|�
\}}|||	fS)Ncs�t��
�
|�
}|�d
�
sdS|
�t|d
d�
|d
d�d�
t|d
d�
d��

z*|dd}|
�d|i
�

td	|�

Wnty�


Yn0dS)
NZheaderszcontent-lengthZetag�"'z
last-modified)rwr~�	timestampzs3cmd-attrsr~zretrieved md5=%s from headers)	rZobject_infor�r��intrgr	r�KeyError)r��remote_item�responser~r�r*r+�_get_remote_attribs�
s






�




z.fetch_remote_list.<locals>._get_remote_attribsTcs�
td
|�

d}t
t��
}|j|��|��|
�
d�}|��}}|}|dkr�|ddkr�|d|�d�
d�}td|��|f�
}t|�
}t	d	d
�
}	d	}
|dD]�}|d}t
|d
�
}
|ddk}||kr�|s�t�|�
}|�
�}d}
t	d	d
�
}	n||d�}|�
�|}|�
std�

q�|
t|d�
|d�d�
|||dd|d�	|	|<d|	|dv�
rl�t|�
|	|�
|	|d}|	�||�
||
7}|
r�
�
q�q�|	|fS)Nz*Retrieving list of remote files for %s ...r
)�prefixr��
uri_paramsr,r�r�rz
s3://%s/%sFrRr�ZKeyZSizeTz-Found empty root object name on S3, ignoring.ZLastModifiedZETagr�)	rwr�r~�
object_key�object_uri_str�base_uriryrzrvr[r~)rrrZbucket_listZbucket�object�rfindrrrr�r
rir�rrrg�
record_md5)Z
remote_urir�r��s3r�Zrem_base_originalZrem_baseZremote_uri_originalZrem_base_lenZrem_listZ	break_nowr�r�Zobject_sizervrXr�r~)r�r�r*r+�_get_filelist_remote�
s\


�

























�







z/fetch_remote_list.<locals>._get_filelist_remoteFrRr�z Expecting S3 URI instead of '%s'r
)
r�z\*|\?r)
�maxsplitr�r�r�z3Expecting S3 URI with a filename or --recursive: %sr�)r�r�r�rvr~rw)
T)rrr�r�r�rGr�rrr r�r�r��rer�r�glob�fnmatchr
rir�r�rW)r�Zrequire_attribsr�r�r�Zremote_urisZremote_listr�r�r�Z
objectlistZtmp_total_sizerXZuri_strZwildcard_split_resultr��restZneed_recursionrvr�r~rUr*)r�rDr�r+r�
sf
J



























�







rc
sd
d�}��
fdd�}|dkr*|dkr*J�
td�

t
��tdd�
}i}td||�
||�
f�

t�}|��D�]R}	td	|	�

|r�|	�d
�
}
|
dkr�|	d|
d�}|r�||v
r�|�|�

z|d|�d
dd
�d�}Wq�t	y�


Yq�Yq�0q�|	�
v�r@�j
�
rtd|	�

||	=�
|	=qlz||�
|||	�}Wn@ttf�
yr


td|	�

t
d|	�

||	=�
|	=YqlYn0|�
r�td|	�

||	=�
|	=n�z|�|	�
}
Wnt�
y�


d}
Yn0|
du
�r|
�
jv�r�
�|
�
}td||	f�

||	}|
|d<||d<|||	<||	=�
|	=n$�
�|	|
�
||	||	<||	=�
|	=qlz|�|	�
}
Wnt�yf


d}
Yn0�
�|
�
}|du
�r�td||	f�

||	}|
|d<||d<|||	<||	=ql�
�|	|
�
ql�
��D]*}||v�s�||v�s�||v�rʈ
|=�q�|�
||fS)Nc


Ss|rd
p
dS)NZremote�localr*)
Z	is_remoter*r*r+�__direction_str5s
z*compare_filelists.<locals>.__direction_strc
	s`
d
}|�|�
}�
�|�
}|r |s>t
d|t|�
t|�
f�

dSd�jvr�|�d�
}|�d�
}	|	du
r�|du
r�|	|kr�td|||	f�

d}d�jv}
|
r�|d
kr�d|dvs�|d
kr�d|dvr�d}
t
d	|�

|
r�|d
d
kr�d}
|�
r\|
�
r\z|�|�
}�
�|�
}Wn2ttf�
y:


td|�

td|�

�Yn0||k�
r\d}td
|||f�

|S)z@Return True if src_list[file] matches dst_list[file], else FalseTzE%s: does not exist in one side or the other: src_list=%s, dst_list=%sFrwNz'xfer: %s (size mismatch: src=%s dst=%s)r~r[zdisabled md5 check for %srvzIGNR: %s (disappeared)�%s: file disappeared, ignoring.z&XFER: %s (md5 mismatch: src=%s dst=%s))	r�r�boolr�rr�rdr�r)
rTZdst_lst�
src_remote�
dst_remoterVZ
attribs_matchZsrc_fileZdst_fileZsrc_sizeZdst_sizeZcompare_md5Zsrc_md5Zdst_md5�rD�dst_listr*r+�_compare8sB






�









(










z#compare_filelists.<locals>._compareFzVerifying attributes...rRz)Comparing filelists (direction: %s -> %s)r-r�r
rr�z!IGNR: '%s' (used --skip-existing)zIGNR: '%s' (disappeared)r�z IGNR: '%s' (transfer not needed)zDST COPY src: '%s' -> '%s'r~Zcopy_srczDST COPY dst: '%s' -> '%s')rrrrrGrSr�rK�rindex�
ValueErrorZ
skip_existingrdr�rr�Zby_md5Zfind_md5_oner�)rTr�r�r�r�r�Zupdate_listZ
copy_pairsZ
src_dir_cacher�Zdir_idxrZ	same_filer~Z
copy_src_fileZsrc_itemror*r�r+r4s�
0


























































r)FNF)*Z
__future__rrrrrZ	BaseUtilsrr	r
ZUtilsrrr
rZ
ExceptionsrrZloggingrrrrrar�r�r�r_r�r�version_infor��__all__r"rrPrQrWrtrrrr*r*r*r+�<module>
s:












(	&
G"