o zqfĘ@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl ZddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ddl%m&Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,dd l-m.Z.d d l/m0Z0m1Z1e2d Z3e2d ej4Z5e2dZ6e2dej4j7Z8d9Z:gdZ;dZdj>ej?edZ@ddZAddZBddZCdDddZDdDddZEdDd d!ZFdedfd"d#ZGd$d%ZHe2d&ej4ZI eHd'd(ZJGd)d*d*ZKGd+d,d,eKZLGd-d.d.eZMe2d/jNZOd0d1ZPd2d3ZQdEd4d5ZRd6d7ZSGd8d9d9ZTGd:d;d;e jUZVejWjXfdd?ZZeRe<eYZYd@dAZ[dBdCZ\dS)Fz$PyPI and direct package downloading.Nwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST parse_version)log)DistutilsError) translate)Wheelunique_everseen)_read_utf8_with_fallback_cfg_read_utf8_with_fallbackz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezsz(interpret_distro_name..Nr) py_versionrErW)r;anyrangerSisdigitrjoin)rMrTrJrarErWr=r^r&r&r'rs$  rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr&r'wrapperszunique_values..wrapperr)rjrkr&rir' unique_valuessrlz3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c cst|D]2}|\}}tttj|d}d|vs#d|vr8t |D]}t j |t |dVq(qdD] }||}|dkr[t ||}|r[t j |t |dVq;dS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,homepager5r)z Home PagezDownload URLr4N)RELfinditergroupssetmapstrstripr,r;HREFr8r"urljoin htmldecoderIfindsearch)r<pagerHtagrelrelsposr&r&r'find_external_linkss    rc@(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCdS)z3 Feed a block of data to the hash. Nr&selfblockr&r&r'feedzContentChecker.feedcCr)zC Check the hash. Return False if validation fails. Tr&rr&r&r'is_validrzContentChecker.is_validcCr)zu Call reporter with information about the checker (hash name) substituted into the template. Nr&)rreportertemplater&r&r'reportszContentChecker.reportN)__name__ __module__ __qualname____doc__rrrr&r&r&r'rs  rc@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSrf) hash_namehashlibnewhashexpected)rrrr&r&r'__init__s  zHashChecker.__init__cCsBtj|d}|s tS|j|}|stS|di|S)z5Construct a (possibly null) ContentChecker from a URLr4Nr&)r8r"r9rpatternrz groupdict)clsr<rCrHr&r&r'from_urls zHashChecker.from_urlcCs|j|dSrf)rupdaterr&r&r'r zHashChecker.feedcCs|j|jkSrf)r hexdigestrrr&r&r'r#rzHashChecker.is_validcCs||j}||Srf)r)rrrmsgr&r&r'r&s zHashChecker.reportN) rrrr\compilerr classmethodrrrrr&r&r&r'r s   rcsfeZdZdZ    dNfdd Zfdd ZdOd d ZdOd dZdOddZddZ ddZ ddZ ddZ ddZ dPddZddZdPfdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*Z dQd+d,ZdRd-d.Zd/d0Zd1Zd2d3Zd4d5ZdPd6d7Zd8d9Zed:d;Zdd?Z!d@dAZ"dBdCZ#dDdEZ$edFdGZ%dHdIZ&dJdKZ'dLdMZ(Z)S)Srz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcsltj|i||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr3|)superrr- index_url scanned_urls fetched_urls package_pagesr\rrersrrHallowsto_scanr8requesturlopenopener)rrhosts ca_bundle verify_sslrgkw __class__r&r'r.s zPackageIndex.__init__cs0zt|jWn tyYdSwt|Srf)rrO Exceptionradd)rdistrr&r'r@s   zPackageIndex.addFc Cs||jvr |s dSd|j|<t|s||dStt|}|r.||s(dS|d||s7|r7||jvrAtt|j |dS||sMd|j|<dS| d|d|j|<d}| |||}|durhdSt |t jjr||jdkr|| d|jd|j|j<d|jd d vr|dS|j}|}t |tst |t jjrd }n|jd pd }||d }|t|D]} t j|t| d} |!| q|"|j#rt$|dddkr|%||}dSdSdS)z.)filterrYr@rr itertoolsstarmap scan_egg_link)r search_pathdirs egg_linksr&r&r'scan_egg_linkss zPackageIndex.scan_egg_linkscCsttj||}ttdttj| }t |dkrdS|\}}t tj||D]}tjj|g|R|_ t |_||q+dS)Nr`)rrYr@rerrrsrtru splitlinesrSr rMrrEr)rr@rcontentlinesegg_path setup_pathrr&r&r'rs  zPackageIndex.scan_egg_linkcCsd}||js |Stttjj|t|jdd}t|dks)d|dvr+|St |d}t |d}d|j | i|<t|t|fS)N)NNr3r`r7rrT)r.rrrsr8r"r:rSr;r r r setdefaultr,r )rrNO_MATCH_SENTINELr=pkgverr&r&r'_scans &  zPackageIndex._scanc Cst|D]}z|tj|t|dWqty!Yqw||\}}|s-dSt ||D]$}t |\}}| drQ|sQ|rL|d||f7}n| || |q2tdd|S)z#Process the contents of a PyPI pagerr.pyz #egg=%s-%scSsd|dddS)Nz%srr`)rI)mr&r&r'sz,PackageIndex.process_index..)rvrprr8r"rwrxrIr#rrDr-need_version_infoscan_urlPYPI_MD5sub) rr<r{rHrrnew_urlr0fragr&r&r'rs&"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allrr<r&r&r'rszPackageIndex.need_version_infocGs<|j|jvr|r|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrrrrgr&r&r'rs  zPackageIndex.scan_allcCsz||j|jd|j|js||j|jd|j|js)||t|j|jdD]}||q3dS)Nr3r&) rr unsafe_namerrkeyrNnot_found_in_indexr)r requirementr<r&r&r' find_packagess  zPackageIndex.find_packagescsN|||||jD]}||vr|S|d||qt||S)Nz%s does not match %s)prescanr rrrobtain)rr installerrrr&r'r s zPackageIndex.obtaincCsL||jd||s$|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrrrrYunlinkrrr/r@rT)rcheckerrZtfpr&r&r' check_hash s zPackageIndex.check_hashcCsN|D]"}|jdust|r|dstt|r||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr.rrrappend)rurlsr<r&r&r'add_find_linkss   zPackageIndex.add_find_linkscCs"|jr tt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrsrrr&r&r'r (s zPackageIndex.prescancCs<||jr |jd}}n|jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)rrmethrr&r&r'r.s   zPackageIndex.not_found_in_indexcCsvt|ts1t|}|r%|||}t|\}}|dr#||||}|Stj |r-|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. rrMN) rrr _download_urlrDr- gen_setuprYr@rr(rfetch_distribution)rr$tmpdirr>foundr0rCr&r&r'r59s     zPackageIndex.downloadc sd|id}d fdd }|r$|||}|s/|dur/|||}|dur@jdur<||}|durO|sO|||}|dur`drZdp[d|dSd||j|jd S) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur}||jD];}|jtkr#s#|vr"d|d|<q ||vo.|jtkp. }|rF|j}||_tj |jrF|Sq dS)Nz&Skipping development or system egg: %sr) rrErrrr5rMdownload_locationrYr@r)reqenvrtestloc develop_okrskippedsourcerr&r'ryys&z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rMrf)rr r rrcloner) rrr force_scanr#r! local_indexrryr&r r'r[s4         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)rrM)rrrr%r#rr&r&r'fetchszPackageIndex.fetchc Cst|}|rddt||ddDpg}t|dkr|tj|}tj||krGtj ||}tj |r?tj ||sGt |||}ttj |dddd}|d|d j|d jtj|d fWd|S1suwY|S|rtd ||ftd ) NcSsg|]}|jr|qSr&)rO)r]dr&r&r' s z*PackageIndex.gen_setup..rzsetup.pywzutf-8)encodingzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) rzCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rGrHrrIrSrYr@rTdirnamerersamefileshutilcopy2openwriterNrOsplitextr) rrZrCrrHrrTdstrr&r&r'rsJ      zPackageIndex.gen_setupi c Cs2|d|d}zt|}||}t|tjjr%td||j |j f|}d}|j }d}d|vrI| d} t tt| }||||||t|d1} ||} | ro|| | | |d 7}||||||nnqP|||| Wdn1swY|W|r|SS|r|ww) NzDownloading %szCan't download %s: %s %srr4zcontent-lengthzContent-LengthwbTr)rrrrrr8rrrrr dl_blocksizeget_allmaxrsint reporthookr0rrr1rr) rr<rZfprrblocknumbssizesizesrrr&r&r' _download_tosF            zPackageIndex._download_tocCsdSrfr&)rr<rZr;blksizer=r&r&r'r9zPackageIndex.reporthookc Cs|dr t|Szt||jWSttjjfyD}z$ddd|j D}|r0| ||n t d||f|WYd}~dSd}~wt j jyY}z|WYd}~Sd}~wt j jy}z|rl| ||jn t d||jf|WYd}~dSd}~wtjjy}z|r| ||jn t d||jf|WYd}~dSd}~wtjjtfy}z|r| ||n t d||f|WYd}~dSd}~ww)Nr cSsg|]}t|qSr&)rt)r]argr&r&r'r)sz)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r. local_openopen_with_authrr#httpclient InvalidURLrergrrr8rrURLErrorreason BadStatusLineline HTTPExceptionOSError)rr<warningvrr&r&r'rsV  zPackageIndex.open_urlcCsvt|\}}|rd|vr|dddd}d|vs nd}|dr(|dd}tj||}|||p:|||S)Nz...\___downloaded__rKr+)rDreplacer-rYr@re _download_vcs_download_other)rr<rr/rCrZr&r&r'r4s   zPackageIndex._download_urlcCsLtj|j}|d\}}}tddgdgt|}tt|h|@dS)a  >>> rvcs = PackageIndex._resolve_vcs >>> rvcs('git+http://foo/bar') 'git' >>> rvcs('hg+https://foo/bar') 'hg' >>> rvcs('git:myhost') 'git' >>> rvcs('hg:myhost') >>> rvcs('http://foo/bar') +svngithgN) r8r"urlsplitr> partitionrrboolnextiter)r<r>preseppostallowedr&r&r' _resolve_vcsEs zPackageIndex._resolve_vcsc Cs||}|s dS|dkrtd||d\}}}||\}}|d|d|d|t|dd||gt|d |d d|g|d |d d d |dgd}|durb|d|t|||S)NrYz/Invalid config, SVN download is not supported: r7zDoing z clone from z to r$z--quietz-Ccheckoutz--cwdupz-rz-q)rZr[z Checking out )rerr]_vcs_split_rev_from_urlr subprocess check_calldict)rr< spec_filenamevcsrZrSrev co_commandsr&r&r'rVXs& zPackageIndex._download_vcscCsDtj|j}|dkrtjtj|jS||d| ||S)NrT) r8r"r\r>r url2pathnamer9r@r_attempt_download)rr<rZr>r&r&r'rWqs   zPackageIndex._download_othercCs||ddS)NT)rrr&r&r'ryrzPackageIndex.scan_urlcCs2|||}d|ddvr||||S|S)Nrrr)r?rr,_invalid_download_html)rr<rZrr&r&r'rq|s zPackageIndex._attempt_downloadcCst|td|)NzUnexpected HTML page found at )rYr r)rr<rrZr&r&r'rrs z#PackageIndex._invalid_download_htmlc Csttj|}|jddd}|jd\}}}|d\}}}|r&||fn|df\}} |j||dd } | | fS) a Given a possible VCS URL, return a clean URL and resolved revision if any. >>> vsrfu = PackageIndex._vcs_split_rev_from_url >>> vsrfu('git+https://github.com/pypa/setuptools@v69.0.0#egg-info=setuptools') ('https://github.com/pypa/setuptools', 'v69.0.0') >>> vsrfu('git+https://github.com/pypa/setuptools#egg-info=setuptools') ('https://github.com/pypa/setuptools', None) >>> vsrfu('http://foo/bar') ('http://foo/bar', None) rXrr4r7@Nr)r>r@rC) r8r"r\r>r;r@r] rpartition_replacegeturl) r<r= clean_schemeno_fragment_pathrSrarbrc clean_pathrnresolvedr&r&r'rhs z$PackageIndex._vcs_split_rev_from_urlcGtj|g|RdSrf)rrrr&r&r'rzPackageIndex.debugcGr{rf)rrrr&r&r'rr|zPackageIndex.infocGr{rf)rrrr&r&r'rr|zPackageIndex.warn)rrNT)Frf)FFFN)FF)*rrrrrrrrrrrrrrrr r rrr rr5rr'rr5r?r9rr staticmethodrerVrWrrqrrrhrrr __classcell__r&r&rr'r+s`  5     & R .$ "  rz!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?cCs|d}t|S)Nr)rIrunescape)rHwhatr&r&r' decode_entitys  rcCs tt|S)a Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subr)textr&r&r'rxs rxcsfdd}|S)Ncsfdd}|S)Nc s:t}tz |i|Wt|St|wrf)socketgetdefaulttimeoutsetdefaulttimeout)rgrh old_timeout)rjtimeoutr&r'_socket_timeouts  z@socket_timeout.._socket_timeout.._socket_timeoutr&)rjrrrir'rsz'socket_timeout.._socket_timeoutr&)rrr&rr'socket_timeouts  rcCs2tj|}|}t|}|}|ddS)a9 Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r)r8r"r:encodebase64 b64encoderrU)authauth_s auth_bytes encoded_bytesencodedr&r&r' _encode_auths   rc@r) Credentialz: A username/password pair. Use like a namedtuple. cCs||_||_dSrfusernamepassword)rrrr&r&r'rs zCredential.__init__ccs|jV|jVdSrfrrr&r&r'__iter__s zCredential.__iter__cCs dt|S)Nz%(username)s:%(password)s)varsrr&r&r'__str__s zCredential.__str__N)rrrrrrrr&r&r&r'rs  rcs8eZdZfddZeddZddZddZZS) PyPIConfigcsPtgdd}t|tjtjdd}tj|r&t ||dSdS)z% Load from ~/.pypirc )rr repositoryr~z.pypircN) rkfromkeysrrrYr@re expanduserrr)rdefaultsrcrr&r'rs   zPyPIConfig.__init__cs&fddD}ttj|S)Ncs g|] }|dr|qS)r)rru)r]sectionrr&r'r) s z2PyPIConfig.creds_by_repository..)sectionsrkrs_get_repo_cred)rsections_with_repositoriesr&rr'creds_by_repositorys zPyPIConfig.creds_by_repositorycCs6||d}|t||d||dfS)Nrrr)rrur)rrrepor&r&r'rs zPyPIConfig._get_repo_credcCs*|jD] \}}||r|SqdS)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)ritemsr.)rr<rcredr&r&r'find_credentials  zPyPIConfig.find_credential) rrrrpropertyrrrr~r&r&rr'rs    rcCs:tj|}|\}}}}}}|drtjd|dvr$t|\} } nd} | sBt |} | rBt | } | j |f} t j dg| R| redt| } || ||||f} tj| }tj|}|d| ntj|}|dt||}| rtj|j\}}}}}}||kr|| kr||||||f} tj| |_|S) z4Open a urllib2 request, handling HTTP authentication:znonnumeric port: '')rFhttpsNz*Authenticating as %s for %s (from .pypirc)zBasic Authorizationz User-Agent)r8r"r9r-rFrGrH _splituserrrrtrrrr urlunparserRequest add_header user_agentr<)r<rparsedr>netlocr@paramsrBrraddressrrr=rrr:s2h2path2param2query2frag2r&r&r'rE$s8          rEcCs$|d\}}}|r||fSd|fS)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.rsN)rt)hostuserdelimr&r&r'rSsrcCs|Srfr&)r<r&r&r' fix_sf_url^rArcCstj|\}}}}}}tj|}tj|rtj|S| drhtj |rhg}t |D]&} tj || } | dkrCt | } n tj | rM| d7} |dj| dq0d} | j|d |d} d\} }nd \} }} d d i}t| }tj|| |||S) z7Read a local path, with special support for directoriesr3z index.htmlz{name})r/zB{url}{files}r)r<files)OK)rzPath not foundz Not foundrz text/html)r8r"r9rrprYr@isfilerr-rrrerrformatioStringIOrr)r<r>r?r@paramrBrrZrrfilepathbodyrstatusmessager body_streamr&r&r'rDbs,       rDrf)r)]rsysrirYr\rr.rrrr configparserr http.clientrF urllib.parser8urllib.request urllib.error functoolsrr! pkg_resourcesrrrrrr r r r r rrrr distutilsrdistutils.errorsrfnmatchrsetuptools.wheelr setuptools.extern.more_itertoolsr unicode_utilsrrrrGIrvrrHrr;rR__all___SOCKET_TIMEOUT_tmplr version_inforr(rrDrrFr[rrlrorrrrrrrrxrrrRawConfigParserrrrrErrrDr&r&r&r's @         # "  !  (/