вEbc @`sddlmZmZmZddlmZddlmZmZddl Z ddl Z ddl m Z m Z ddlmZddlmZmZmZmZdd lmZdd lmZegeD]Zejd ^qZegeD]Zejd ^qZegeD]Zejd ^qZeed d gBZdZej reddkrej!ddkst"e j#ed e$ddZ%ne j#eZ%ddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1h Z&e j#d2Z'iZ(d3e)fd4YZ*d5Z+d6e)fd7YZ,d8e,fd9YZ-d:e.fd;YZ/d<e)fd=YZ0d>e)fd?YZ1d@Z2dS(Ai(tabsolute_importtdivisiontunicode_literals(t text_type(t http_clientturllibN(tBytesIOtStringIO(t webencodingsi(tEOFtspaceCharacterst asciiLetterstasciiUppercase(t_ReparseException(t_utilsuasciit>t/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt__init__:s  cC`sJd}x,|j|jd D]}|t|7}qW||jd7}|S(Nii(RRtlen(Rtpostchunk((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyttell?s cC`sx||jkst|}d}x>t|j||krd|t|j|8}|d7}q'W||g|_dS(Nii(t_bufferedBytestAssertionErrorRRR(RRtoffsetti((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytseekFscC`sp|js|j|S|jdt|jkr_|jdt|jdkr_|j|S|j|SdS(Niii(Rt _readStreamRRt_readFromBuffer(Rtbytes((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytreadOs     cC`s&tg|jD]}t|^q S(N(tsumRR(Rtitem((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRXscC`sL|jj|}|jj||jdcd7/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR [s cC`s"|}g}|jd}|jd}x|t|jkr|dkr|dks\t|j|}|t||kr|}|||g|_n/t||}|t|g|_|d7}|j||||!||8}d}q)W|r|j|j|ndj|S(Niit(RRRRR&R tjoin(RR"tremainingBytestrvt bufferIndext bufferOffsett bufferedDatat bytesToRead((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR!bs&  $    ( t__name__t __module__t__doc__RRRR#RR R!(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR3s    cK`st|tjs<t|tjjrEt|jtjrEt}n9t|drot|j dt }nt|t }|rg|D]}|j dr|^q}|rt d|nt ||St||SdS(Nureadiu _encodingu3Cannot set an encoding with a unicode input, set %r(t isinstanceRt HTTPResponseRtresponsetaddbasetfptFalsethasattrR#Rtendswitht TypeErrortHTMLUnicodeInputStreamtHTMLBinaryInputStream(tsourcetkwargst isUnicodetxt encodings((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytHTMLInputStream}s ( R<cB`s}eZdZdZdZdZdZdZdZdZ d dZ d Z d Z ed Zd ZRS(uProvides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing incorrect byte-sequences and also provides column and line tracking. i(cC`stjsd|_n-tddkr6|j|_n |j|_dg|_tddf|_ |j ||_ |j dS(uInitialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) u􏿿iiuutf-8ucertainN( Rtsupports_lone_surrogatestNonetreportCharacterErrorsRtcharacterErrorsUCS4tcharacterErrorsUCS2tnewLinestlookupEncodingt charEncodingt openStreamt dataStreamtreset(RR>((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs    cC`sCd|_d|_d|_g|_d|_d|_d|_dS(Nui(Rt chunkSizet chunkOffsetterrorst prevNumLinest prevNumColsREt_bufferedCharacter(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRNs      cC`s(t|dr|}n t|}|S(uvProduces a file object from source. source can be either a file object, local filename or a string. uread(R9R(RR>R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRLs  cC`st|j}|jdd|}|j|}|jdd|}|dkr\|j|}n||d}||fS(Nu iii(RtcountRRtrfindRS(RRRtnLinest positionLinet lastLinePostpositionColumn((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt _positions   cC`s&|j|j\}}|d|fS(u:Returns (line, col) of the current position in the stream.i(R[RP(Rtlinetcol((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRscC`sL|j|jkr%|js%tSn|j}|j|}|d|_|S(uo Read one character from the stream or queue if available. Return EOF when EOF is reached. i(RPROt readChunkR R(RRPtchar((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR_s    cC`sO|dkr|j}n|j|j\|_|_d|_d|_d|_|jj |}|j r|j |}d|_ n |st St |dkrt |d}|dksd|kodknr|d|_ |d }qn|jr|j|n|jdd }|jd d }||_t ||_tS( Nuiiii iiu u u (REt_defaultChunkSizeR[RORRRSRRPRMR#RTR8RtordRFtreplacetTrue(RROR'tlastv((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR^s0        (   cC`s:x3tttj|D]}|jjdqWdS(Nuinvalid-codepoint(trangeRtinvalid_unicode_retfindallRQR&(RR't_((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRGs"cC`st}xtj|D]}|r(qnt|j}|j}tj|||d!rtj|||d!}|t kr|j j dnt }q|dkr|dkr|t |dkr|j j dqt}|j j dqWdS(Niuinvalid-codepointiii(R8RftfinditerRatgrouptstartRtisSurrogatePairtsurrogatePairToCodepointtnon_bmp_invalid_codepointsRQR&RcR(RR'tskiptmatcht codepointRtchar_val((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRH#s    c C`s}yt||f}Wntk rx&|D]}t|dks+tq+Wdjg|D]}dt|^qZ}|sd|}ntjd|}t||f/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt charsUntil:s2  , %     cC`so|tk rk|jdkr=||j|_|jd7_qk|jd8_|j|j|ksktndS(Nii(R RPRROR(RR_((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytungetis  N(R0R1R2R`RRNRLR[RR_RER^RGRHR8RR(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR<s     &   /R=cB`sbeZdZd d d d dedZdZdZedZdZ dZ dZ RS( uProvides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing incorrect byte-sequences and also provides column and line tracking. u windows-1252cC`s|j||_tj||jd|_d|_||_||_||_||_ ||_ |j ||_ |j ddk st|jdS(uInitialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) iidiN(RLt rawStreamR<Rt numBytesMetatnumBytesChardettoverride_encodingttransport_encodingtsame_origin_parent_encodingtlikely_encodingtdefault_encodingtdetermineEncodingRKRERRN(RR>RRRRRt useChardet((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs       cC`s3|jdjj|jd|_tj|dS(Niureplace(RKt codec_infot streamreaderRRMR<RN(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRNs"cC`s_t|dr|}n t|}y|j|jWntk rZt|}nX|S(uvProduces a file object from source. source can be either a file object, local filename or a string. uread(R9RRRt ExceptionR(RR>R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRLs   cC`s!|jdf}|ddk r&|St|jdf}|ddk rO|St|jdf}|ddk rx|S|jdf}|ddk r|St|jdf}|ddk r|djjd r|St|j df}|ddk r|S|ryddl m }Wnt k r4qXg}|}x[|j s|jj|j}t|tszt|sPn|j||j|qGW|jt|jd}|jjd|dk r|dfSnt|jdf}|ddk r|StddfS(Nucertainiu tentativeuutf-16(tUniversalDetectoruencodingu windows-1252(t detectBOMRERJRRtdetectEncodingMetaRtnamet startswithRt%pip._vendor.chardet.universaldetectorRt ImportErrortdoneRR#RR3R"RR&tfeedtclosetresultRR(RtchardetRKRtbufferstdetectorRtencoding((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRsR'       cC`s|jddkstt|}|dkr5dS|jdkretd}|dk stnr||jdkr|jddf|_nF|jjd|df|_|jtd|jd|fdS( Niucertainuutf-16beuutf-16leuutf-8iuEncoding changed from %s to %s(uutf-16beuutf-16le( RKRRJRERRRRNR (Rt newEncoding((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytchangeEncodings    cC`sidtj6dtj6dtj6dtj6dtj6}|jjd}t|t s_t |j |d }d}|s|j |}d}|s|j |d }d}qn|r|jj |t |S|jj d d Sd S( uAttempts to detect at BOM at the start of the stream. If an encoding can be determined from the BOM return the name of the encoding otherwise return Noneuutf-8uutf-16leuutf-16beuutf-32leuutf-32beiiiiN(tcodecstBOM_UTF8t BOM_UTF16_LEt BOM_UTF16_BEt BOM_UTF32_LEt BOM_UTF32_BERR#R3R"RtgetRRJRE(RtbomDicttstringRR((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs&   cC`s|jj|j}t|ts*tt|}|jjd|j}|dk r||j dkr|t d}n|S(u9Report the encoding declared by the meta element iuutf-16beuutf-16leuutf-8N(uutf-16beuutf-16le( RR#RR3R"RtEncodingParserRt getEncodingRERRJ(RRtparserR((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR3s  N( R0R1R2RERcRRNRLRRRR(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR=zs(   >  "t EncodingBytescB`seZdZdZdZdZdZdZdZdZ dZ e e e Z d Z e e Zed Zd Zd Zd ZRS(uString-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raisedcC`s+t|tsttj||jS(N(R3R"Rt__new__tlower(Rtvalue((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRFscC`s d|_dS(Ni(R[(RR((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRJscC`s|S(N((R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt__iter__NscC`sS|jd}|_|t|kr/tn|dkrDtn|||d!S(Nii(R[Rt StopIterationR;(Rtp((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt__next__Qs    cC`s |jS(N(R(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytnextYscC`sY|j}|t|kr$tn|dkr9tn|d|_}|||d!S(Nii(R[RRR;(RR((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytprevious]s    cC`s+|jt|krtn||_dS(N(R[RR(RR((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt setPositionfs cC`s<|jt|krtn|jdkr4|jSdSdS(Ni(R[RRRE(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt getPositionks  cC`s||j|jd!S(Ni(R(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytgetCurrentByteuscC`sc|j}xJ|t|krU|||d!}||krH||_|S|d7}q W||_dS(uSkip past a list of charactersiN(RRR[RE(RRzRR{((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRozs    cC`sc|j}xJ|t|krU|||d!}||krH||_|S|d7}q W||_dS(Ni(RRR[RE(RRzRR{((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt skipUntils    cC`s7|j||j}|r3|jt|7_n|S(uLook for a sequence of bytes at the start of a string. If the bytes are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone(RRR(RR"R+((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyt matchBytesscC`sHy*|j||jt|d|_Wntk rCtnXtS(uLook for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the matchi(tindexRRR[t ValueErrorRRc(RR"((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytjumpTos *  (R0R1R2RRRRRRRRtpropertyRRt currentBytetspaceCharactersBytesRoRRR(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRBs           RcB`s_eZdZdZdZdZdZdZdZdZ dZ d Z RS( u?Mini parser for detecting character encoding from meta elementscC`st||_d|_dS(u3string - the data to work on for encoding detectionN(RR'RER(RR'((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRscC`s d|jkrdSd|jfd|jfd|jfd|jfd|jfd|jff}x|jD]}t}y|jjdWnt k rPnXxR|D]J\}}|jj |ry|}PWqt k rt }PqXqqW|skPqkqkW|j S(Ns/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs2         cC`s|jjdS(uSkip over commentss-->(R'R(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRscC`sK|jjtkrtSt}d}x"trF|j}|dkrGtS|ddkr|ddk}|rC|dk rC||_tSq%|ddkr|d}t|}|dk rC||_tSq%|ddkr%t t |d}|j }|dk rCt|}|dk r@|r4||_tS|}q@qCq%q%WdS(Nis http-equivis content-typetcharsettcontent( R'RRRcR8REt getAttributeRRJtContentAttrParserRtparse(Rt hasPragmatpendingEncodingtattrttentativeEncodingtcodect contentParser((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs:             cC`s |jtS(N(thandlePossibleTagR8(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRscC`st|j|jtS(N(RR'RRc(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs cC`s|j}|jtkr9|r5|j|jntS|jt}|dkra|jn+|j}x|dk r|j}qpWtS(NR( R'RtasciiLettersBytesRRRcRtspacesAngleBracketsRRE(RtendTagR'R{R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs      cC`s|jjdS(NR(R'R(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRscC`s|j}|jttdgB}|dksIt|dksIt|d krYdSg}g}xtr |dkr|rPnz|tkr|j}Pn^|d krdj|dfS|t kr|j |j n|dkrdS|j |t |}qhW|dkr7|j dj|dfSt ||j}|d kr|}xtrt |}||krt |dj|dj|fS|t kr|j |j qb|j |qbWn^|dkrdj|dfS|t kr|j |j n|dkr-dS|j |x}trt |}|tkrwdj|dj|fS|t kr|j |j q=|dkrdS|j |q=WdS( u_Return a name,value pair for the next attribute in the stream, if one is found, or Nonet/iRt=R(t't"N(RN(RR(RR(R'RoRt frozensetRERRRcR)tasciiUppercaseBytesR&RRRR(RR'R{tattrNamet attrValuet quoteChar((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRsh $                           ( R0R1R2RRRRRRRRR(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRs    $    RcB`seZdZdZRS(cC`s"t|tst||_dS(N(R3R"RR'(RR'((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRascC`s:y!|jjd|jjd7_|jj|jjdksHdS|jjd7_|jj|jjdkr|jj}|jjd7_|jj}|jj|r|j||jj!SdSnP|jj}y(|jjt|j||jj!SWntk r|j|SXWntk r5dSXdS(NRiRRR(RR( R'RRRoRRERRR(Rt quoteMarkt oldPosition((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRes.       (R0R1RR(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyR`s cC`swt|tr:y|jd}Wq:tk r6dSXn|dk roytj|SWqstk rkdSXndSdS(u{Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.uasciiN(R3R"tdecodetUnicodeDecodeErrorRERtlookuptAttributeError(R((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pyRJs   (3t __future__RRRtpip._vendor.sixRtpip._vendor.six.movesRRRRutioRRt pip._vendorRt constantsR R R R R R(RRR%tencodeRRRRtinvalid_unicode_no_surrogateRDRURRvtevalRfRntascii_punctuation_reRstobjectRRCR<R=R"RRRRJ(((s>/tmp/pip-build-UPPWic/pip/pip/_vendor/html5lib/_inputstream.pytsJ  "((( +  J b'