parent
9c34e98234
commit
4f7d618d21
@ -0,0 +1,27 @@ |
|||||||
|
Copyright (c) 2009 The Go Authors. All rights reserved. |
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without |
||||||
|
modification, are permitted provided that the following conditions are |
||||||
|
met: |
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright |
||||||
|
notice, this list of conditions and the following disclaimer. |
||||||
|
* Redistributions in binary form must reproduce the above |
||||||
|
copyright notice, this list of conditions and the following disclaimer |
||||||
|
in the documentation and/or other materials provided with the |
||||||
|
distribution. |
||||||
|
* Neither the name of Google Inc. nor the names of its |
||||||
|
contributors may be used to endorse or promote products derived from |
||||||
|
this software without specific prior written permission. |
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
@ -0,0 +1,22 @@ |
|||||||
|
Additional IP Rights Grant (Patents) |
||||||
|
|
||||||
|
"This implementation" means the copyrightable works distributed by |
||||||
|
Google as part of the Go project. |
||||||
|
|
||||||
|
Google hereby grants to You a perpetual, worldwide, non-exclusive, |
||||||
|
no-charge, royalty-free, irrevocable (except as stated in this section) |
||||||
|
patent license to make, have made, use, offer to sell, sell, import, |
||||||
|
transfer and otherwise run, modify and propagate the contents of this |
||||||
|
implementation of Go, where such license applies only to those patent |
||||||
|
claims, both currently owned or controlled by Google and acquired in |
||||||
|
the future, licensable by Google that are necessarily infringed by this |
||||||
|
implementation of Go. This grant does not include claims that would be |
||||||
|
infringed only as a consequence of further modification of this |
||||||
|
implementation. If you or your agent or exclusive licensee institute or |
||||||
|
order or agree to the institution of patent litigation against any |
||||||
|
entity (including a cross-claim or counterclaim in a lawsuit) alleging |
||||||
|
that this implementation of Go or any code incorporated within this |
||||||
|
implementation of Go constitutes direct or contributory patent |
||||||
|
infringement, or inducement of patent infringement, then any patent |
||||||
|
rights granted to you under this License for this implementation of Go |
||||||
|
shall terminate as of the date such litigation is filed. |
@ -0,0 +1,257 @@ |
|||||||
|
// Copyright 2013 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Package charset provides common text encodings for HTML documents.
|
||||||
|
//
|
||||||
|
// The mapping from encoding labels to encodings is defined at
|
||||||
|
// https://encoding.spec.whatwg.org/.
|
||||||
|
package charset // import "golang.org/x/net/html/charset"
|
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"fmt" |
||||||
|
"io" |
||||||
|
"mime" |
||||||
|
"strings" |
||||||
|
"unicode/utf8" |
||||||
|
|
||||||
|
"golang.org/x/net/html" |
||||||
|
"golang.org/x/text/encoding" |
||||||
|
"golang.org/x/text/encoding/charmap" |
||||||
|
"golang.org/x/text/encoding/htmlindex" |
||||||
|
"golang.org/x/text/transform" |
||||||
|
) |
||||||
|
|
||||||
|
// Lookup returns the encoding with the specified label, and its canonical
|
||||||
|
// name. It returns nil and the empty string if label is not one of the
|
||||||
|
// standard encodings for HTML. Matching is case-insensitive and ignores
|
||||||
|
// leading and trailing whitespace. Encoders will use HTML escape sequences for
|
||||||
|
// runes that are not supported by the character set.
|
||||||
|
func Lookup(label string) (e encoding.Encoding, name string) { |
||||||
|
e, err := htmlindex.Get(label) |
||||||
|
if err != nil { |
||||||
|
return nil, "" |
||||||
|
} |
||||||
|
name, _ = htmlindex.Name(e) |
||||||
|
return &htmlEncoding{e}, name |
||||||
|
} |
||||||
|
|
||||||
|
type htmlEncoding struct{ encoding.Encoding } |
||||||
|
|
||||||
|
func (h *htmlEncoding) NewEncoder() *encoding.Encoder { |
||||||
|
// HTML requires a non-terminating legacy encoder. We use HTML escapes to
|
||||||
|
// substitute unsupported code points.
|
||||||
|
return encoding.HTMLEscapeUnsupported(h.Encoding.NewEncoder()) |
||||||
|
} |
||||||
|
|
||||||
|
// DetermineEncoding determines the encoding of an HTML document by examining
|
||||||
|
// up to the first 1024 bytes of content and the declared Content-Type.
|
||||||
|
//
|
||||||
|
// See http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#determining-the-character-encoding
|
||||||
|
func DetermineEncoding(content []byte, contentType string) (e encoding.Encoding, name string, certain bool) { |
||||||
|
if len(content) > 1024 { |
||||||
|
content = content[:1024] |
||||||
|
} |
||||||
|
|
||||||
|
for _, b := range boms { |
||||||
|
if bytes.HasPrefix(content, b.bom) { |
||||||
|
e, name = Lookup(b.enc) |
||||||
|
return e, name, true |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if _, params, err := mime.ParseMediaType(contentType); err == nil { |
||||||
|
if cs, ok := params["charset"]; ok { |
||||||
|
if e, name = Lookup(cs); e != nil { |
||||||
|
return e, name, true |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if len(content) > 0 { |
||||||
|
e, name = prescan(content) |
||||||
|
if e != nil { |
||||||
|
return e, name, false |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Try to detect UTF-8.
|
||||||
|
// First eliminate any partial rune at the end.
|
||||||
|
for i := len(content) - 1; i >= 0 && i > len(content)-4; i-- { |
||||||
|
b := content[i] |
||||||
|
if b < 0x80 { |
||||||
|
break |
||||||
|
} |
||||||
|
if utf8.RuneStart(b) { |
||||||
|
content = content[:i] |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
hasHighBit := false |
||||||
|
for _, c := range content { |
||||||
|
if c >= 0x80 { |
||||||
|
hasHighBit = true |
||||||
|
break |
||||||
|
} |
||||||
|
} |
||||||
|
if hasHighBit && utf8.Valid(content) { |
||||||
|
return encoding.Nop, "utf-8", false |
||||||
|
} |
||||||
|
|
||||||
|
// TODO: change default depending on user's locale?
|
||||||
|
return charmap.Windows1252, "windows-1252", false |
||||||
|
} |
||||||
|
|
||||||
|
// NewReader returns an io.Reader that converts the content of r to UTF-8.
|
||||||
|
// It calls DetermineEncoding to find out what r's encoding is.
|
||||||
|
func NewReader(r io.Reader, contentType string) (io.Reader, error) { |
||||||
|
preview := make([]byte, 1024) |
||||||
|
n, err := io.ReadFull(r, preview) |
||||||
|
switch { |
||||||
|
case err == io.ErrUnexpectedEOF: |
||||||
|
preview = preview[:n] |
||||||
|
r = bytes.NewReader(preview) |
||||||
|
case err != nil: |
||||||
|
return nil, err |
||||||
|
default: |
||||||
|
r = io.MultiReader(bytes.NewReader(preview), r) |
||||||
|
} |
||||||
|
|
||||||
|
if e, _, _ := DetermineEncoding(preview, contentType); e != encoding.Nop { |
||||||
|
r = transform.NewReader(r, e.NewDecoder()) |
||||||
|
} |
||||||
|
return r, nil |
||||||
|
} |
||||||
|
|
||||||
|
// NewReaderLabel returns a reader that converts from the specified charset to
|
||||||
|
// UTF-8. It uses Lookup to find the encoding that corresponds to label, and
|
||||||
|
// returns an error if Lookup returns nil. It is suitable for use as
|
||||||
|
// encoding/xml.Decoder's CharsetReader function.
|
||||||
|
func NewReaderLabel(label string, input io.Reader) (io.Reader, error) { |
||||||
|
e, _ := Lookup(label) |
||||||
|
if e == nil { |
||||||
|
return nil, fmt.Errorf("unsupported charset: %q", label) |
||||||
|
} |
||||||
|
return transform.NewReader(input, e.NewDecoder()), nil |
||||||
|
} |
||||||
|
|
||||||
|
func prescan(content []byte) (e encoding.Encoding, name string) { |
||||||
|
z := html.NewTokenizer(bytes.NewReader(content)) |
||||||
|
for { |
||||||
|
switch z.Next() { |
||||||
|
case html.ErrorToken: |
||||||
|
return nil, "" |
||||||
|
|
||||||
|
case html.StartTagToken, html.SelfClosingTagToken: |
||||||
|
tagName, hasAttr := z.TagName() |
||||||
|
if !bytes.Equal(tagName, []byte("meta")) { |
||||||
|
continue |
||||||
|
} |
||||||
|
attrList := make(map[string]bool) |
||||||
|
gotPragma := false |
||||||
|
|
||||||
|
const ( |
||||||
|
dontKnow = iota |
||||||
|
doNeedPragma |
||||||
|
doNotNeedPragma |
||||||
|
) |
||||||
|
needPragma := dontKnow |
||||||
|
|
||||||
|
name = "" |
||||||
|
e = nil |
||||||
|
for hasAttr { |
||||||
|
var key, val []byte |
||||||
|
key, val, hasAttr = z.TagAttr() |
||||||
|
ks := string(key) |
||||||
|
if attrList[ks] { |
||||||
|
continue |
||||||
|
} |
||||||
|
attrList[ks] = true |
||||||
|
for i, c := range val { |
||||||
|
if 'A' <= c && c <= 'Z' { |
||||||
|
val[i] = c + 0x20 |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
switch ks { |
||||||
|
case "http-equiv": |
||||||
|
if bytes.Equal(val, []byte("content-type")) { |
||||||
|
gotPragma = true |
||||||
|
} |
||||||
|
|
||||||
|
case "content": |
||||||
|
if e == nil { |
||||||
|
name = fromMetaElement(string(val)) |
||||||
|
if name != "" { |
||||||
|
e, name = Lookup(name) |
||||||
|
if e != nil { |
||||||
|
needPragma = doNeedPragma |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
case "charset": |
||||||
|
e, name = Lookup(string(val)) |
||||||
|
needPragma = doNotNeedPragma |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if needPragma == dontKnow || needPragma == doNeedPragma && !gotPragma { |
||||||
|
continue |
||||||
|
} |
||||||
|
|
||||||
|
if strings.HasPrefix(name, "utf-16") { |
||||||
|
name = "utf-8" |
||||||
|
e = encoding.Nop |
||||||
|
} |
||||||
|
|
||||||
|
if e != nil { |
||||||
|
return e, name |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func fromMetaElement(s string) string { |
||||||
|
for s != "" { |
||||||
|
csLoc := strings.Index(s, "charset") |
||||||
|
if csLoc == -1 { |
||||||
|
return "" |
||||||
|
} |
||||||
|
s = s[csLoc+len("charset"):] |
||||||
|
s = strings.TrimLeft(s, " \t\n\f\r") |
||||||
|
if !strings.HasPrefix(s, "=") { |
||||||
|
continue |
||||||
|
} |
||||||
|
s = s[1:] |
||||||
|
s = strings.TrimLeft(s, " \t\n\f\r") |
||||||
|
if s == "" { |
||||||
|
return "" |
||||||
|
} |
||||||
|
if q := s[0]; q == '"' || q == '\'' { |
||||||
|
s = s[1:] |
||||||
|
closeQuote := strings.IndexRune(s, rune(q)) |
||||||
|
if closeQuote == -1 { |
||||||
|
return "" |
||||||
|
} |
||||||
|
return s[:closeQuote] |
||||||
|
} |
||||||
|
|
||||||
|
end := strings.IndexAny(s, "; \t\n\f\r") |
||||||
|
if end == -1 { |
||||||
|
end = len(s) |
||||||
|
} |
||||||
|
return s[:end] |
||||||
|
} |
||||||
|
return "" |
||||||
|
} |
||||||
|
|
||||||
|
var boms = []struct { |
||||||
|
bom []byte |
||||||
|
enc string |
||||||
|
}{ |
||||||
|
{[]byte{0xfe, 0xff}, "utf-16be"}, |
||||||
|
{[]byte{0xff, 0xfe}, "utf-16le"}, |
||||||
|
{[]byte{0xef, 0xbb, 0xbf}, "utf-8"}, |
||||||
|
} |
Loading…
Reference in new issue