mirror of
https://github.com/restic/restic.git
synced 2024-08-21 20:03:23 +02:00
Among others, this updates minio-go, so that the new "eu-west-3" zone for AWS is supported.
509 lines
14 KiB
Go
509 lines
14 KiB
Go
// Copyright 2011 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package norm
|
|
|
|
import "unicode/utf8"
|
|
|
|
const (
|
|
maxNonStarters = 30
|
|
// The maximum number of characters needed for a buffer is
|
|
// maxNonStarters + 1 for the starter + 1 for the GCJ
|
|
maxBufferSize = maxNonStarters + 2
|
|
maxNFCExpansion = 3 // NFC(0x1D160)
|
|
maxNFKCExpansion = 18 // NFKC(0xFDFA)
|
|
|
|
maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
|
|
)
|
|
|
|
// ssState is used for reporting the segment state after inserting a rune.
|
|
// It is returned by streamSafe.next.
|
|
type ssState int
|
|
|
|
const (
|
|
// Indicates a rune was successfully added to the segment.
|
|
ssSuccess ssState = iota
|
|
// Indicates a rune starts a new segment and should not be added.
|
|
ssStarter
|
|
// Indicates a rune caused a segment overflow and a CGJ should be inserted.
|
|
ssOverflow
|
|
)
|
|
|
|
// streamSafe implements the policy of when a CGJ should be inserted.
|
|
type streamSafe uint8
|
|
|
|
// first inserts the first rune of a segment. It is a faster version of next if
|
|
// it is known p represents the first rune in a segment.
|
|
func (ss *streamSafe) first(p Properties) {
|
|
*ss = streamSafe(p.nTrailingNonStarters())
|
|
}
|
|
|
|
// insert returns a ssState value to indicate whether a rune represented by p
|
|
// can be inserted.
|
|
func (ss *streamSafe) next(p Properties) ssState {
|
|
if *ss > maxNonStarters {
|
|
panic("streamSafe was not reset")
|
|
}
|
|
n := p.nLeadingNonStarters()
|
|
if *ss += streamSafe(n); *ss > maxNonStarters {
|
|
*ss = 0
|
|
return ssOverflow
|
|
}
|
|
// The Stream-Safe Text Processing prescribes that the counting can stop
|
|
// as soon as a starter is encountered. However, there are some starters,
|
|
// like Jamo V and T, that can combine with other runes, leaving their
|
|
// successive non-starters appended to the previous, possibly causing an
|
|
// overflow. We will therefore consider any rune with a non-zero nLead to
|
|
// be a non-starter. Note that it always hold that if nLead > 0 then
|
|
// nLead == nTrail.
|
|
if n == 0 {
|
|
*ss = streamSafe(p.nTrailingNonStarters())
|
|
return ssStarter
|
|
}
|
|
return ssSuccess
|
|
}
|
|
|
|
// backwards is used for checking for overflow and segment starts
|
|
// when traversing a string backwards. Users do not need to call first
|
|
// for the first rune. The state of the streamSafe retains the count of
|
|
// the non-starters loaded.
|
|
func (ss *streamSafe) backwards(p Properties) ssState {
|
|
if *ss > maxNonStarters {
|
|
panic("streamSafe was not reset")
|
|
}
|
|
c := *ss + streamSafe(p.nTrailingNonStarters())
|
|
if c > maxNonStarters {
|
|
return ssOverflow
|
|
}
|
|
*ss = c
|
|
if p.nLeadingNonStarters() == 0 {
|
|
return ssStarter
|
|
}
|
|
return ssSuccess
|
|
}
|
|
|
|
func (ss streamSafe) isMax() bool {
|
|
return ss == maxNonStarters
|
|
}
|
|
|
|
// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
|
|
const GraphemeJoiner = "\u034F"
|
|
|
|
// reorderBuffer is used to normalize a single segment. Characters inserted with
|
|
// insert are decomposed and reordered based on CCC. The compose method can
|
|
// be used to recombine characters. Note that the byte buffer does not hold
|
|
// the UTF-8 characters in order. Only the rune array is maintained in sorted
|
|
// order. flush writes the resulting segment to a byte array.
|
|
type reorderBuffer struct {
|
|
rune [maxBufferSize]Properties // Per character info.
|
|
byte [maxByteBufferSize]byte // UTF-8 buffer. Referenced by runeInfo.pos.
|
|
nbyte uint8 // Number or bytes.
|
|
ss streamSafe // For limiting length of non-starter sequence.
|
|
nrune int // Number of runeInfos.
|
|
f formInfo
|
|
|
|
src input
|
|
nsrc int
|
|
tmpBytes input
|
|
|
|
out []byte
|
|
flushF func(*reorderBuffer) bool
|
|
}
|
|
|
|
func (rb *reorderBuffer) init(f Form, src []byte) {
|
|
rb.f = *formTable[f]
|
|
rb.src.setBytes(src)
|
|
rb.nsrc = len(src)
|
|
rb.ss = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) initString(f Form, src string) {
|
|
rb.f = *formTable[f]
|
|
rb.src.setString(src)
|
|
rb.nsrc = len(src)
|
|
rb.ss = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
|
|
rb.out = out
|
|
rb.flushF = f
|
|
}
|
|
|
|
// reset discards all characters from the buffer.
|
|
func (rb *reorderBuffer) reset() {
|
|
rb.nrune = 0
|
|
rb.nbyte = 0
|
|
}
|
|
|
|
func (rb *reorderBuffer) doFlush() bool {
|
|
if rb.f.composing {
|
|
rb.compose()
|
|
}
|
|
res := rb.flushF(rb)
|
|
rb.reset()
|
|
return res
|
|
}
|
|
|
|
// appendFlush appends the normalized segment to rb.out.
|
|
func appendFlush(rb *reorderBuffer) bool {
|
|
for i := 0; i < rb.nrune; i++ {
|
|
start := rb.rune[i].pos
|
|
end := start + rb.rune[i].size
|
|
rb.out = append(rb.out, rb.byte[start:end]...)
|
|
}
|
|
return true
|
|
}
|
|
|
|
// flush appends the normalized segment to out and resets rb.
|
|
func (rb *reorderBuffer) flush(out []byte) []byte {
|
|
for i := 0; i < rb.nrune; i++ {
|
|
start := rb.rune[i].pos
|
|
end := start + rb.rune[i].size
|
|
out = append(out, rb.byte[start:end]...)
|
|
}
|
|
rb.reset()
|
|
return out
|
|
}
|
|
|
|
// flushCopy copies the normalized segment to buf and resets rb.
|
|
// It returns the number of bytes written to buf.
|
|
func (rb *reorderBuffer) flushCopy(buf []byte) int {
|
|
p := 0
|
|
for i := 0; i < rb.nrune; i++ {
|
|
runep := rb.rune[i]
|
|
p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
|
|
}
|
|
rb.reset()
|
|
return p
|
|
}
|
|
|
|
// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
|
|
// It returns false if the buffer is not large enough to hold the rune.
|
|
// It is used internally by insert and insertString only.
|
|
func (rb *reorderBuffer) insertOrdered(info Properties) {
|
|
n := rb.nrune
|
|
b := rb.rune[:]
|
|
cc := info.ccc
|
|
if cc > 0 {
|
|
// Find insertion position + move elements to make room.
|
|
for ; n > 0; n-- {
|
|
if b[n-1].ccc <= cc {
|
|
break
|
|
}
|
|
b[n] = b[n-1]
|
|
}
|
|
}
|
|
rb.nrune += 1
|
|
pos := uint8(rb.nbyte)
|
|
rb.nbyte += utf8.UTFMax
|
|
info.pos = pos
|
|
b[n] = info
|
|
}
|
|
|
|
// insertErr is an error code returned by insert. Using this type instead
|
|
// of error improves performance up to 20% for many of the benchmarks.
|
|
type insertErr int
|
|
|
|
const (
|
|
iSuccess insertErr = -iota
|
|
iShortDst
|
|
iShortSrc
|
|
)
|
|
|
|
// insertFlush inserts the given rune in the buffer ordered by CCC.
|
|
// If a decomposition with multiple segments are encountered, they leading
|
|
// ones are flushed.
|
|
// It returns a non-zero error code if the rune was not inserted.
|
|
func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
|
|
if rune := src.hangul(i); rune != 0 {
|
|
rb.decomposeHangul(rune)
|
|
return iSuccess
|
|
}
|
|
if info.hasDecomposition() {
|
|
return rb.insertDecomposed(info.Decomposition())
|
|
}
|
|
rb.insertSingle(src, i, info)
|
|
return iSuccess
|
|
}
|
|
|
|
// insertUnsafe inserts the given rune in the buffer ordered by CCC.
|
|
// It is assumed there is sufficient space to hold the runes. It is the
|
|
// responsibility of the caller to ensure this. This can be done by checking
|
|
// the state returned by the streamSafe type.
|
|
func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
|
|
if rune := src.hangul(i); rune != 0 {
|
|
rb.decomposeHangul(rune)
|
|
}
|
|
if info.hasDecomposition() {
|
|
// TODO: inline.
|
|
rb.insertDecomposed(info.Decomposition())
|
|
} else {
|
|
rb.insertSingle(src, i, info)
|
|
}
|
|
}
|
|
|
|
// insertDecomposed inserts an entry in to the reorderBuffer for each rune
|
|
// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
|
|
// It flushes the buffer on each new segment start.
|
|
func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
|
|
rb.tmpBytes.setBytes(dcomp)
|
|
// As the streamSafe accounting already handles the counting for modifiers,
|
|
// we don't have to call next. However, we do need to keep the accounting
|
|
// intact when flushing the buffer.
|
|
for i := 0; i < len(dcomp); {
|
|
info := rb.f.info(rb.tmpBytes, i)
|
|
if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
|
|
return iShortDst
|
|
}
|
|
i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
|
|
rb.insertOrdered(info)
|
|
}
|
|
return iSuccess
|
|
}
|
|
|
|
// insertSingle inserts an entry in the reorderBuffer for the rune at
|
|
// position i. info is the runeInfo for the rune at position i.
|
|
func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
|
|
src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
|
|
rb.insertOrdered(info)
|
|
}
|
|
|
|
// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
|
|
func (rb *reorderBuffer) insertCGJ() {
|
|
rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
|
|
}
|
|
|
|
// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
|
|
func (rb *reorderBuffer) appendRune(r rune) {
|
|
bn := rb.nbyte
|
|
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
|
|
rb.nbyte += utf8.UTFMax
|
|
rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
|
|
rb.nrune++
|
|
}
|
|
|
|
// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) assignRune(pos int, r rune) {
|
|
bn := rb.rune[pos].pos
|
|
sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
|
|
rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
|
|
}
|
|
|
|
// runeAt returns the rune at position n. It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) runeAt(n int) rune {
|
|
inf := rb.rune[n]
|
|
r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
|
|
return r
|
|
}
|
|
|
|
// bytesAt returns the UTF-8 encoding of the rune at position n.
|
|
// It is used for Hangul and recomposition.
|
|
func (rb *reorderBuffer) bytesAt(n int) []byte {
|
|
inf := rb.rune[n]
|
|
return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
|
|
}
|
|
|
|
// For Hangul we combine algorithmically, instead of using tables.
|
|
const (
|
|
hangulBase = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
|
|
hangulBase0 = 0xEA
|
|
hangulBase1 = 0xB0
|
|
hangulBase2 = 0x80
|
|
|
|
hangulEnd = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
|
|
hangulEnd0 = 0xED
|
|
hangulEnd1 = 0x9E
|
|
hangulEnd2 = 0xA4
|
|
|
|
jamoLBase = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
|
|
jamoLBase0 = 0xE1
|
|
jamoLBase1 = 0x84
|
|
jamoLEnd = 0x1113
|
|
jamoVBase = 0x1161
|
|
jamoVEnd = 0x1176
|
|
jamoTBase = 0x11A7
|
|
jamoTEnd = 0x11C3
|
|
|
|
jamoTCount = 28
|
|
jamoVCount = 21
|
|
jamoVTCount = 21 * 28
|
|
jamoLVTCount = 19 * 21 * 28
|
|
)
|
|
|
|
const hangulUTF8Size = 3
|
|
|
|
func isHangul(b []byte) bool {
|
|
if len(b) < hangulUTF8Size {
|
|
return false
|
|
}
|
|
b0 := b[0]
|
|
if b0 < hangulBase0 {
|
|
return false
|
|
}
|
|
b1 := b[1]
|
|
switch {
|
|
case b0 == hangulBase0:
|
|
return b1 >= hangulBase1
|
|
case b0 < hangulEnd0:
|
|
return true
|
|
case b0 > hangulEnd0:
|
|
return false
|
|
case b1 < hangulEnd1:
|
|
return true
|
|
}
|
|
return b1 == hangulEnd1 && b[2] < hangulEnd2
|
|
}
|
|
|
|
func isHangulString(b string) bool {
|
|
if len(b) < hangulUTF8Size {
|
|
return false
|
|
}
|
|
b0 := b[0]
|
|
if b0 < hangulBase0 {
|
|
return false
|
|
}
|
|
b1 := b[1]
|
|
switch {
|
|
case b0 == hangulBase0:
|
|
return b1 >= hangulBase1
|
|
case b0 < hangulEnd0:
|
|
return true
|
|
case b0 > hangulEnd0:
|
|
return false
|
|
case b1 < hangulEnd1:
|
|
return true
|
|
}
|
|
return b1 == hangulEnd1 && b[2] < hangulEnd2
|
|
}
|
|
|
|
// Caller must ensure len(b) >= 2.
|
|
func isJamoVT(b []byte) bool {
|
|
// True if (rune & 0xff00) == jamoLBase
|
|
return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
|
|
}
|
|
|
|
func isHangulWithoutJamoT(b []byte) bool {
|
|
c, _ := utf8.DecodeRune(b)
|
|
c -= hangulBase
|
|
return c < jamoLVTCount && c%jamoTCount == 0
|
|
}
|
|
|
|
// decomposeHangul writes the decomposed Hangul to buf and returns the number
|
|
// of bytes written. len(buf) should be at least 9.
|
|
func decomposeHangul(buf []byte, r rune) int {
|
|
const JamoUTF8Len = 3
|
|
r -= hangulBase
|
|
x := r % jamoTCount
|
|
r /= jamoTCount
|
|
utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
|
|
utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
|
|
if x != 0 {
|
|
utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
|
|
return 3 * JamoUTF8Len
|
|
}
|
|
return 2 * JamoUTF8Len
|
|
}
|
|
|
|
// decomposeHangul algorithmically decomposes a Hangul rune into
|
|
// its Jamo components.
|
|
// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
|
|
func (rb *reorderBuffer) decomposeHangul(r rune) {
|
|
r -= hangulBase
|
|
x := r % jamoTCount
|
|
r /= jamoTCount
|
|
rb.appendRune(jamoLBase + r/jamoVCount)
|
|
rb.appendRune(jamoVBase + r%jamoVCount)
|
|
if x != 0 {
|
|
rb.appendRune(jamoTBase + x)
|
|
}
|
|
}
|
|
|
|
// combineHangul algorithmically combines Jamo character components into Hangul.
|
|
// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
|
|
func (rb *reorderBuffer) combineHangul(s, i, k int) {
|
|
b := rb.rune[:]
|
|
bn := rb.nrune
|
|
for ; i < bn; i++ {
|
|
cccB := b[k-1].ccc
|
|
cccC := b[i].ccc
|
|
if cccB == 0 {
|
|
s = k - 1
|
|
}
|
|
if s != k-1 && cccB >= cccC {
|
|
// b[i] is blocked by greater-equal cccX below it
|
|
b[k] = b[i]
|
|
k++
|
|
} else {
|
|
l := rb.runeAt(s) // also used to compare to hangulBase
|
|
v := rb.runeAt(i) // also used to compare to jamoT
|
|
switch {
|
|
case jamoLBase <= l && l < jamoLEnd &&
|
|
jamoVBase <= v && v < jamoVEnd:
|
|
// 11xx plus 116x to LV
|
|
rb.assignRune(s, hangulBase+
|
|
(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
|
|
case hangulBase <= l && l < hangulEnd &&
|
|
jamoTBase < v && v < jamoTEnd &&
|
|
((l-hangulBase)%jamoTCount) == 0:
|
|
// ACxx plus 11Ax to LVT
|
|
rb.assignRune(s, l+v-jamoTBase)
|
|
default:
|
|
b[k] = b[i]
|
|
k++
|
|
}
|
|
}
|
|
}
|
|
rb.nrune = k
|
|
}
|
|
|
|
// compose recombines the runes in the buffer.
|
|
// It should only be used to recompose a single segment, as it will not
|
|
// handle alternations between Hangul and non-Hangul characters correctly.
|
|
func (rb *reorderBuffer) compose() {
|
|
// UAX #15, section X5 , including Corrigendum #5
|
|
// "In any character sequence beginning with starter S, a character C is
|
|
// blocked from S if and only if there is some character B between S
|
|
// and C, and either B is a starter or it has the same or higher
|
|
// combining class as C."
|
|
bn := rb.nrune
|
|
if bn == 0 {
|
|
return
|
|
}
|
|
k := 1
|
|
b := rb.rune[:]
|
|
for s, i := 0, 1; i < bn; i++ {
|
|
if isJamoVT(rb.bytesAt(i)) {
|
|
// Redo from start in Hangul mode. Necessary to support
|
|
// U+320E..U+321E in NFKC mode.
|
|
rb.combineHangul(s, i, k)
|
|
return
|
|
}
|
|
ii := b[i]
|
|
// We can only use combineForward as a filter if we later
|
|
// get the info for the combined character. This is more
|
|
// expensive than using the filter. Using combinesBackward()
|
|
// is safe.
|
|
if ii.combinesBackward() {
|
|
cccB := b[k-1].ccc
|
|
cccC := ii.ccc
|
|
blocked := false // b[i] blocked by starter or greater or equal CCC?
|
|
if cccB == 0 {
|
|
s = k - 1
|
|
} else {
|
|
blocked = s != k-1 && cccB >= cccC
|
|
}
|
|
if !blocked {
|
|
combined := combine(rb.runeAt(s), rb.runeAt(i))
|
|
if combined != 0 {
|
|
rb.assignRune(s, combined)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
b[k] = b[i]
|
|
k++
|
|
}
|
|
rb.nrune = k
|
|
}
|