2017-11-23 07:22:33 +01:00
|
|
|
// Copyright 2011 Google Inc. All rights reserved.
|
|
|
|
// Use of this source code is governed by the Apache 2.0
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
package blobstore
|
|
|
|
|
|
|
|
import (
|
2018-07-07 06:18:14 +02:00
|
|
|
"bytes"
|
|
|
|
"encoding/base64"
|
|
|
|
"fmt"
|
2017-11-23 07:22:33 +01:00
|
|
|
"io"
|
2018-07-07 06:18:14 +02:00
|
|
|
"mime/multipart"
|
|
|
|
"mime/quotedprintable"
|
|
|
|
"net/http"
|
|
|
|
"net/textproto"
|
2017-11-23 07:22:33 +01:00
|
|
|
"os"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"testing"
|
|
|
|
|
2018-07-07 06:18:14 +02:00
|
|
|
"golang.org/x/text/encoding/htmlindex"
|
|
|
|
|
2017-11-23 07:22:33 +01:00
|
|
|
"google.golang.org/appengine"
|
|
|
|
"google.golang.org/appengine/internal/aetesting"
|
|
|
|
|
|
|
|
pb "google.golang.org/appengine/internal/blobstore"
|
|
|
|
)
|
|
|
|
|
|
|
|
const rbs = readBufferSize
|
|
|
|
|
2018-07-07 06:18:14 +02:00
|
|
|
const charsetUTF8 = "utf-8"
|
|
|
|
const charsetISO2022JP = "iso-2022-jp"
|
|
|
|
const nonASCIIStr = "Hello, 世界"
|
|
|
|
|
2017-11-23 07:22:33 +01:00
|
|
|
func min(x, y int) int {
|
|
|
|
if x < y {
|
|
|
|
return x
|
|
|
|
}
|
|
|
|
return y
|
|
|
|
}
|
|
|
|
|
|
|
|
func fakeFetchData(req *pb.FetchDataRequest, res *pb.FetchDataResponse) error {
|
|
|
|
i0 := int(*req.StartIndex)
|
|
|
|
i1 := int(*req.EndIndex + 1) // Blobstore's end-indices are inclusive; Go's are exclusive.
|
|
|
|
bk := *req.BlobKey
|
|
|
|
if i := strings.Index(bk, "."); i != -1 {
|
|
|
|
// Strip everything past the ".".
|
|
|
|
bk = bk[:i]
|
|
|
|
}
|
|
|
|
switch bk {
|
|
|
|
case "a14p":
|
|
|
|
const s = "abcdefghijklmnop"
|
|
|
|
i0 := min(len(s), i0)
|
|
|
|
i1 := min(len(s), i1)
|
|
|
|
res.Data = []byte(s[i0:i1])
|
|
|
|
case "longBlob":
|
|
|
|
res.Data = make([]byte, i1-i0)
|
|
|
|
for i := range res.Data {
|
|
|
|
res.Data[i] = 'A' + uint8(i0/rbs)
|
|
|
|
i0++
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// step is one step of a readerTest.
|
|
|
|
// It consists of a Reader method to call, the method arguments
|
|
|
|
// (lenp, offset, whence) and the expected results.
|
|
|
|
type step struct {
|
|
|
|
method string
|
|
|
|
lenp int
|
|
|
|
offset int64
|
|
|
|
whence int
|
|
|
|
want string
|
|
|
|
wantErr error
|
|
|
|
}
|
|
|
|
|
|
|
|
var readerTest = []struct {
|
|
|
|
blobKey string
|
|
|
|
step []step
|
|
|
|
}{
|
|
|
|
{"noSuchBlobKey", []step{
|
|
|
|
{"Read", 8, 0, 0, "", io.EOF},
|
|
|
|
}},
|
|
|
|
{"a14p.0", []step{
|
|
|
|
// Test basic reads.
|
|
|
|
{"Read", 1, 0, 0, "a", nil},
|
|
|
|
{"Read", 3, 0, 0, "bcd", nil},
|
|
|
|
{"Read", 1, 0, 0, "e", nil},
|
|
|
|
{"Read", 2, 0, 0, "fg", nil},
|
|
|
|
// Test Seek.
|
|
|
|
{"Seek", 0, 2, os.SEEK_SET, "2", nil},
|
|
|
|
{"Read", 5, 0, 0, "cdefg", nil},
|
|
|
|
{"Seek", 0, 2, os.SEEK_CUR, "9", nil},
|
|
|
|
{"Read", 1, 0, 0, "j", nil},
|
|
|
|
// Test reads up to and past EOF.
|
|
|
|
{"Read", 5, 0, 0, "klmno", nil},
|
|
|
|
{"Read", 5, 0, 0, "p", nil},
|
|
|
|
{"Read", 5, 0, 0, "", io.EOF},
|
|
|
|
// Test ReadAt.
|
|
|
|
{"ReadAt", 4, 0, 0, "abcd", nil},
|
|
|
|
{"ReadAt", 4, 3, 0, "defg", nil},
|
|
|
|
{"ReadAt", 4, 12, 0, "mnop", nil},
|
|
|
|
{"ReadAt", 4, 13, 0, "nop", io.EOF},
|
|
|
|
{"ReadAt", 4, 99, 0, "", io.EOF},
|
|
|
|
}},
|
|
|
|
{"a14p.1", []step{
|
|
|
|
// Test Seek before any reads.
|
|
|
|
{"Seek", 0, 2, os.SEEK_SET, "2", nil},
|
|
|
|
{"Read", 1, 0, 0, "c", nil},
|
|
|
|
// Test that ReadAt doesn't affect the Read offset.
|
|
|
|
{"ReadAt", 3, 9, 0, "jkl", nil},
|
|
|
|
{"Read", 3, 0, 0, "def", nil},
|
|
|
|
}},
|
|
|
|
{"a14p.2", []step{
|
|
|
|
// Test ReadAt before any reads or seeks.
|
|
|
|
{"ReadAt", 2, 14, 0, "op", nil},
|
|
|
|
}},
|
|
|
|
{"longBlob.0", []step{
|
|
|
|
// Test basic read.
|
|
|
|
{"Read", 1, 0, 0, "A", nil},
|
|
|
|
// Test that Read returns early when the buffer is exhausted.
|
|
|
|
{"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil},
|
|
|
|
{"Read", 5, 0, 0, "AA", nil},
|
|
|
|
{"Read", 3, 0, 0, "BBB", nil},
|
|
|
|
// Test that what we just read is still in the buffer.
|
|
|
|
{"Seek", 0, rbs - 2, os.SEEK_SET, strconv.Itoa(rbs - 2), nil},
|
|
|
|
{"Read", 5, 0, 0, "AABBB", nil},
|
|
|
|
// Test ReadAt.
|
|
|
|
{"ReadAt", 3, rbs - 4, 0, "AAA", nil},
|
|
|
|
{"ReadAt", 6, rbs - 4, 0, "AAAABB", nil},
|
|
|
|
{"ReadAt", 8, rbs - 4, 0, "AAAABBBB", nil},
|
|
|
|
{"ReadAt", 5, rbs - 4, 0, "AAAAB", nil},
|
|
|
|
{"ReadAt", 2, rbs - 4, 0, "AA", nil},
|
|
|
|
// Test seeking backwards from the Read offset.
|
|
|
|
{"Seek", 0, 2*rbs - 8, os.SEEK_SET, strconv.Itoa(2*rbs - 8), nil},
|
|
|
|
{"Read", 1, 0, 0, "B", nil},
|
|
|
|
{"Read", 1, 0, 0, "B", nil},
|
|
|
|
{"Read", 1, 0, 0, "B", nil},
|
|
|
|
{"Read", 1, 0, 0, "B", nil},
|
|
|
|
{"Read", 8, 0, 0, "BBBBCCCC", nil},
|
|
|
|
}},
|
|
|
|
{"longBlob.1", []step{
|
|
|
|
// Test ReadAt with a slice larger than the buffer size.
|
|
|
|
{"LargeReadAt", 2*rbs - 2, 0, 0, strconv.Itoa(2*rbs - 2), nil},
|
|
|
|
{"LargeReadAt", 2*rbs - 1, 0, 0, strconv.Itoa(2*rbs - 1), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 0, 0, 0, strconv.Itoa(2*rbs + 0), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 1, 0, 0, strconv.Itoa(2*rbs + 1), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 2, 0, 0, strconv.Itoa(2*rbs + 2), nil},
|
|
|
|
{"LargeReadAt", 2*rbs - 2, 1, 0, strconv.Itoa(2*rbs - 2), nil},
|
|
|
|
{"LargeReadAt", 2*rbs - 1, 1, 0, strconv.Itoa(2*rbs - 1), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 0, 1, 0, strconv.Itoa(2*rbs + 0), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 1, 1, 0, strconv.Itoa(2*rbs + 1), nil},
|
|
|
|
{"LargeReadAt", 2*rbs + 2, 1, 0, strconv.Itoa(2*rbs + 2), nil},
|
|
|
|
}},
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestReader(t *testing.T) {
|
|
|
|
for _, rt := range readerTest {
|
|
|
|
c := aetesting.FakeSingleContext(t, "blobstore", "FetchData", fakeFetchData)
|
|
|
|
r := NewReader(c, appengine.BlobKey(rt.blobKey))
|
|
|
|
for i, step := range rt.step {
|
|
|
|
var (
|
|
|
|
got string
|
|
|
|
gotErr error
|
|
|
|
n int
|
|
|
|
offset int64
|
|
|
|
)
|
|
|
|
switch step.method {
|
|
|
|
case "LargeReadAt":
|
|
|
|
p := make([]byte, step.lenp)
|
|
|
|
n, gotErr = r.ReadAt(p, step.offset)
|
|
|
|
got = strconv.Itoa(n)
|
|
|
|
case "Read":
|
|
|
|
p := make([]byte, step.lenp)
|
|
|
|
n, gotErr = r.Read(p)
|
|
|
|
got = string(p[:n])
|
|
|
|
case "ReadAt":
|
|
|
|
p := make([]byte, step.lenp)
|
|
|
|
n, gotErr = r.ReadAt(p, step.offset)
|
|
|
|
got = string(p[:n])
|
|
|
|
case "Seek":
|
|
|
|
offset, gotErr = r.Seek(step.offset, step.whence)
|
|
|
|
got = strconv.FormatInt(offset, 10)
|
|
|
|
default:
|
|
|
|
t.Fatalf("unknown method: %s", step.method)
|
|
|
|
}
|
|
|
|
if gotErr != step.wantErr {
|
|
|
|
t.Fatalf("%s step %d: got error %v want %v", rt.blobKey, i, gotErr, step.wantErr)
|
|
|
|
}
|
|
|
|
if got != step.want {
|
|
|
|
t.Fatalf("%s step %d: got %q want %q", rt.blobKey, i, got, step.want)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-07-07 06:18:14 +02:00
|
|
|
|
|
|
|
// doPlainTextParseUploadTest tests ParseUpload's decoding of non-file form fields.
|
|
|
|
// It ensures that MIME multipart parts with Content-Type not equal to
|
|
|
|
// "message/external-body" (i.e. form fields that are not file uploads) are decoded
|
|
|
|
// correctly according to the value of their Content-Transfer-Encoding header field.
|
|
|
|
// If charset is not the empty string it will be set in the request's Content-Type
|
|
|
|
// header field, and if encoding is not the empty string then the Content-Transfer-Encoding
|
|
|
|
// header field will be set.
|
|
|
|
func doPlainTextParseUploadTest(t *testing.T, charset string, encoding string,
|
|
|
|
rawContent string, encodedContent string) {
|
|
|
|
bodyBuf := &bytes.Buffer{}
|
|
|
|
w := multipart.NewWriter(bodyBuf)
|
|
|
|
|
|
|
|
fieldName := "foo"
|
|
|
|
hdr := textproto.MIMEHeader{}
|
|
|
|
hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", fieldName))
|
|
|
|
|
|
|
|
if charset != "" {
|
|
|
|
hdr.Set("Content-Type", fmt.Sprintf("text/plain; charset=%q", charset))
|
|
|
|
} else {
|
|
|
|
hdr.Set("Content-Type", "text/plain")
|
|
|
|
}
|
|
|
|
|
|
|
|
if encoding != "" {
|
|
|
|
hdr.Set("Content-Transfer-Encoding", encoding)
|
|
|
|
}
|
|
|
|
|
|
|
|
pw, err := w.CreatePart(hdr)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error creating part: %v", err)
|
|
|
|
}
|
|
|
|
pw.Write([]byte(encodedContent))
|
|
|
|
|
|
|
|
if err := w.Close(); err != nil {
|
|
|
|
t.Fatalf("error closing multipart writer: %v\n", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
req, err := http.NewRequest("POST", "/upload", bodyBuf)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error creating request: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", w.FormDataContentType())
|
|
|
|
_, other, err := ParseUpload(req)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error parsing upload: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if other[fieldName][0] != rawContent {
|
|
|
|
t.Errorf("got %q expected %q", other[fieldName][0], rawContent)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestParseUploadUTF8Base64Encoding(t *testing.T) {
|
|
|
|
encoded := base64.StdEncoding.EncodeToString([]byte(nonASCIIStr))
|
|
|
|
doPlainTextParseUploadTest(t, charsetUTF8, "base64", nonASCIIStr, encoded)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestParseUploadUTF8Base64EncodingMultiline(t *testing.T) {
|
|
|
|
testStr := "words words words words words words words words words words words words"
|
|
|
|
encoded := "d29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29yZHMgd29y\r\nZHMgd29yZHMgd29yZHM="
|
|
|
|
doPlainTextParseUploadTest(t, charsetUTF8, "base64", testStr, encoded)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestParseUploadUTF8QuotedPrintableEncoding(t *testing.T) {
|
|
|
|
var encoded bytes.Buffer
|
|
|
|
writer := quotedprintable.NewWriter(&encoded)
|
|
|
|
writer.Write([]byte(nonASCIIStr))
|
|
|
|
writer.Close()
|
|
|
|
|
|
|
|
doPlainTextParseUploadTest(t, charsetUTF8, "quoted-printable", nonASCIIStr,
|
|
|
|
encoded.String())
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestParseUploadISO2022JPBase64Encoding(t *testing.T) {
|
|
|
|
testStr := "こんにちは"
|
|
|
|
encoding, err := htmlindex.Get(charsetISO2022JP)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error getting encoding: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
charsetEncoded, err := encoding.NewEncoder().String(testStr)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("error encoding string: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
base64Encoded := base64.StdEncoding.EncodeToString([]byte(charsetEncoded))
|
|
|
|
doPlainTextParseUploadTest(t, charsetISO2022JP, "base64", testStr, base64Encoded)
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestParseUploadNoEncoding(t *testing.T) {
|
|
|
|
doPlainTextParseUploadTest(t, "", "", "Hello", "Hello")
|
|
|
|
}
|