beszel/golang-github-pocketbase-pocketbase
1
0
Fork 0
Code Issues Activity

Adding upstream version 0.28.1.

Browse source 
Signed-off-by: Daniel Baumann <daniel@debian.org>
This commit is contained in:
Daniel Baumann 2025-05-22 10:57:38 +02:00
parent 88f1d47ab6
commit e28c88ef14
Signed by: daniel
GPG key ID: FBB4F0E80A80222F
933 changed files with 194711 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

736
tools/filesystem/blob/bucket.go Normal file
View file

@ -0,0 +1,736 @@
// Package blob defines a lightweight abstration for interacting with
// various storage services (local filesystem, S3, etc.).
//
// NB!
// For compatibility with earlier PocketBase versions and to prevent
// unnecessary breaking changes, this package is based and implemented
// as a minimal, stripped down version of the previously used gocloud.dev/blob.
// While there is no promise that it won't diverge in the future to accommodate
// better some PocketBase specific use cases, currently it copies and
// tries to follow as close as possible the same implementations,
// conventions and rules for the key escaping/unescaping, blob read/write
// interfaces and struct options as gocloud.dev/blob, therefore the
// credits goes to the original Go Cloud Development Kit Authors.
package blob
import (
"bytes"
"context"
"crypto/md5"
"errors"
"fmt"
"io"
"log"
"mime"
"runtime"
"strings"
"sync"
"time"
"unicode/utf8"
)
var (
ErrNotFound = errors.New("resource not found")
ErrClosed = errors.New("bucket or blob is closed")
)
// Bucket provides an easy and portable way to interact with blobs
// within a "bucket", including read, write, and list operations.
// To create a Bucket, use constructors found in driver subpackages.
type Bucket struct {
drv Driver
// mu protects the closed variable.
// Read locks are kept to allow holding a read lock for long-running calls,
// and thereby prevent closing until a call finishes.
mu sync.RWMutex
closed bool
}
// NewBucket creates a new *Bucket based on a specific driver implementation.
func NewBucket(drv Driver) *Bucket {
return &Bucket{drv: drv}
}
// ListOptions sets options for listing blobs via Bucket.List.
type ListOptions struct {
// Prefix indicates that only blobs with a key starting with this prefix
// should be returned.
Prefix string
// Delimiter sets the delimiter used to define a hierarchical namespace,
// like a filesystem with "directories". It is highly recommended that you
// use "" or "/" as the Delimiter. Other values should work through this API,
// but service UIs generally assume "/".
//
// An empty delimiter means that the bucket is treated as a single flat
// namespace.
//
// A non-empty delimiter means that any result with the delimiter in its key
// after Prefix is stripped will be returned with ListObject.IsDir = true,
// ListObject.Key truncated after the delimiter, and zero values for other
// ListObject fields. These results represent "directories". Multiple results
// in a "directory" are returned as a single result.
Delimiter string
// PageSize sets the maximum number of objects to be returned.
// 0 means no maximum; driver implementations should choose a reasonable
// max. It is guaranteed to be >= 0.
PageSize int
// PageToken may be filled in with the NextPageToken from a previous
// ListPaged call.
PageToken []byte
}
// ListPage represents a page of results return from ListPaged.
type ListPage struct {
// Objects is the slice of objects found. If ListOptions.PageSize > 0,
// it should have at most ListOptions.PageSize entries.
//
// Objects should be returned in lexicographical order of UTF-8 encoded keys,
// including across pages. I.e., all objects returned from a ListPage request
// made using a PageToken from a previous ListPage request's NextPageToken
// should have Key >= the Key for all objects from the previous request.
Objects []*ListObject `json:"objects"`
// NextPageToken should be left empty unless there are more objects
// to return. The value may be returned as ListOptions.PageToken on a
// subsequent ListPaged call, to fetch the next page of results.
// It can be an arbitrary []byte; it need not be a valid key.
NextPageToken []byte `json:"nextPageToken"`
}
// ListIterator iterates over List results.
type ListIterator struct {
b *Bucket
opts *ListOptions
page *ListPage
nextIdx int
}
// Next returns a *ListObject for the next blob.
// It returns (nil, io.EOF) if there are no more.
func (i *ListIterator) Next(ctx context.Context) (*ListObject, error) {
if i.page != nil {
// We've already got a page of results.
if i.nextIdx < len(i.page.Objects) {
// Next object is in the page; return it.
dobj := i.page.Objects[i.nextIdx]
i.nextIdx++
return &ListObject{
Key: dobj.Key,
ModTime: dobj.ModTime,
Size: dobj.Size,
MD5: dobj.MD5,
IsDir: dobj.IsDir,
}, nil
}
if len(i.page.NextPageToken) == 0 {
// Done with current page, and there are no more; return io.EOF.
return nil, io.EOF
}
// We need to load the next page.
i.opts.PageToken = i.page.NextPageToken
}
i.b.mu.RLock()
defer i.b.mu.RUnlock()
if i.b.closed {
return nil, ErrClosed
}
// Loading a new page.
p, err := i.b.drv.ListPaged(ctx, i.opts)
if err != nil {
return nil, wrapError(i.b.drv, err, "")
}
i.page = p
i.nextIdx = 0
return i.Next(ctx)
}
// ListObject represents a single blob returned from List.
type ListObject struct {
// Key is the key for this blob.
Key string `json:"key"`
// ModTime is the time the blob was last modified.
ModTime time.Time `json:"modTime"`
// Size is the size of the blob's content in bytes.
Size int64 `json:"size"`
// MD5 is an MD5 hash of the blob contents or nil if not available.
MD5 []byte `json:"md5"`
// IsDir indicates that this result represents a "directory" in the
// hierarchical namespace, ending in ListOptions.Delimiter. Key can be
// passed as ListOptions.Prefix to list items in the "directory".
// Fields other than Key and IsDir will not be set if IsDir is true.
IsDir bool `json:"isDir"`
}
// List returns a ListIterator that can be used to iterate over blobs in a
// bucket, in lexicographical order of UTF-8 encoded keys. The underlying
// implementation fetches results in pages.
//
// A nil ListOptions is treated the same as the zero value.
//
// List is not guaranteed to include all recently-written blobs;
// some services are only eventually consistent.
func (b *Bucket) List(opts *ListOptions) *ListIterator {
if opts == nil {
opts = &ListOptions{}
}
dopts := &ListOptions{
Prefix: opts.Prefix,
Delimiter: opts.Delimiter,
}
return &ListIterator{b: b, opts: dopts}
}
// FirstPageToken is the pageToken to pass to ListPage to retrieve the first page of results.
var FirstPageToken = []byte("first page")
// ListPage returns a page of ListObject results for blobs in a bucket, in lexicographical
// order of UTF-8 encoded keys.
//
// To fetch the first page, pass FirstPageToken as the pageToken. For subsequent pages, pass
// the pageToken returned from a previous call to ListPage.
// It is not possible to "skip ahead" pages.
//
// Each call will return pageSize results, unless there are not enough blobs to fill the
// page, in which case it will return fewer results (possibly 0).
//
// If there are no more blobs available, ListPage will return an empty pageToken. Note that
// this may happen regardless of the number of returned results -- the last page might have
// 0 results (i.e., if the last item was deleted), pageSize results, or anything in between.
//
// Calling ListPage with an empty pageToken will immediately return io.EOF. When looping
// over pages, callers can either check for an empty pageToken, or they can make one more
// call and check for io.EOF.
//
// The underlying implementation fetches results in pages, but one call to ListPage may
// require multiple page fetches (and therefore, multiple calls to the BeforeList callback).
//
// A nil ListOptions is treated the same as the zero value.
//
// ListPage is not guaranteed to include all recently-written blobs;
// some services are only eventually consistent.
func (b *Bucket) ListPage(ctx context.Context, pageToken []byte, pageSize int, opts *ListOptions) (retval []*ListObject, nextPageToken []byte, err error) {
if opts == nil {
opts = &ListOptions{}
}
if pageSize <= 0 {
return nil, nil, fmt.Errorf("pageSize must be > 0 (%d)", pageSize)
}
// Nil pageToken means no more results.
if len(pageToken) == 0 {
return nil, nil, io.EOF
}
// FirstPageToken fetches the first page. Drivers use nil.
// The public API doesn't use nil for the first page because it would be too easy to
// keep fetching forever (since the last page return nil for the next pageToken).
if bytes.Equal(pageToken, FirstPageToken) {
pageToken = nil
}
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return nil, nil, ErrClosed
}
dopts := &ListOptions{
Prefix: opts.Prefix,
Delimiter: opts.Delimiter,
PageToken: pageToken,
PageSize: pageSize,
}
retval = make([]*ListObject, 0, pageSize)
for len(retval) < pageSize {
p, err := b.drv.ListPaged(ctx, dopts)
if err != nil {
return nil, nil, wrapError(b.drv, err, "")
}
for _, dobj := range p.Objects {
retval = append(retval, &ListObject{
Key: dobj.Key,
ModTime: dobj.ModTime,
Size: dobj.Size,
MD5: dobj.MD5,
IsDir: dobj.IsDir,
})
}
// ListPaged may return fewer results than pageSize. If there are more results
// available, signalled by non-empty p.NextPageToken, try to fetch the remainder
// of the page.
// It does not work to ask for more results than we need, because then we'd have
// a NextPageToken on a non-page boundary.
dopts.PageSize = pageSize - len(retval)
dopts.PageToken = p.NextPageToken
if len(dopts.PageToken) == 0 {
dopts.PageToken = nil
break
}
}
return retval, dopts.PageToken, nil
}
// Attributes contains attributes about a blob.
type Attributes struct {
// CacheControl specifies caching attributes that services may use
// when serving the blob.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
CacheControl string `json:"cacheControl"`
// ContentDisposition specifies whether the blob content is expected to be
// displayed inline or as an attachment.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
ContentDisposition string `json:"contentDisposition"`
// ContentEncoding specifies the encoding used for the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
ContentEncoding string `json:"contentEncoding"`
// ContentLanguage specifies the language used in the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language
ContentLanguage string `json:"contentLanguage"`
// ContentType is the MIME type of the blob. It will not be empty.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type
ContentType string `json:"contentType"`
// Metadata holds key/value pairs associated with the blob.
// Keys are guaranteed to be in lowercase, even if the backend service
// has case-sensitive keys (although note that Metadata written via
// this package will always be lowercased). If there are duplicate
// case-insensitive keys (e.g., "foo" and "FOO"), only one value
// will be kept, and it is undefined which one.
Metadata map[string]string `json:"metadata"`
// CreateTime is the time the blob was created, if available. If not available,
// CreateTime will be the zero time.
CreateTime time.Time `json:"createTime"`
// ModTime is the time the blob was last modified.
ModTime time.Time `json:"modTime"`
// Size is the size of the blob's content in bytes.
Size int64 `json:"size"`
// MD5 is an MD5 hash of the blob contents or nil if not available.
MD5 []byte `json:"md5"`
// ETag for the blob; see https://en.wikipedia.org/wiki/HTTP_ETag.
ETag string `json:"etag"`
}
// Attributes returns attributes for the blob stored at key.
//
// If the blob does not exist, Attributes returns an error for which
// gcerrors.Code will return gcerrors.NotFound.
func (b *Bucket) Attributes(ctx context.Context, key string) (_ *Attributes, err error) {
if !utf8.ValidString(key) {
return nil, fmt.Errorf("Attributes key must be a valid UTF-8 string: %q", key)
}
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return nil, ErrClosed
}
a, err := b.drv.Attributes(ctx, key)
if err != nil {
return nil, wrapError(b.drv, err, key)
}
var md map[string]string
if len(a.Metadata) > 0 {
// Services are inconsistent, but at least some treat keys
// as case-insensitive. To make the behavior consistent, we
// force-lowercase them when writing and reading.
md = make(map[string]string, len(a.Metadata))
for k, v := range a.Metadata {
md[strings.ToLower(k)] = v
}
}
return &Attributes{
CacheControl: a.CacheControl,
ContentDisposition: a.ContentDisposition,
ContentEncoding: a.ContentEncoding,
ContentLanguage: a.ContentLanguage,
ContentType: a.ContentType,
Metadata: md,
CreateTime: a.CreateTime,
ModTime: a.ModTime,
Size: a.Size,
MD5: a.MD5,
ETag: a.ETag,
}, nil
}
// Exists returns true if a blob exists at key, false if it does not exist, or
// an error.
//
// It is a shortcut for calling Attributes and checking if it returns an error
// with code ErrNotFound.
func (b *Bucket) Exists(ctx context.Context, key string) (bool, error) {
_, err := b.Attributes(ctx, key)
if err == nil {
return true, nil
}
if errors.Is(err, ErrNotFound) {
return false, nil
}
return false, err
}
// NewReader is a shortcut for NewRangeReader with offset=0 and length=-1.
func (b *Bucket) NewReader(ctx context.Context, key string) (*Reader, error) {
return b.newRangeReader(ctx, key, 0, -1)
}
// NewRangeReader returns a Reader to read content from the blob stored at key.
// It reads at most length bytes starting at offset (>= 0).
// If length is negative, it will read till the end of the blob.
//
// For the purposes of Seek, the returned Reader will start at offset and
// end at the minimum of the actual end of the blob or (if length > 0) offset + length.
//
// Note that ctx is used for all reads performed during the lifetime of the reader.
//
// If the blob does not exist, NewRangeReader returns an error for which
// gcerrors.Code will return gcerrors.NotFound. Exists is a lighter-weight way
// to check for existence.
//
// A nil ReaderOptions is treated the same as the zero value.
//
// The caller must call Close on the returned Reader when done reading.
func (b *Bucket) NewRangeReader(ctx context.Context, key string, offset, length int64) (_ *Reader, err error) {
return b.newRangeReader(ctx, key, offset, length)
}
func (b *Bucket) newRangeReader(ctx context.Context, key string, offset, length int64) (_ *Reader, err error) {
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return nil, ErrClosed
}
if offset < 0 {
return nil, fmt.Errorf("NewRangeReader offset must be non-negative (%d)", offset)
}
if !utf8.ValidString(key) {
return nil, fmt.Errorf("NewRangeReader key must be a valid UTF-8 string: %q", key)
}
var dr DriverReader
dr, err = b.drv.NewRangeReader(ctx, key, offset, length)
if err != nil {
return nil, wrapError(b.drv, err, key)
}
r := &Reader{
drv: b.drv,
r: dr,
key: key,
ctx: ctx,
baseOffset: offset,
baseLength: length,
savedOffset: -1,
}
_, file, lineno, ok := runtime.Caller(2)
runtime.SetFinalizer(r, func(r *Reader) {
if !r.closed {
var caller string
if ok {
caller = fmt.Sprintf(" (%s:%d)", file, lineno)
}
log.Printf("A blob.Reader reading from %q was never closed%s", key, caller)
}
})
return r, nil
}
// WriterOptions sets options for NewWriter.
type WriterOptions struct {
// BufferSize changes the default size in bytes of the chunks that
// Writer will upload in a single request; larger blobs will be split into
// multiple requests.
//
// This option may be ignored by some drivers.
//
// If 0, the driver will choose a reasonable default.
//
// If the Writer is used to do many small writes concurrently, using a
// smaller BufferSize may reduce memory usage.
BufferSize int
// MaxConcurrency changes the default concurrency for parts of an upload.
//
// This option may be ignored by some drivers.
//
// If 0, the driver will choose a reasonable default.
MaxConcurrency int
// CacheControl specifies caching attributes that services may use
// when serving the blob.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
CacheControl string
// ContentDisposition specifies whether the blob content is expected to be
// displayed inline or as an attachment.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
ContentDisposition string
// ContentEncoding specifies the encoding used for the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
ContentEncoding string
// ContentLanguage specifies the language used in the blob's content, if any.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Language
ContentLanguage string
// ContentType specifies the MIME type of the blob being written. If not set,
// it will be inferred from the content using the algorithm described at
// http://mimesniff.spec.whatwg.org/.
// Set DisableContentTypeDetection to true to disable the above and force
// the ContentType to stay empty.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type
ContentType string
// When true, if ContentType is the empty string, it will stay the empty
// string rather than being inferred from the content.
// Note that while the blob will be written with an empty string ContentType,
// most providers will fill one in during reads, so don't expect an empty
// ContentType if you read the blob back.
DisableContentTypeDetection bool
// ContentMD5 is used as a message integrity check.
// If len(ContentMD5) > 0, the MD5 hash of the bytes written must match
// ContentMD5, or Close will return an error without completing the write.
// https://tools.ietf.org/html/rfc1864
ContentMD5 []byte
// Metadata holds key/value strings to be associated with the blob, or nil.
// Keys may not be empty, and are lowercased before being written.
// Duplicate case-insensitive keys (e.g., "foo" and "FOO") will result in
// an error.
Metadata map[string]string
}
// NewWriter returns a Writer that writes to the blob stored at key.
// A nil WriterOptions is treated the same as the zero value.
//
// If a blob with this key already exists, it will be replaced.
// The blob being written is not guaranteed to be readable until Close
// has been called; until then, any previous blob will still be readable.
// Even after Close is called, newly written blobs are not guaranteed to be
// returned from List; some services are only eventually consistent.
//
// The returned Writer will store ctx for later use in Write and/or Close.
// To abort a write, cancel ctx; otherwise, it must remain open until
// Close is called.
//
// The caller must call Close on the returned Writer, even if the write is
// aborted.
func (b *Bucket) NewWriter(ctx context.Context, key string, opts *WriterOptions) (_ *Writer, err error) {
if !utf8.ValidString(key) {
return nil, fmt.Errorf("NewWriter key must be a valid UTF-8 string: %q", key)
}
if opts == nil {
opts = &WriterOptions{}
}
dopts := &WriterOptions{
CacheControl: opts.CacheControl,
ContentDisposition: opts.ContentDisposition,
ContentEncoding: opts.ContentEncoding,
ContentLanguage: opts.ContentLanguage,
ContentMD5: opts.ContentMD5,
BufferSize: opts.BufferSize,
MaxConcurrency: opts.MaxConcurrency,
DisableContentTypeDetection: opts.DisableContentTypeDetection,
}
if len(opts.Metadata) > 0 {
// Services are inconsistent, but at least some treat keys
// as case-insensitive. To make the behavior consistent, we
// force-lowercase them when writing and reading.
md := make(map[string]string, len(opts.Metadata))
for k, v := range opts.Metadata {
if k == "" {
return nil, errors.New("WriterOptions.Metadata keys may not be empty strings")
}
if !utf8.ValidString(k) {
return nil, fmt.Errorf("WriterOptions.Metadata keys must be valid UTF-8 strings: %q", k)
}
if !utf8.ValidString(v) {
return nil, fmt.Errorf("WriterOptions.Metadata values must be valid UTF-8 strings: %q", v)
}
lowerK := strings.ToLower(k)
if _, found := md[lowerK]; found {
return nil, fmt.Errorf("WriterOptions.Metadata has a duplicate case-insensitive metadata key: %q", lowerK)
}
md[lowerK] = v
}
dopts.Metadata = md
}
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return nil, ErrClosed
}
ctx, cancel := context.WithCancel(ctx)
w := &Writer{
drv: b.drv,
cancel: cancel,
key: key,
contentMD5: opts.ContentMD5,
md5hash: md5.New(),
}
if opts.ContentType != "" || opts.DisableContentTypeDetection {
var ct string
if opts.ContentType != "" {
t, p, err := mime.ParseMediaType(opts.ContentType)
if err != nil {
cancel()
return nil, err
}
ct = mime.FormatMediaType(t, p)
}
dw, err := b.drv.NewTypedWriter(ctx, key, ct, dopts)
if err != nil {
cancel()
return nil, wrapError(b.drv, err, key)
}
w.w = dw
} else {
// Save the fields needed to called NewTypedWriter later, once we've gotten
// sniffLen bytes; see the comment on Writer.
w.ctx = ctx
w.opts = dopts
w.buf = bytes.NewBuffer([]byte{})
}
_, file, lineno, ok := runtime.Caller(1)
runtime.SetFinalizer(w, func(w *Writer) {
if !w.closed {
var caller string
if ok {
caller = fmt.Sprintf(" (%s:%d)", file, lineno)
}
log.Printf("A blob.Writer writing to %q was never closed%s", key, caller)
}
})
return w, nil
}
// Copy the blob stored at srcKey to dstKey.
// A nil CopyOptions is treated the same as the zero value.
//
// If the source blob does not exist, Copy returns an error for which
// gcerrors.Code will return gcerrors.NotFound.
//
// If the destination blob already exists, it is overwritten.
func (b *Bucket) Copy(ctx context.Context, dstKey, srcKey string) (err error) {
if !utf8.ValidString(srcKey) {
return fmt.Errorf("Copy srcKey must be a valid UTF-8 string: %q", srcKey)
}
if !utf8.ValidString(dstKey) {
return fmt.Errorf("Copy dstKey must be a valid UTF-8 string: %q", dstKey)
}
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return ErrClosed
}
return wrapError(b.drv, b.drv.Copy(ctx, dstKey, srcKey), fmt.Sprintf("%s -> %s", srcKey, dstKey))
}
// Delete deletes the blob stored at key.
//
// If the blob does not exist, Delete returns an error for which
// gcerrors.Code will return gcerrors.NotFound.
func (b *Bucket) Delete(ctx context.Context, key string) (err error) {
if !utf8.ValidString(key) {
return fmt.Errorf("Delete key must be a valid UTF-8 string: %q", key)
}
b.mu.RLock()
defer b.mu.RUnlock()
if b.closed {
return ErrClosed
}
return wrapError(b.drv, b.drv.Delete(ctx, key), key)
}
// Close releases any resources used for the bucket.
//
// @todo Consider removing it.
func (b *Bucket) Close() error {
b.mu.Lock()
prev := b.closed
b.closed = true
b.mu.Unlock()
if prev {
return ErrClosed
}
return wrapError(b.drv, b.drv.Close(), "")
}
func wrapError(b Driver, err error, key string) error {
if err == nil {
return nil
}
// don't wrap or normalize EOF errors since there are many places
// in the standard library (e.g. io.ReadAll) that rely on checks
// such as "err == io.EOF" and they will fail
if errors.Is(err, io.EOF) {
return err
}
err = b.NormalizeError(err)
if key != "" {
err = fmt.Errorf("[key: %s] %w", key, err)
}
return err
}

107
tools/filesystem/blob/driver.go Normal file
View file

@ -0,0 +1,107 @@
package blob
import (
"context"
"io"
"time"
)
// ReaderAttributes contains a subset of attributes about a blob that are
// accessible from Reader.
type ReaderAttributes struct {
// ContentType is the MIME type of the blob object. It must not be empty.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type
ContentType string `json:"contentType"`
// ModTime is the time the blob object was last modified.
ModTime time.Time `json:"modTime"`
// Size is the size of the object in bytes.
Size int64 `json:"size"`
}
// DriverReader reads an object from the blob.
type DriverReader interface {
io.ReadCloser
// Attributes returns a subset of attributes about the blob.
// The portable type will not modify the returned ReaderAttributes.
Attributes() *ReaderAttributes
}
// DriverWriter writes an object to the blob.
type DriverWriter interface {
io.WriteCloser
}
// Driver provides read, write and delete operations on objects within it on the
// blob service.
type Driver interface {
NormalizeError(err error) error
// Attributes returns attributes for the blob. If the specified object does
// not exist, Attributes must return an error for which ErrorCode returns ErrNotFound.
// The portable type will not modify the returned Attributes.
Attributes(ctx context.Context, key string) (*Attributes, error)
// ListPaged lists objects in the bucket, in lexicographical order by
// UTF-8-encoded key, returning pages of objects at a time.
// Services are only required to be eventually consistent with respect
// to recently written or deleted objects. That is to say, there is no
// guarantee that an object that's been written will immediately be returned
// from ListPaged.
// opts is guaranteed to be non-nil.
ListPaged(ctx context.Context, opts *ListOptions) (*ListPage, error)
// NewRangeReader returns a Reader that reads part of an object, reading at
// most length bytes starting at the given offset. If length is negative, it
// will read until the end of the object. If the specified object does not
// exist, NewRangeReader must return an error for which ErrorCode returns ErrNotFound.
// opts is guaranteed to be non-nil.
//
// The returned Reader *may* also implement Downloader if the underlying
// implementation can take advantage of that. The Download call is guaranteed
// to be the only call to the Reader. For such readers, offset will always
// be 0 and length will always be -1.
NewRangeReader(ctx context.Context, key string, offset, length int64) (DriverReader, error)
// NewTypedWriter returns Writer that writes to an object associated with key.
//
// A new object will be created unless an object with this key already exists.
// Otherwise any previous object with the same key will be replaced.
// The object may not be available (and any previous object will remain)
// until Close has been called.
//
// contentType sets the MIME type of the object to be written.
// opts is guaranteed to be non-nil.
//
// The caller must call Close on the returned Writer when done writing.
//
// Implementations should abort an ongoing write if ctx is later canceled,
// and do any necessary cleanup in Close. Close should then return ctx.Err().
//
// The returned Writer *may* also implement Uploader if the underlying
// implementation can take advantage of that. The Upload call is guaranteed
// to be the only non-Close call to the Writer..
NewTypedWriter(ctx context.Context, key, contentType string, opts *WriterOptions) (DriverWriter, error)
// Copy copies the object associated with srcKey to dstKey.
//
// If the source object does not exist, Copy must return an error for which
// ErrorCode returns ErrNotFound.
//
// If the destination object already exists, it should be overwritten.
//
// opts is guaranteed to be non-nil.
Copy(ctx context.Context, dstKey, srcKey string) error
// Delete deletes the object associated with key. If the specified object does
// not exist, Delete must return an error for which ErrorCode returns ErrNotFound.
Delete(ctx context.Context, key string) error
// Close cleans up any resources used by the Bucket. Once Close is called,
// there will be no method calls to the Bucket other than As, ErrorAs, and
// ErrorCode. There may be open readers or writers that will receive calls.
// It is up to the driver as to how these will be handled.
Close() error
}

153
tools/filesystem/blob/hex.go Normal file
View file

@ -0,0 +1,153 @@
package blob
import (
"fmt"
"strconv"
)
// Copied from gocloud.dev/blob to avoid nuances around the specific
// HEX escaping/unescaping rules.
//
// -------------------------------------------------------------------
// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// -------------------------------------------------------------------
// HexEscape returns s, with all runes for which shouldEscape returns true
// escaped to "__0xXXX__", where XXX is the hex representation of the rune
// value. For example, " " would escape to "__0x20__".
//
// Non-UTF-8 strings will have their non-UTF-8 characters escaped to
// unicode.ReplacementChar; the original value is lost. Please file an
// issue if you need non-UTF8 support.
//
// Note: shouldEscape takes the whole string as a slice of runes and an
// index. Passing it a single byte or a single rune doesn't provide
// enough context for some escape decisions; for example, the caller might
// want to escape the second "/" in "//" but not the first one.
// We pass a slice of runes instead of the string or a slice of bytes
// because some decisions will be made on a rune basis (e.g., encode
// all non-ASCII runes).
func HexEscape(s string, shouldEscape func(s []rune, i int) bool) string {
// Do a first pass to see which runes (if any) need escaping.
runes := []rune(s)
var toEscape []int
for i := range runes {
if shouldEscape(runes, i) {
toEscape = append(toEscape, i)
}
}
if len(toEscape) == 0 {
return s
}
// Each escaped rune turns into at most 14 runes ("__0x7fffffff__"),
// so allocate an extra 13 for each. We'll reslice at the end
// if we didn't end up using them.
escaped := make([]rune, len(runes)+13*len(toEscape))
n := 0 // current index into toEscape
j := 0 // current index into escaped
for i, r := range runes {
if n < len(toEscape) && i == toEscape[n] {
// We were asked to escape this rune.
for _, x := range fmt.Sprintf("__%#x__", r) {
escaped[j] = x
j++
}
n++
} else {
escaped[j] = r
j++
}
}
return string(escaped[0:j])
}
// unescape tries to unescape starting at r[i].
// It returns a boolean indicating whether the unescaping was successful,
// and (if true) the unescaped rune and the last index of r that was used
// during unescaping.
func unescape(r []rune, i int) (bool, rune, int) {
// Look for "__0x".
if r[i] != '_' {
return false, 0, 0
}
i++
if i >= len(r) || r[i] != '_' {
return false, 0, 0
}
i++
if i >= len(r) || r[i] != '0' {
return false, 0, 0
}
i++
if i >= len(r) || r[i] != 'x' {
return false, 0, 0
}
i++
// Capture the digits until the next "_" (if any).
var hexdigits []rune
for ; i < len(r) && r[i] != '_'; i++ {
hexdigits = append(hexdigits, r[i])
}
// Look for the trailing "__".
if i >= len(r) || r[i] != '_' {
return false, 0, 0
}
i++
if i >= len(r) || r[i] != '_' {
return false, 0, 0
}
// Parse the hex digits into an int32.
retval, err := strconv.ParseInt(string(hexdigits), 16, 32)
if err != nil {
return false, 0, 0
}
return true, rune(retval), i
}
// HexUnescape reverses HexEscape.
func HexUnescape(s string) string {
var unescaped []rune
runes := []rune(s)
for i := 0; i < len(runes); i++ {
if ok, newR, newI := unescape(runes, i); ok {
// We unescaped some runes starting at i, resulting in the
// unescaped rune newR. The last rune used was newI.
if unescaped == nil {
// This is the first rune we've encountered that
// needed unescaping. Allocate a buffer and copy any
// previous runes.
unescaped = make([]rune, i)
copy(unescaped, runes)
}
unescaped = append(unescaped, newR)
i = newI
} else if unescaped != nil {
unescaped = append(unescaped, runes[i])
}
}
if unescaped == nil {
return s
}
return string(unescaped)
}

196
tools/filesystem/blob/reader.go Normal file
View file

@ -0,0 +1,196 @@
package blob
import (
"context"
"fmt"
"io"
"log"
"time"
)
// Largely copied from gocloud.dev/blob.Reader to minimize breaking changes.
//
// -------------------------------------------------------------------
// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// -------------------------------------------------------------------
var _ io.ReadSeekCloser = (*Reader)(nil)
// Reader reads bytes from a blob.
// It implements io.ReadSeekCloser, and must be closed after reads are finished.
type Reader struct {
ctx context.Context // Used to recreate r after Seeks
r DriverReader
drv Driver
key string
baseOffset int64 // The base offset provided to NewRangeReader.
baseLength int64 // The length provided to NewRangeReader (may be negative).
relativeOffset int64 // Current offset (relative to baseOffset).
savedOffset int64 // Last relativeOffset for r, saved after relativeOffset is changed in Seek, or -1 if no Seek.
closed bool
}
// Read implements io.Reader (https://golang.org/pkg/io/#Reader).
func (r *Reader) Read(p []byte) (int, error) {
if r.savedOffset != -1 {
// We've done one or more Seeks since the last read. We may have
// to recreate the Reader.
//
// Note that remembering the savedOffset and lazily resetting the
// reader like this allows the caller to Seek, then Seek again back,
// to the original offset, without having to recreate the reader.
// We only have to recreate the reader if we actually read after a Seek.
// This is an important optimization because it's common to Seek
// to (SeekEnd, 0) and use the return value to determine the size
// of the data, then Seek back to (SeekStart, 0).
saved := r.savedOffset
if r.relativeOffset == saved {
// Nope! We're at the same place we left off.
r.savedOffset = -1
} else {
// Yep! We've changed the offset. Recreate the reader.
length := r.baseLength
if length >= 0 {
length -= r.relativeOffset
if length < 0 {
// Shouldn't happen based on checks in Seek.
return 0, fmt.Errorf("invalid Seek (base length %d, relative offset %d)", r.baseLength, r.relativeOffset)
}
}
newR, err := r.drv.NewRangeReader(r.ctx, r.key, r.baseOffset+r.relativeOffset, length)
if err != nil {
return 0, wrapError(r.drv, err, r.key)
}
_ = r.r.Close()
r.savedOffset = -1
r.r = newR
}
}
n, err := r.r.Read(p)
r.relativeOffset += int64(n)
return n, wrapError(r.drv, err, r.key)
}
// Seek implements io.Seeker (https://golang.org/pkg/io/#Seeker).
func (r *Reader) Seek(offset int64, whence int) (int64, error) {
if r.savedOffset == -1 {
// Save the current offset for our reader. If the Seek changes the
// offset, and then we try to read, we'll need to recreate the reader.
// See comment above in Read for why we do it lazily.
r.savedOffset = r.relativeOffset
}
// The maximum relative offset is the minimum of:
// 1. The actual size of the blob, minus our initial baseOffset.
// 2. The length provided to NewRangeReader (if it was non-negative).
maxRelativeOffset := r.Size() - r.baseOffset
if r.baseLength >= 0 && r.baseLength < maxRelativeOffset {
maxRelativeOffset = r.baseLength
}
switch whence {
case io.SeekStart:
r.relativeOffset = offset
case io.SeekCurrent:
r.relativeOffset += offset
case io.SeekEnd:
r.relativeOffset = maxRelativeOffset + offset
}
if r.relativeOffset < 0 {
// "Seeking to an offset before the start of the file is an error."
invalidOffset := r.relativeOffset
r.relativeOffset = 0
return 0, fmt.Errorf("Seek resulted in invalid offset %d, using 0", invalidOffset)
}
if r.relativeOffset > maxRelativeOffset {
// "Seeking to any positive offset is legal, but the behavior of subsequent
// I/O operations on the underlying object is implementation-dependent."
// We'll choose to set the offset to the EOF.
log.Printf("blob.Reader.Seek set an offset after EOF (base offset/length from NewRangeReader %d, %d; actual blob size %d; relative offset %d -> absolute offset %d).", r.baseOffset, r.baseLength, r.Size(), r.relativeOffset, r.baseOffset+r.relativeOffset)
r.relativeOffset = maxRelativeOffset
}
return r.relativeOffset, nil
}
// Close implements io.Closer (https://golang.org/pkg/io/#Closer).
func (r *Reader) Close() error {
r.closed = true
err := wrapError(r.drv, r.r.Close(), r.key)
return err
}
// ContentType returns the MIME type of the blob.
func (r *Reader) ContentType() string {
return r.r.Attributes().ContentType
}
// ModTime returns the time the blob was last modified.
func (r *Reader) ModTime() time.Time {
return r.r.Attributes().ModTime
}
// Size returns the size of the blob content in bytes.
func (r *Reader) Size() int64 {
return r.r.Attributes().Size
}
// WriteTo reads from r and writes to w until there's no more data or
// an error occurs.
// The return value is the number of bytes written to w.
//
// It implements the io.WriterTo interface.
func (r *Reader) WriteTo(w io.Writer) (int64, error) {
// If the writer has a ReaderFrom method, use it to do the copy.
// Don't do this for our own *Writer to avoid infinite recursion.
// Avoids an allocation and a copy.
switch w.(type) {
case *Writer:
default:
if rf, ok := w.(io.ReaderFrom); ok {
n, err := rf.ReadFrom(r)
return n, err
}
}
_, nw, err := readFromWriteTo(r, w)
return nw, err
}
// readFromWriteTo is a helper for ReadFrom and WriteTo.
// It reads data from r and writes to w, until EOF or a read/write error.
// It returns the number of bytes read from r and the number of bytes
// written to w.
func readFromWriteTo(r io.Reader, w io.Writer) (int64, int64, error) {
// Note: can't use io.Copy because it will try to use r.WriteTo
// or w.WriteTo, which is recursive in this context.
buf := make([]byte, 1024)
var totalRead, totalWritten int64
for {
numRead, rerr := r.Read(buf)
if numRead > 0 {
totalRead += int64(numRead)
numWritten, werr := w.Write(buf[0:numRead])
totalWritten += int64(numWritten)
if werr != nil {
return totalRead, totalWritten, werr
}
}
if rerr == io.EOF {
// Done!
return totalRead, totalWritten, nil
}
if rerr != nil {
return totalRead, totalWritten, rerr
}
}
}

184
tools/filesystem/blob/writer.go Normal file
View file

@ -0,0 +1,184 @@
package blob
import (
"bytes"
"context"
"fmt"
"hash"
"io"
"net/http"
)
// Largely copied from gocloud.dev/blob.Writer to minimize breaking changes.
//
// -------------------------------------------------------------------
// Copyright 2019 The Go Cloud Development Kit Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// -------------------------------------------------------------------
var _ io.WriteCloser = (*Writer)(nil)
// Writer writes bytes to a blob.
//
// It implements io.WriteCloser (https://golang.org/pkg/io/#Closer), and must be
// closed after all writes are done.
type Writer struct {
drv Driver
w DriverWriter
key string
cancel func() // cancels the ctx provided to NewTypedWriter if contentMD5 verification fails
contentMD5 []byte
md5hash hash.Hash
bytesWritten int
closed bool
// These fields are non-zero values only when w is nil (not yet created).
//
// A ctx is stored in the Writer since we need to pass it into NewTypedWriter
// when we finish detecting the content type of the blob and create the
// underlying driver.Writer. This step happens inside Write or Close and
// neither of them take a context.Context as an argument.
//
// All 3 fields are only initialized when we create the Writer without
// setting the w field, and are reset to zero values after w is created.
ctx context.Context
opts *WriterOptions
buf *bytes.Buffer
}
// sniffLen is the byte size of Writer.buf used to detect content-type.
const sniffLen = 512
// Write implements the io.Writer interface (https://golang.org/pkg/io/#Writer).
//
// Writes may happen asynchronously, so the returned error can be nil
// even if the actual write eventually fails. The write is only guaranteed to
// have succeeded if Close returns no error.
func (w *Writer) Write(p []byte) (int, error) {
if len(w.contentMD5) > 0 {
if _, err := w.md5hash.Write(p); err != nil {
return 0, err
}
}
if w.w != nil {
return w.write(p)
}
// If w is not yet created due to no content-type being passed in, try to sniff
// the MIME type based on at most 512 bytes of the blob content of p.
// Detect the content-type directly if the first chunk is at least 512 bytes.
if w.buf.Len() == 0 && len(p) >= sniffLen {
return w.open(p)
}
// Store p in w.buf and detect the content-type when the size of content in
// w.buf is at least 512 bytes.
n, err := w.buf.Write(p)
if err != nil {
return 0, err
}
if w.buf.Len() >= sniffLen {
// Note that w.open will return the full length of the buffer; we don't want
// to return that as the length of this write since some of them were written in
// previous writes. Instead, we return the n from this write, above.
_, err := w.open(w.buf.Bytes())
return n, err
}
return n, nil
}
// Close closes the blob writer. The write operation is not guaranteed
// to have succeeded until Close returns with no error.
//
// Close may return an error if the context provided to create the
// Writer is canceled or reaches its deadline.
func (w *Writer) Close() (err error) {
w.closed = true
// Verify the MD5 hash of what was written matches the ContentMD5 provided by the user.
if len(w.contentMD5) > 0 {
md5sum := w.md5hash.Sum(nil)
if !bytes.Equal(md5sum, w.contentMD5) {
// No match! Return an error, but first cancel the context and call the
// driver's Close function to ensure the write is aborted.
w.cancel()
if w.w != nil {
_ = w.w.Close()
}
return fmt.Errorf("the WriterOptions.ContentMD5 you specified (%X) did not match what was written (%X)", w.contentMD5, md5sum)
}
}
defer w.cancel()
if w.w != nil {
return wrapError(w.drv, w.w.Close(), w.key)
}
if _, err := w.open(w.buf.Bytes()); err != nil {
return err
}
return wrapError(w.drv, w.w.Close(), w.key)
}
// open tries to detect the MIME type of p and write it to the blob.
// The error it returns is wrapped.
func (w *Writer) open(p []byte) (int, error) {
ct := http.DetectContentType(p)
var err error
w.w, err = w.drv.NewTypedWriter(w.ctx, w.key, ct, w.opts)
if err != nil {
return 0, wrapError(w.drv, err, w.key)
}
// Set the 3 fields needed for lazy NewTypedWriter back to zero values
// (see the comment on Writer).
w.buf = nil
w.ctx = nil
w.opts = nil
return w.write(p)
}
func (w *Writer) write(p []byte) (int, error) {
n, err := w.w.Write(p)
w.bytesWritten += n
return n, wrapError(w.drv, err, w.key)
}
// ReadFrom reads from r and writes to w until EOF or error.
// The return value is the number of bytes read from r.
//
// It implements the io.ReaderFrom interface.
func (w *Writer) ReadFrom(r io.Reader) (int64, error) {
// If the reader has a WriteTo method, use it to do the copy.
// Don't do this for our own *Reader to avoid infinite recursion.
// Avoids an allocation and a copy.
switch r.(type) {
case *Reader:
default:
if wt, ok := r.(io.WriterTo); ok {
return wt.WriteTo(w)
}
}
nr, _, err := readFromWriteTo(r, w)
return nr, err
}