Adding upstream version 0.28.1.

Signed-off-by: Daniel Baumann <daniel@debian.org>

tools/filesystem/internal/s3blob/s3/s3.go

@@ -0,0 +1,370 @@
+// Package s3 implements a lightweight client for interacting with the
+// REST APIs of any S3 compatible service.
+//
+// It implements only the minimal functionality required by PocketBase
+// such as objects list, get, copy, delete and upload.
+//
+// For more details why we don't use the official aws-sdk-go-v2, you could check
+// https://github.com/pocketbase/pocketbase/discussions/6562.
+//
+// Example:
+//
+//	client := &s3.S3{
+//		Endpoint:     "example.com",
+//		Region:       "us-east-1",
+//		Bucket:       "test",
+//		AccessKey:    "...",
+//		SecretKey:    "...",
+//		UsePathStyle: true,
+//	}
+//	resp, err := client.GetObject(context.Background(), "abc.txt")
+package s3
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+)
+
+const (
+	awsS3ServiceCode     = "s3"
+	awsSignAlgorithm     = "AWS4-HMAC-SHA256"
+	awsTerminationString = "aws4_request"
+	metadataPrefix       = "x-amz-meta-"
+	dateTimeFormat       = "20060102T150405Z"
+)
+
+type HTTPClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+type S3 struct {
+	// Client specifies a custom HTTP client to send the request with.
+	//
+	// If not explicitly set, fallbacks to http.DefaultClient.
+	Client HTTPClient
+
+	Bucket       string
+	Region       string
+	Endpoint     string // can be with or without the schema
+	AccessKey    string
+	SecretKey    string
+	UsePathStyle bool
+}
+
+// URL constructs an S3 request URL based on the current configuration.
+func (s3 *S3) URL(path string) string {
+	scheme := "https"
+	endpoint := strings.TrimRight(s3.Endpoint, "/")
+	if after, ok := strings.CutPrefix(endpoint, "https://"); ok {
+		endpoint = after
+	} else if after, ok := strings.CutPrefix(endpoint, "http://"); ok {
+		endpoint = after
+		scheme = "http"
+	}
+
+	path = strings.TrimLeft(path, "/")
+
+	if s3.UsePathStyle {
+		return fmt.Sprintf("%s://%s/%s/%s", scheme, endpoint, s3.Bucket, path)
+	}
+
+	return fmt.Sprintf("%s://%s.%s/%s", scheme, s3.Bucket, endpoint, path)
+}
+
+// SignAndSend signs the provided request per AWS Signature v4 and sends it.
+//
+// It automatically normalizes all 40x/50x responses to ResponseError.
+//
+// Note: Don't forget to call resp.Body.Close() after done with the result.
+func (s3 *S3) SignAndSend(req *http.Request) (*http.Response, error) {
+	s3.sign(req)
+
+	client := s3.Client
+	if client == nil {
+		client = http.DefaultClient
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 {
+		defer resp.Body.Close()
+
+		respErr := &ResponseError{
+			Status: resp.StatusCode,
+		}
+
+		respErr.Raw, err = io.ReadAll(resp.Body)
+		if err != nil && !errors.Is(err, io.EOF) {
+			return nil, errors.Join(err, respErr)
+		}
+
+		if len(respErr.Raw) > 0 {
+			err = xml.Unmarshal(respErr.Raw, respErr)
+			if err != nil {
+				return nil, errors.Join(err, respErr)
+			}
+		}
+
+		return nil, respErr
+	}
+
+	return resp, nil
+}
+
+// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-signed-request-steps
+func (s3 *S3) sign(req *http.Request) {
+	// fallback to the Unsigned payload option
+	// (data integrity checks could be still applied via the content-md5 or x-amz-checksum-* headers)
+	if req.Header.Get("x-amz-content-sha256") == "" {
+		req.Header.Set("x-amz-content-sha256", "UNSIGNED-PAYLOAD")
+	}
+
+	reqDateTime, _ := time.Parse(dateTimeFormat, req.Header.Get("x-amz-date"))
+	if reqDateTime.IsZero() {
+		reqDateTime = time.Now().UTC()
+		req.Header.Set("x-amz-date", reqDateTime.Format(dateTimeFormat))
+	}
+
+	req.Header.Set("host", req.URL.Host)
+
+	date := reqDateTime.Format("20060102")
+
+	dateTime := reqDateTime.Format(dateTimeFormat)
+
+	// 1. Create canonical request
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-canonical-request
+	// ---------------------------------------------------------------
+	canonicalHeaders, signedHeaders := canonicalAndSignedHeaders(req)
+
+	canonicalParts := []string{
+		req.Method,
+		escapePath(req.URL.Path),
+		escapeQuery(req.URL.Query()),
+		canonicalHeaders,
+		signedHeaders,
+		req.Header.Get("x-amz-content-sha256"),
+	}
+
+	// 2. Create a hash of the canonical request
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-canonical-request-hash
+	// ---------------------------------------------------------------
+	hashedCanonicalRequest := sha256Hex([]byte(strings.Join(canonicalParts, "\n")))
+
+	// 3. Create a string to sign
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#create-string-to-sign
+	// ---------------------------------------------------------------
+	scope := strings.Join([]string{
+		date,
+		s3.Region,
+		awsS3ServiceCode,
+		awsTerminationString,
+	}, "/")
+
+	stringToSign := strings.Join([]string{
+		awsSignAlgorithm,
+		dateTime,
+		scope,
+		hashedCanonicalRequest,
+	}, "\n")
+
+	// 4. Derive a signing key for SigV4
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#derive-signing-key
+	// ---------------------------------------------------------------
+	dateKey := hmacSHA256([]byte("AWS4"+s3.SecretKey), date)
+	dateRegionKey := hmacSHA256(dateKey, s3.Region)
+	dateRegionServiceKey := hmacSHA256(dateRegionKey, awsS3ServiceCode)
+	signingKey := hmacSHA256(dateRegionServiceKey, awsTerminationString)
+	signature := hex.EncodeToString(hmacSHA256(signingKey, stringToSign))
+
+	// 5. Add the signature to the request
+	// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html#add-signature-to-request
+	authorization := fmt.Sprintf(
+		"%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+		awsSignAlgorithm,
+		s3.AccessKey,
+		scope,
+		signedHeaders,
+		signature,
+	)
+
+	req.Header.Set("authorization", authorization)
+}
+
+func sha256Hex(content []byte) string {
+	h := sha256.New()
+	h.Write(content)
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+func hmacSHA256(key []byte, content string) []byte {
+	mac := hmac.New(sha256.New, key)
+	mac.Write([]byte(content))
+	return mac.Sum(nil)
+}
+
+func canonicalAndSignedHeaders(req *http.Request) (string, string) {
+	signed := []string{}
+	canonical := map[string]string{}
+
+	for key, values := range req.Header {
+		normalizedKey := strings.ToLower(key)
+
+		if normalizedKey != "host" &&
+			normalizedKey != "content-type" &&
+			!strings.HasPrefix(normalizedKey, "x-amz-") {
+			continue
+		}
+
+		signed = append(signed, normalizedKey)
+
+		// for each value:
+		// trim any leading or trailing spaces
+		// convert sequential spaces to a single space
+		normalizedValues := make([]string, len(values))
+		for i, v := range values {
+			normalizedValues[i] = strings.ReplaceAll(strings.TrimSpace(v), "  ", " ")
+		}
+
+		canonical[normalizedKey] = strings.Join(normalizedValues, ",")
+	}
+
+	slices.Sort(signed)
+
+	var sortedCanonical strings.Builder
+	for _, key := range signed {
+		sortedCanonical.WriteString(key)
+		sortedCanonical.WriteString(":")
+		sortedCanonical.WriteString(canonical[key])
+		sortedCanonical.WriteString("\n")
+	}
+
+	return sortedCanonical.String(), strings.Join(signed, ";")
+}
+
+// extractMetadata parses and extracts and the metadata from the specified request headers.
+//
+// The metadata keys are all lowercased and without the "x-amz-meta-" prefix.
+func extractMetadata(headers http.Header) map[string]string {
+	result := map[string]string{}
+
+	for k, v := range headers {
+		if len(v) == 0 {
+			continue
+		}
+
+		metadataKey, ok := strings.CutPrefix(strings.ToLower(k), metadataPrefix)
+		if !ok {
+			continue
+		}
+
+		result[metadataKey] = v[0]
+	}
+
+	return result
+}
+
+// escapeQuery returns the URI encoded request query parameters according to the AWS S3 spec requirements
+// (it is similar to url.Values.Encode but instead of url.QueryEscape uses our own escape method).
+func escapeQuery(values url.Values) string {
+	if len(values) == 0 {
+		return ""
+	}
+
+	var buf strings.Builder
+
+	keys := make([]string, 0, len(values))
+	for k := range values {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+
+	for _, k := range keys {
+		vs := values[k]
+		keyEscaped := escape(k)
+		for _, values := range vs {
+			if buf.Len() > 0 {
+				buf.WriteByte('&')
+			}
+			buf.WriteString(keyEscaped)
+			buf.WriteByte('=')
+			buf.WriteString(escape(values))
+		}
+	}
+
+	return buf.String()
+}
+
+// escapePath returns the URI encoded request path according to the AWS S3 spec requirements.
+func escapePath(path string) string {
+	parts := strings.Split(path, "/")
+
+	for i, part := range parts {
+		parts[i] = escape(part)
+	}
+
+	return strings.Join(parts, "/")
+}
+
+const upperhex = "0123456789ABCDEF"
+
+// escape is similar to the std url.escape but implements the AWS [UriEncode requirements]:
+//   - URI encode every byte except the unreserved characters: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~'.
+//   - The space character is a reserved character and must be encoded as "%20" (and not as "+").
+//   - Each URI encoded byte is formed by a '%' and the two-digit hexadecimal value of the byte.
+//   - Letters in the hexadecimal value must be uppercase, for example "%1A".
+//
+// [UriEncode requirements]: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html
+func escape(s string) string {
+	hexCount := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			hexCount++
+		}
+	}
+
+	if hexCount == 0 {
+		return s
+	}
+
+	result := make([]byte, len(s)+2*hexCount)
+
+	j := 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c) {
+			result[j] = '%'
+			result[j+1] = upperhex[c>>4]
+			result[j+2] = upperhex[c&15]
+			j += 3
+		} else {
+			result[j] = c
+			j++
+		}
+	}
+
+	return string(result)
+}
+
+// > "URI encode every byte except the unreserved characters: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~'."
+func shouldEscape(c byte) bool {
+	isUnreserved := (c >= 'A' && c <= 'Z') ||
+		(c >= 'a' && c <= 'z') ||
+		(c >= '0' && c <= '9') ||
+		c == '-' || c == '.' || c == '_' || c == '~'
+
+	return !isUnreserved
+}