From 321410c88be2c7513f43a37e8077938833749069 Mon Sep 17 00:00:00 2001 From: LinPr <314573849@qq.com> Date: Tue, 18 Feb 2025 11:59:46 +0800 Subject: [PATCH 1/4] feature: add extra virtual host style endpoints support --- command/app.go | 28 ++++++++++++++---------- storage/s3.go | 19 +++++++++++++--- storage/storage.go | 54 ++++++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/command/app.go b/command/app.go index f1699e43d..ccc1ff388 100644 --- a/command/app.go +++ b/command/app.go @@ -90,6 +90,11 @@ var app = &cli.App{ Name: "credentials-file", Usage: "use the specified credentials file instead of the default credentials file", }, + &cli.StringFlag{ + Name: "support-virtual-host-style-endpoints", + Usage: "add extra endpoints that support virtual host style requests, delimited by comma", + EnvVars: []string{"S3_SUPPORT_VIRTUAL_HOST_STYLE_ENDPOINTS"}, + }, }, Before: func(c *cli.Context) error { retryCount := c.Int("retry-count") @@ -179,17 +184,18 @@ var app = &cli.App{ // NewStorageOpts creates storage.Options object from the given context. func NewStorageOpts(c *cli.Context) storage.Options { return storage.Options{ - DryRun: c.Bool("dry-run"), - Endpoint: c.String("endpoint-url"), - MaxRetries: c.Int("retry-count"), - NoSignRequest: c.Bool("no-sign-request"), - NoVerifySSL: c.Bool("no-verify-ssl"), - RequestPayer: c.String("request-payer"), - UseListObjectsV1: c.Bool("use-list-objects-v1"), - Profile: c.String("profile"), - CredentialFile: c.String("credentials-file"), - LogLevel: log.LevelFromString(c.String("log")), - NoSuchUploadRetryCount: c.Int("no-such-upload-retry-count"), + DryRun: c.Bool("dry-run"), + Endpoint: c.String("endpoint-url"), + MaxRetries: c.Int("retry-count"), + NoSignRequest: c.Bool("no-sign-request"), + NoVerifySSL: c.Bool("no-verify-ssl"), + RequestPayer: c.String("request-payer"), + UseListObjectsV1: c.Bool("use-list-objects-v1"), + Profile: c.String("profile"), + CredentialFile: c.String("credentials-file"), + LogLevel: log.LevelFromString(c.String("log")), + NoSuchUploadRetryCount: c.Int("no-such-upload-retry-count"), + SupportVirtualHostStyleEndpoints: c.String("support-virtual-host-style-endpoints"), } } diff --git a/storage/s3.go b/storage/s3.go index 3313be66e..b6eb1276b 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -1255,7 +1255,7 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session. // use virtual-host-style if the endpoint is known to support it, // otherwise use the path-style approach. - isVirtualHostStyle := isVirtualHostStyle(endpointURL) + isVirtualHostStyle := isVirtualHostStyle(endpointURL, opts.SupportVirtualHostStyleEndpoints) useAccelerate := supportsTransferAcceleration(endpointURL) // AWS SDK handles transfer acceleration automatically. Setting the @@ -1422,11 +1422,24 @@ func IsGoogleEndpoint(endpoint urlpkg.URL) bool { return endpoint.Hostname() == gcsEndpoint } +func IsOtherVirtualHostStyleEndpoint(endpoint urlpkg.URL, supportVirtualHostStyleEndpoints string) bool { + supportVirtualHostStyleEndpointsSlice := strings.Split(supportVirtualHostStyleEndpoints, ",") + for _, supportVirtualHostStyleEndpoint := range supportVirtualHostStyleEndpointsSlice { + if endpoint.Hostname() == supportVirtualHostStyleEndpoint { + return true + } + } + return false +} + // isVirtualHostStyle reports whether the given endpoint supports S3 virtual // host style bucket name resolving. If a custom S3 API compatible endpoint is // given, resolve the bucketname from the URL path. -func isVirtualHostStyle(endpoint urlpkg.URL) bool { - return endpoint == sentinelURL || supportsTransferAcceleration(endpoint) || IsGoogleEndpoint(endpoint) +func isVirtualHostStyle(endpoint urlpkg.URL, supportVirtualHostStyleEndpoints string) bool { + return endpoint == sentinelURL || + supportsTransferAcceleration(endpoint) || + IsGoogleEndpoint(endpoint) || + IsOtherVirtualHostStyleEndpoint(endpoint, supportVirtualHostStyleEndpoints) } func errHasCode(err error, code string) bool { diff --git a/storage/storage.go b/storage/storage.go index 1a6171263..f5242daa6 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -58,19 +58,20 @@ func NewLocalClient(opts Options) *Filesystem { func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error) { newOpts := Options{ - MaxRetries: opts.MaxRetries, - NoSuchUploadRetryCount: opts.NoSuchUploadRetryCount, - Endpoint: opts.Endpoint, - NoVerifySSL: opts.NoVerifySSL, - DryRun: opts.DryRun, - NoSignRequest: opts.NoSignRequest, - UseListObjectsV1: opts.UseListObjectsV1, - RequestPayer: opts.RequestPayer, - Profile: opts.Profile, - CredentialFile: opts.CredentialFile, - LogLevel: opts.LogLevel, - bucket: url.Bucket, - region: opts.region, + MaxRetries: opts.MaxRetries, + NoSuchUploadRetryCount: opts.NoSuchUploadRetryCount, + Endpoint: opts.Endpoint, + NoVerifySSL: opts.NoVerifySSL, + DryRun: opts.DryRun, + NoSignRequest: opts.NoSignRequest, + UseListObjectsV1: opts.UseListObjectsV1, + RequestPayer: opts.RequestPayer, + Profile: opts.Profile, + CredentialFile: opts.CredentialFile, + LogLevel: opts.LogLevel, + bucket: url.Bucket, + region: opts.region, + SupportVirtualHostStyleEndpoints: opts.SupportVirtualHostStyleEndpoints, } return newS3Storage(ctx, newOpts) } @@ -84,19 +85,20 @@ func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error) // Options stores configuration for storage. type Options struct { - MaxRetries int - NoSuchUploadRetryCount int - Endpoint string - NoVerifySSL bool - DryRun bool - NoSignRequest bool - UseListObjectsV1 bool - LogLevel log.LogLevel - RequestPayer string - Profile string - CredentialFile string - bucket string - region string + MaxRetries int + NoSuchUploadRetryCount int + Endpoint string + NoVerifySSL bool + DryRun bool + NoSignRequest bool + UseListObjectsV1 bool + LogLevel log.LogLevel + RequestPayer string + Profile string + CredentialFile string + bucket string + region string + SupportVirtualHostStyleEndpoints string } func (o *Options) SetRegion(region string) { From 814e70a8e5821a77e573d326e547d73d2c46bde0 Mon Sep 17 00:00:00 2001 From: LinPr <314573849@qq.com> Date: Tue, 18 Feb 2025 17:52:32 +0800 Subject: [PATCH 2/4] feature: control path or virtual host style with addressing-style flag or S3_ADDRESSING_STYLE environment virable --- command/app.go | 36 ++++++++++++++++------------- main.go | 3 ++- storage/s3.go | 18 ++++++--------- storage/storage.go | 56 +++++++++++++++++++++++----------------------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/command/app.go b/command/app.go index ccc1ff388..6fa949363 100644 --- a/command/app.go +++ b/command/app.go @@ -90,10 +90,14 @@ var app = &cli.App{ Name: "credentials-file", Usage: "use the specified credentials file instead of the default credentials file", }, - &cli.StringFlag{ - Name: "support-virtual-host-style-endpoints", - Usage: "add extra endpoints that support virtual host style requests, delimited by comma", - EnvVars: []string{"S3_SUPPORT_VIRTUAL_HOST_STYLE_ENDPOINTS"}, + &cli.GenericFlag{ + Name: "addressing-style", + Usage: "use virtual host style or path style endpoint: (path, virtual)", + Value: &EnumValue{ + Enum: []string{"path", "virtual"}, + Default: "path", + }, + EnvVars: []string{"S3_ADDRESSING_STYLE"}, }, }, Before: func(c *cli.Context) error { @@ -184,18 +188,18 @@ var app = &cli.App{ // NewStorageOpts creates storage.Options object from the given context. func NewStorageOpts(c *cli.Context) storage.Options { return storage.Options{ - DryRun: c.Bool("dry-run"), - Endpoint: c.String("endpoint-url"), - MaxRetries: c.Int("retry-count"), - NoSignRequest: c.Bool("no-sign-request"), - NoVerifySSL: c.Bool("no-verify-ssl"), - RequestPayer: c.String("request-payer"), - UseListObjectsV1: c.Bool("use-list-objects-v1"), - Profile: c.String("profile"), - CredentialFile: c.String("credentials-file"), - LogLevel: log.LevelFromString(c.String("log")), - NoSuchUploadRetryCount: c.Int("no-such-upload-retry-count"), - SupportVirtualHostStyleEndpoints: c.String("support-virtual-host-style-endpoints"), + DryRun: c.Bool("dry-run"), + Endpoint: c.String("endpoint-url"), + MaxRetries: c.Int("retry-count"), + NoSignRequest: c.Bool("no-sign-request"), + NoVerifySSL: c.Bool("no-verify-ssl"), + RequestPayer: c.String("request-payer"), + UseListObjectsV1: c.Bool("use-list-objects-v1"), + Profile: c.String("profile"), + CredentialFile: c.String("credentials-file"), + LogLevel: log.LevelFromString(c.String("log")), + NoSuchUploadRetryCount: c.Int("no-such-upload-retry-count"), + AddressingStyle: c.String("addressing-style"), } } diff --git a/main.go b/main.go index 3d835cbcb..1482b21c9 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "log" "os" "os/signal" "syscall" @@ -12,8 +13,8 @@ import ( func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() - if err := command.Main(ctx, os.Args); err != nil { + log.Printf("error: %v", err) os.Exit(1) } } diff --git a/storage/s3.go b/storage/s3.go index b6eb1276b..b078f93d5 100644 --- a/storage/s3.go +++ b/storage/s3.go @@ -50,6 +50,8 @@ const ( // the key of the object metadata which is used to handle retry decision on NoSuchUpload error metadataKeyRetryID = "s5cmd-upload-retry-id" + + AddressingVirtualHostStyle = "virtual" ) // Re-used AWS sessions dramatically improve performance. @@ -1255,7 +1257,7 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session. // use virtual-host-style if the endpoint is known to support it, // otherwise use the path-style approach. - isVirtualHostStyle := isVirtualHostStyle(endpointURL, opts.SupportVirtualHostStyleEndpoints) + isVirtualHostStyle := isVirtualHostStyle(endpointURL, opts.AddressingStyle) useAccelerate := supportsTransferAcceleration(endpointURL) // AWS SDK handles transfer acceleration automatically. Setting the @@ -1422,24 +1424,18 @@ func IsGoogleEndpoint(endpoint urlpkg.URL) bool { return endpoint.Hostname() == gcsEndpoint } -func IsOtherVirtualHostStyleEndpoint(endpoint urlpkg.URL, supportVirtualHostStyleEndpoints string) bool { - supportVirtualHostStyleEndpointsSlice := strings.Split(supportVirtualHostStyleEndpoints, ",") - for _, supportVirtualHostStyleEndpoint := range supportVirtualHostStyleEndpointsSlice { - if endpoint.Hostname() == supportVirtualHostStyleEndpoint { - return true - } - } - return false +func ForcedVirtualHostStyle(endpoint urlpkg.URL, addressingStyle string) bool { + return addressingStyle == AddressingVirtualHostStyle } // isVirtualHostStyle reports whether the given endpoint supports S3 virtual // host style bucket name resolving. If a custom S3 API compatible endpoint is // given, resolve the bucketname from the URL path. -func isVirtualHostStyle(endpoint urlpkg.URL, supportVirtualHostStyleEndpoints string) bool { +func isVirtualHostStyle(endpoint urlpkg.URL, addressingStyle string) bool { return endpoint == sentinelURL || supportsTransferAcceleration(endpoint) || IsGoogleEndpoint(endpoint) || - IsOtherVirtualHostStyleEndpoint(endpoint, supportVirtualHostStyleEndpoints) + ForcedVirtualHostStyle(endpoint, addressingStyle) } func errHasCode(err error, code string) bool { diff --git a/storage/storage.go b/storage/storage.go index f5242daa6..4dbc38113 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -58,20 +58,20 @@ func NewLocalClient(opts Options) *Filesystem { func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error) { newOpts := Options{ - MaxRetries: opts.MaxRetries, - NoSuchUploadRetryCount: opts.NoSuchUploadRetryCount, - Endpoint: opts.Endpoint, - NoVerifySSL: opts.NoVerifySSL, - DryRun: opts.DryRun, - NoSignRequest: opts.NoSignRequest, - UseListObjectsV1: opts.UseListObjectsV1, - RequestPayer: opts.RequestPayer, - Profile: opts.Profile, - CredentialFile: opts.CredentialFile, - LogLevel: opts.LogLevel, - bucket: url.Bucket, - region: opts.region, - SupportVirtualHostStyleEndpoints: opts.SupportVirtualHostStyleEndpoints, + MaxRetries: opts.MaxRetries, + NoSuchUploadRetryCount: opts.NoSuchUploadRetryCount, + Endpoint: opts.Endpoint, + NoVerifySSL: opts.NoVerifySSL, + DryRun: opts.DryRun, + NoSignRequest: opts.NoSignRequest, + UseListObjectsV1: opts.UseListObjectsV1, + RequestPayer: opts.RequestPayer, + Profile: opts.Profile, + CredentialFile: opts.CredentialFile, + LogLevel: opts.LogLevel, + bucket: url.Bucket, + region: opts.region, + AddressingStyle: opts.AddressingStyle, } return newS3Storage(ctx, newOpts) } @@ -85,20 +85,20 @@ func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error) // Options stores configuration for storage. type Options struct { - MaxRetries int - NoSuchUploadRetryCount int - Endpoint string - NoVerifySSL bool - DryRun bool - NoSignRequest bool - UseListObjectsV1 bool - LogLevel log.LogLevel - RequestPayer string - Profile string - CredentialFile string - bucket string - region string - SupportVirtualHostStyleEndpoints string + MaxRetries int + NoSuchUploadRetryCount int + Endpoint string + NoVerifySSL bool + DryRun bool + NoSignRequest bool + UseListObjectsV1 bool + LogLevel log.LogLevel + RequestPayer string + Profile string + CredentialFile string + bucket string + region string + AddressingStyle string } func (o *Options) SetRegion(region string) { From fb75435c5b566efec2f0bb9d808b1358e665951b Mon Sep 17 00:00:00 2001 From: LinPr <314573849@qq.com> Date: Tue, 18 Feb 2025 18:14:57 +0800 Subject: [PATCH 3/4] feature: control path or virtual host style with addressing-style flag or S3_ADDRESSING_STYLE environment virable --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index 1482b21c9..f2cad4b7c 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,6 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() if err := command.Main(ctx, os.Args); err != nil { - log.Printf("error: %v", err) os.Exit(1) } } From 7e97a9a248bd332e0677065e2036fa732eb46426 Mon Sep 17 00:00:00 2001 From: LinPr <314573849@qq.com> Date: Tue, 18 Feb 2025 18:19:34 +0800 Subject: [PATCH 4/4] feature: control path or virtual host style with addressing-style flag or S3_ADDRESSING_STYLE environment virable --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index f2cad4b7c..495a3e5fe 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "log" "os" "os/signal" "syscall"