Skip to content

feature: add extra virtual host style endpoints support #795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions command/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,15 @@ var app = &cli.App{
Name: "credentials-file",
Usage: "use the specified credentials file instead of the default credentials file",
},
&cli.GenericFlag{
Name: "addressing-style",
Usage: "use virtual host style or path style endpoint: (path, virtual)",
Value: &EnumValue{
Enum: []string{"path", "virtual"},
Default: "path",
},
EnvVars: []string{"S3_ADDRESSING_STYLE"},
},
},
Before: func(c *cli.Context) error {
retryCount := c.Int("retry-count")
Expand Down Expand Up @@ -190,6 +199,7 @@ func NewStorageOpts(c *cli.Context) storage.Options {
CredentialFile: c.String("credentials-file"),
LogLevel: log.LevelFromString(c.String("log")),
NoSuchUploadRetryCount: c.Int("no-such-upload-retry-count"),
AddressingStyle: c.String("addressing-style"),
}
}

Expand Down
1 change: 0 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer cancel()

if err := command.Main(ctx, os.Args); err != nil {
os.Exit(1)
}
Expand Down
15 changes: 12 additions & 3 deletions storage/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ const (

// the key of the object metadata which is used to handle retry decision on NoSuchUpload error
metadataKeyRetryID = "s5cmd-upload-retry-id"

AddressingVirtualHostStyle = "virtual"
)

// Re-used AWS sessions dramatically improve performance.
Expand Down Expand Up @@ -1255,7 +1257,7 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session.

// use virtual-host-style if the endpoint is known to support it,
// otherwise use the path-style approach.
isVirtualHostStyle := isVirtualHostStyle(endpointURL)
isVirtualHostStyle := isVirtualHostStyle(endpointURL, opts.AddressingStyle)

useAccelerate := supportsTransferAcceleration(endpointURL)
// AWS SDK handles transfer acceleration automatically. Setting the
Expand Down Expand Up @@ -1422,11 +1424,18 @@ func IsGoogleEndpoint(endpoint urlpkg.URL) bool {
return endpoint.Hostname() == gcsEndpoint
}

func ForcedVirtualHostStyle(endpoint urlpkg.URL, addressingStyle string) bool {
return addressingStyle == AddressingVirtualHostStyle
}

// isVirtualHostStyle reports whether the given endpoint supports S3 virtual
// host style bucket name resolving. If a custom S3 API compatible endpoint is
// given, resolve the bucketname from the URL path.
func isVirtualHostStyle(endpoint urlpkg.URL) bool {
return endpoint == sentinelURL || supportsTransferAcceleration(endpoint) || IsGoogleEndpoint(endpoint)
func isVirtualHostStyle(endpoint urlpkg.URL, addressingStyle string) bool {
return endpoint == sentinelURL ||
supportsTransferAcceleration(endpoint) ||
IsGoogleEndpoint(endpoint) ||
ForcedVirtualHostStyle(endpoint, addressingStyle)
}

func errHasCode(err error, code string) bool {
Expand Down
2 changes: 2 additions & 0 deletions storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro
LogLevel: opts.LogLevel,
bucket: url.Bucket,
region: opts.region,
AddressingStyle: opts.AddressingStyle,
}
return newS3Storage(ctx, newOpts)
}
Expand All @@ -97,6 +98,7 @@ type Options struct {
CredentialFile string
bucket string
region string
AddressingStyle string
}

func (o *Options) SetRegion(region string) {
Expand Down