diff --git a/cache-cli/pkg/storage/s3_list.go b/cache-cli/pkg/storage/s3_list.go index b0af5129..3bacbb8e 100644 --- a/cache-cli/pkg/storage/s3_list.go +++ b/cache-cli/pkg/storage/s3_list.go @@ -2,33 +2,35 @@ package storage import ( "context" - "fmt" "sort" "strings" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" ) func (s *S3Storage) List() ([]CacheKey, error) { - output, err := s.Client.ListObjects(context.TODO(), s.listObjectsInput(nil)) - if err != nil { - return nil, err - } - + prefix := s.Project + "/" + paginator := s3.NewListObjectsV2Paginator(s.Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.Bucket), + Prefix: aws.String(prefix), + }) keys := make([]CacheKey, 0) - keys = s.appendToListResult(keys, output.Contents) - - for output.IsTruncated { - nextMarker := findNextMarker(output) - output, err = s.Client.ListObjects(context.TODO(), s.listObjectsInput(&nextMarker)) + for paginator.HasMorePages() { + output, err := paginator.NextPage(context.TODO()) if err != nil { return nil, err } - - keys = s.appendToListResult(keys, output.Contents) + for _, object := range output.Contents { + name := strings.TrimPrefix(aws.ToString(object.Key), prefix) + keys = append(keys, CacheKey{ + Name: name, + StoredAt: object.LastModified, + LastAccessedAt: object.LastModified, + Size: object.Size, + }) + } } - return s.sortKeys(keys), nil } @@ -47,42 +49,3 @@ func (s *S3Storage) sortKeys(keys []CacheKey) []CacheKey { return keys } - -func (s *S3Storage) listObjectsInput(nextMarker *string) *s3.ListObjectsInput { - if nextMarker != nil { - return &s3.ListObjectsInput{ - Bucket: &s.Bucket, - Prefix: &s.Project, - Marker: nextMarker, - } - } - - return &s3.ListObjectsInput{ - Bucket: &s.Bucket, - Prefix: &s.Project, - } -} - -func (s *S3Storage) appendToListResult(keys []CacheKey, objects []types.Object) []CacheKey { - for _, object := range objects { - keyWithoutProject := strings.ReplaceAll(*object.Key, fmt.Sprintf("%s/", s.Project), "") - keys = append(keys, CacheKey{ - Name: keyWithoutProject, - StoredAt: object.LastModified, - LastAccessedAt: object.LastModified, - Size: object.Size, - }) - } - - return keys -} - -func findNextMarker(output *s3.ListObjectsOutput) string { - if output.NextMarker != nil { - return *output.NextMarker - } - - contents := output.Contents - lastElement := contents[len(contents)-1] - return *lastElement.Key -}