diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java index e8bf61bdfb..07ccfbbf09 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java @@ -126,6 +126,7 @@ public final class Messages { public static final String CONTROLLER_URL = "Controller URL: {0}"; public static final String PLATFORM = "Platform: {0}"; public static final String MAX_UPLOAD_SIZE = "Max upload size: {0}"; + public static final String OBJECTSTORE_REGIONS_ARE = "ObjectStore regions are: {0}"; public static final String MAX_MTA_DESCRIPTOR_SIZE = "Max mta descriptor size: {0}"; public static final String MAX_MANIFEST_SIZE = "Max manifest size is set to: {0}"; public static final String MAX_RESOURCE_FILE_SIZE = "Max resource file size is set to: {0}"; diff --git a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java index 10fe7f0273..1ff1292d9c 100644 --- a/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java +++ b/multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java @@ -45,6 +45,7 @@ public class ApplicationConfiguration { static final String CFG_PLATFORM = "PLATFORM"; // Mandatory static final String CFG_MAX_UPLOAD_SIZE = "MAX_UPLOAD_SIZE"; + static final String OBJECTSTORE_REGIONS = "OBJECTSTORE_REGIONS"; static final String CFG_MAX_MTA_DESCRIPTOR_SIZE = "MAX_MTA_DESCRIPTOR_SIZE"; static final String CFG_MAX_MANIFEST_SIZE = "DEFAULT_MAX_MANIFEST_SIZE"; static final String CFG_MAX_RESOURCE_FILE_SIZE = "DEFAULT_MAX_RESOURCE_FILE_SIZE"; @@ -210,6 +211,7 @@ public class ApplicationConfiguration { private Integer threadsForFileUploadToController; private Integer threadsForFileStorageUpload; private Boolean isHealthCheckEnabled; + private Set objectStoreRegions; public ApplicationConfiguration() { this(new Environment()); @@ -257,6 +259,7 @@ public void load() { getServiceHandlingMaxParallelThreads(); getAbortedOperationsTtlInSeconds(); getFilesAsyncUploadExecutorMaxThreads(); + getObjectStoreRegions(); } public Map getNotSensitiveVariables() { @@ -300,6 +303,13 @@ public Long getMaxUploadSize() { return maxUploadSize; } + public Set getObjectStoreRegions() { + if (objectStoreRegions == null) { + objectStoreRegions = getObjectStoreRegionsFromEnvironment(); + } + return objectStoreRegions; + } + public Long getMaxMtaDescriptorSize() { if (maxMtaDescriptorSize == null) { maxMtaDescriptorSize = getMaxMtaDescriptorSizeFromEnvironment(); @@ -666,6 +676,12 @@ private Long getMaxUploadSizeFromEnvironment() { return value; } + private Set getObjectStoreRegionsFromEnvironment() { + String value = environment.getString(OBJECTSTORE_REGIONS); + logEnvironmentVariable(OBJECTSTORE_REGIONS, Messages.OBJECTSTORE_REGIONS_ARE, value); + return value == null ? Set.of() : Set.of(value.split(",")); + } + private Long getMaxMtaDescriptorSizeFromEnvironment() { Long value = environment.getLong(CFG_MAX_MTA_DESCRIPTOR_SIZE, DEFAULT_MAX_MTA_DESCRIPTOR_SIZE); logEnvironmentVariable(CFG_MAX_MTA_DESCRIPTOR_SIZE, Messages.MAX_MTA_DESCRIPTOR_SIZE, value); diff --git a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java index df0bf421ef..e968387926 100644 --- a/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java +++ b/multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import io.jsonwebtoken.lang.Strings; import org.cloudfoundry.multiapps.common.test.TestUtil; import org.cloudfoundry.multiapps.common.util.JsonUtil; import org.cloudfoundry.multiapps.controller.core.Messages; @@ -488,6 +489,8 @@ void testIsOnStartFilesWithoutContentCleanerEnabledThroughEnvironmentWithEnv() { @Test void testLoad() { + Mockito.when(environment.getString(ApplicationConfiguration.OBJECTSTORE_REGIONS, Strings.EMPTY)) + .thenReturn(Strings.EMPTY); Map vcapApplication = injectFileInEnvironment(VCAP_APPLICATION, ApplicationConfiguration.CFG_VCAP_APPLICATION); configuration.load(); Assertions.assertEquals(vcapApplication.get("cf_api"), configuration.getControllerUrl() diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java index c7218f2dfe..e265d65f53 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/Constants.java @@ -29,9 +29,11 @@ private Constants() { public static final String BUCKET = "bucket"; public static final String REGION = "region"; public static final String ENDPOINT = "endpoint"; + public static final String ENDPOINT_URL = "endpoint-url"; public static final String ACCOUNT_NAME = "account_name"; public static final String SAS_TOKEN = "sas_token"; public static final String CONTAINER_NAME = "container_name"; + public static final String CONTAINER_NAME_WITH_DASH = "container-name"; public static final String CONTAINER_URI = "container_uri"; public static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; public static final String HOST = "host"; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java index 5a8d590c41..8994c622cf 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java @@ -1,5 +1,7 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import jakarta.inject.Inject; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.configuration.bean.factory.ObjectStoreFileStorageFactoryBean; import org.springframework.context.annotation.Bean; @@ -7,8 +9,6 @@ import org.springframework.web.multipart.MultipartResolver; import org.springframework.web.multipart.support.StandardServletMultipartResolver; -import jakarta.inject.Inject; - @Configuration public class FileStorageConfiguration { @@ -16,8 +16,9 @@ public class FileStorageConfiguration { @Inject @Bean - public ObjectStoreFileStorageFactoryBean objectStoreFileStorage(EnvironmentServicesFinder vcapServiceFinder) { - return new ObjectStoreFileStorageFactoryBean(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder); + public ObjectStoreFileStorageFactoryBean objectStoreFileStorage(EnvironmentServicesFinder vcapServiceFinder, + ApplicationConfiguration applicationConfiguration) { + return new ObjectStoreFileStorageFactoryBean(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration); } @Bean(name = "filterMultipartResolver") diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java index 410af21209..98f945979a 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBean.java @@ -5,9 +5,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import com.google.common.base.Joiner; import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; @@ -15,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfo; import org.cloudfoundry.multiapps.controller.web.configuration.service.ObjectStoreServiceInfoCreator; import org.jclouds.ContextBuilder; +import org.jclouds.aws.domain.Region; import org.jclouds.blobstore.BlobStoreContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,14 +31,19 @@ public class ObjectStoreFileStorageFactoryBean implements FactoryBean, InitializingBean { private static final Logger LOGGER = LoggerFactory.getLogger(ObjectStoreFileStorageFactoryBean.class); + private static final Set CUSTOM_REGIONS = Set.of("eu-south-1"); + private static final String JCLOUDS_REGIONS = "jclouds.regions"; private final String serviceName; private final EnvironmentServicesFinder environmentServicesFinder; + private final ApplicationConfiguration applicationConfiguration; private ObjectStoreFileStorage objectStoreFileStorage; - public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServicesFinder environmentServicesFinder) { + public ObjectStoreFileStorageFactoryBean(String serviceName, EnvironmentServicesFinder environmentServicesFinder, + ApplicationConfiguration applicationConfiguration) { this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; + this.applicationConfiguration = applicationConfiguration; } @Override @@ -77,7 +89,7 @@ private List getProvidersServiceInfo() { private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { ContextBuilder contextBuilder = ContextBuilder.newBuilder(serviceInfo.getProvider()); applyCredentials(serviceInfo, contextBuilder); - + addCustomRegions(contextBuilder); resolveContextEndpoint(serviceInfo, contextBuilder); BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class); @@ -88,6 +100,17 @@ private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) return context; } + private void addCustomRegions(ContextBuilder contextBuilder) { + Properties properties = new Properties(); + Set mergedRegions = Stream.of(CUSTOM_REGIONS, Region.DEFAULT_REGIONS, applicationConfiguration.getObjectStoreRegions()) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + properties.setProperty(JCLOUDS_REGIONS, Joiner.on(',') + .join(mergedRegions)); + + contextBuilder.overrides(properties); + } + private void applyCredentials(ObjectStoreServiceInfo serviceInfo, ContextBuilder contextBuilder) { if (serviceInfo.getCredentialsSupplier() != null) { contextBuilder.credentialsSupplier(serviceInfo.getCredentialsSupplier()); diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java index 8a5ae9a4ce..5b3f9cc4b1 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/service/ObjectStoreServiceInfoCreator.java @@ -1,12 +1,5 @@ package org.cloudfoundry.multiapps.controller.web.configuration.service; -import com.google.common.base.Supplier; -import io.pivotal.cfenv.core.CfService; -import org.cloudfoundry.multiapps.controller.web.Constants; -import org.cloudfoundry.multiapps.controller.web.Messages; -import org.jclouds.domain.Credentials; -import org.jclouds.googlecloud.GoogleCredentialsFromJson; - import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -14,13 +7,21 @@ import java.util.List; import java.util.Map; +import com.google.common.base.Supplier; +import io.pivotal.cfenv.core.CfService; +import org.cloudfoundry.multiapps.controller.web.Constants; +import org.cloudfoundry.multiapps.controller.web.Messages; +import org.jclouds.domain.Credentials; +import org.jclouds.googlecloud.GoogleCredentialsFromJson; + public class ObjectStoreServiceInfoCreator { public List getAllProvidersServiceInfo(CfService service) { Map credentials = service.getCredentials() .getMap(); - return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), createServiceInfoForAzure(credentials), - createServiceInfoForGcpCloud(credentials)); + return List.of(createServiceInfoForAws(credentials), createServiceInfoForAliCloud(credentials), + createServiceInfoForAzure(credentials), createServiceInfoForGcpCloud(credentials), + createServiceInfoForCcee(credentials)); } private ObjectStoreServiceInfo createServiceInfoForAws(Map credentials) { @@ -67,6 +68,22 @@ private ObjectStoreServiceInfo createServiceInfoForAzure(Map cre .build(); } + private ObjectStoreServiceInfo createServiceInfoForCcee(Map credentials) { + String accessKeyId = (String) credentials.get(Constants.ACCESS_KEY_ID); + String containerName = (String) credentials.get(Constants.CONTAINER_NAME_WITH_DASH); + String endpointUrl = (String) credentials.get(Constants.ENDPOINT_URL); + String region = (String) credentials.get(Constants.REGION); + String secretAccessKey = (String) credentials.get(Constants.SECRET_ACCESS_KEY); + return ImmutableObjectStoreServiceInfo.builder() + .provider(Constants.AWS_S_3) + .identity(accessKeyId) + .container(containerName) + .endpoint(endpointUrl) + .region(region) + .credential(secretAccessKey) + .build(); + } + private URL getContainerUriEndpoint(Map credentials) { if (!credentials.containsKey(Constants.CONTAINER_URI)) { return null; diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java index f6ac120ca2..f613f33452 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreFileStorageFactoryBeanTest.java @@ -1,15 +1,12 @@ package org.cloudfoundry.multiapps.controller.web.configuration.bean.factory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.when; - import java.util.HashMap; import java.util.Map; +import java.util.Set; +import io.pivotal.cfenv.core.CfCredentials; +import io.pivotal.cfenv.core.CfService; +import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.ObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.Constants; @@ -22,8 +19,12 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; -import io.pivotal.cfenv.core.CfCredentials; -import io.pivotal.cfenv.core.CfService; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; class ObjectStoreFileStorageFactoryBeanTest { @@ -36,13 +37,17 @@ class ObjectStoreFileStorageFactoryBeanTest { @Mock private EnvironmentServicesFinder environmentServicesFinder; @Mock + private ApplicationConfiguration applicationConfiguration; + @Mock private ObjectStoreFileStorage objectStoreFileStorage; @BeforeEach void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock("deploy-service-os", environmentServicesFinder); + when(applicationConfiguration.getObjectStoreRegions()).thenReturn(Set.of()); + objectStoreFileStorageFactoryBean = new ObjectStoreFileStorageFactoryBeanMock("deploy-service-os", environmentServicesFinder, + applicationConfiguration); } @Test @@ -87,8 +92,9 @@ private static Map buildCredentials() { private class ObjectStoreFileStorageFactoryBeanMock extends ObjectStoreFileStorageFactoryBean { - public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder) { - super(serviceName, environmentServicesFinder); + public ObjectStoreFileStorageFactoryBeanMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder, + ApplicationConfiguration applicationConfiguration) { + super(serviceName, environmentServicesFinder, applicationConfiguration); } @Override