Skip to content

Parallel execution of maven build #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Parallel execution of maven build #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions docker-java-orchestration-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,22 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>

<parent>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration</artifactId>
<version>2.11.30-SNAPSHOT</version>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand Down Expand Up @@ -42,12 +54,12 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration-model</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration-plugin-api</artifactId>
<version>${project.version}</version>
</dependency>
Expand Down
93 changes: 82 additions & 11 deletions ...stration-core/src/main/java/com/alexecollins/docker/orchestration/DockerOrchestrator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import java.nio.file.Files;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.*;
import java.util.zip.GZIPOutputStream;

import static java.util.Arrays.asList;
Expand Down Expand Up @@ -71,6 +71,7 @@ public boolean accept(File pathname) {
private final DefinitionFilter definitionFilter;
private final boolean permissionErrorTolerant;

private Set<String> startedContainer = new HashSet<>();
/**
* @deprecated Please use builder from now on.
*/
Expand Down Expand Up @@ -434,20 +435,36 @@ private boolean haveBuildFlag(BuildFlag flag) {
return buildFlags.contains(flag);
}

private void start(final Id id) {
if (id == null) {
throw new IllegalArgumentException("id is null");
}

logger.info("Starting " + id);

private void build(final Id id, CountDownLatch countDownLatch){
try {
if (!imageExists(id)) {
logger.info("Image does not exist, so building it");
build(id);
}
} catch (DockerException e) {
throw new OrchestrationException(e);
}finally {
countDownLatch.countDown();
}

}

private void start(final Id id, CountDownLatch countDownLatch) {
if (id == null) {
countDownLatch.countDown();
startedContainer.add(id.toString());
throw new IllegalArgumentException("id is null");
}

logger.info("Starting " + id);

while (!isAllContainerStarted(id)) {
logger.info("Sleeping for 1 second for dependent container to start");
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}

try {
Expand Down Expand Up @@ -475,15 +492,15 @@ private void start(final Id id) {
for (Plugin plugin : plugins) {
plugin.started(id, conf);
}

sleep(id);

healthCheck(id);

} catch (Exception e) {
logger.error("Error starting container with id " + id + ": " + e.getMessage());
throw new OrchestrationException(e);
} finally {
startedContainer.add(id.toString());
countDownLatch.countDown();
final Container container = findContainer(id);
if (container == null) {
logger.error("Could not find container with id {}. No logs can be obtained", id);
Expand Down Expand Up @@ -872,13 +889,67 @@ public void validate() {
throw new OrchestrationException(innerException);
}

/*
To start without new thread creation
*/
public void startAsync() {
for (Id id : ids()) {
if (!inclusive(id)) {
continue;
}
build(id, new CountDownLatch(1));
start(id, new CountDownLatch(1));
}
}

public void start() {
this.start(1);
}

public void start(int threadCount) {
ExecutorService executorService = Executors.newFixedThreadPool(threadCount);
CountDownLatch countDownLatch = new CountDownLatch(ids().size());
for (Id id : ids()) {
if (!inclusive(id)) {
countDownLatch.countDown();
continue;
}
start(id);
Runnable task = ()-> build(id, countDownLatch);
executorService.submit(task);
}
try {
countDownLatch.await();
} catch (InterruptedException e) {
logger.error("InterruptedException occurred", e);
}

CountDownLatch countDownLatch1 = new CountDownLatch(ids().size());

for (Id id : ids()) {
if (!inclusive(id)) {
countDownLatch1.countDown();
continue;
}
Runnable task = ()-> start(id, countDownLatch1);
executorService.submit(task);
}

try {
countDownLatch1.await();
} catch (InterruptedException e) {
logger.error("InterruptedException occurred", e);
}
}

private boolean isAllContainerStarted(Id id) {
Conf conf = conf(id);
List<Depend> depends = conf.getDepends();
for (Depend depend : depends) {
if (!startedContainer.contains(depend.getId().toString())) {
return false;
}
}
return true;
}

public void copy(String resource, String hostpath) {
Expand Down
4 changes: 2 additions & 2 deletions ...tion-core/src/test/java/com/alexecollins/docker/orchestration/DockerOrchestratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ public void testWaitForLineFailEndOfInput() {
mockLogContainerCmd("Bar");

try {
testObj.start();
testObj.startAsync();
fail();
} catch (OrchestrationException e) {
assertThat(e.getMessage(), endsWith(String.format("%s's log ended before [\"^Foo$\"] appeared in output", idMock)));
Expand Down Expand Up @@ -621,7 +621,7 @@ public void timeOutWaitingForLogs() {
mockLogContainerCmd("Bar");

try {
testObj.start();
testObj.startAsync();
fail();
} catch (OrchestrationException e) {
assertThat(e.getMessage(), endsWith(String.format("timeout after 0 while waiting for \"%s\" in %s's logs", firstLogPattern.getPattern(), idMock)));
Expand Down
4 changes: 2 additions & 2 deletions docker-java-orchestration-model/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>docker-java-orchestration</artifactId>
<groupId>com.alexecollins.docker</groupId>
<version>2.11.30-SNAPSHOT</version>
<groupId>com.alexecollins.kghimire.docker</groupId>
<version>1.0.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
1 change: 1 addition & 0 deletions ...a-orchestration-model/src/main/java/com/alexecollins/docker/orchestration/model/Conf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class Conf {
*/
private ContainerConf container = new ContainerConf();
private List<Link> links = new ArrayList<>();
private List<Depend> depends = new ArrayList<>();
private Packaging packaging = new Packaging();
/**
* E.g. "8080" or "8080 8080" where the former is the exposed port and the latter the container port.
Expand Down
38 changes: 38 additions & 0 deletions ...orchestration-model/src/main/java/com/alexecollins/docker/orchestration/model/Depend.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package com.alexecollins.docker.orchestration.model;

public class Depend {
private final String value;

public Depend(String value) {
this.value = value;
}

public Id getId() {
return new Id(value.replaceFirst(":.*", ""));
}

public String getAlias() {
return value.contains(":") ? value.replaceFirst(".*:", "") : value;
}

@Override
public String toString() {
return value;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

Depend id = (Depend) o;

return value.equals(id.value);

}

@Override
public int hashCode() {
return value.hashCode();
}
}
6 changes: 3 additions & 3 deletions docker-java-orchestration-plugin-api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration</artifactId>
<version>2.11.30-SNAPSHOT</version>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand All @@ -14,7 +14,7 @@

<dependencies>
<dependency>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration-model</artifactId>
<version>${project.version}</version>
</dependency>
Expand Down
6 changes: 3 additions & 3 deletions docker-java-orchestration-plugin-boot2docker/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration</artifactId>
<version>2.11.30-SNAPSHOT</version>
<version>1.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>

Expand All @@ -14,7 +14,7 @@

<dependencies>
<dependency>
<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration-plugin-api</artifactId>
<version>${project.version}</version>
</dependency>
Expand Down
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
<version>7</version>
</parent>

<groupId>com.alexecollins.docker</groupId>
<groupId>com.alexecollins.kghimire.docker</groupId>
<artifactId>docker-java-orchestration</artifactId>
<version>2.11.30-SNAPSHOT</version>
<version>1.0.0</version>
<packaging>pom</packaging>

<name>Docker Java Orchestration</name>
Expand Down