Merge pull request #352 from ibi-group/dev

Release

Author: landonreed

.travis.yml

@@ -36,8 +36,8 @@ before_script:
   # Create dir for GTFS+ files (used during testing)
   - mkdir /tmp/gtfsplus
 script:
-  # run mvn package to make sure unit tests and packaging can work
-  - mvn package
+  # run mvn package (only print errors) to make sure unit tests and packaging can work
+  - mvn -q package
   # Restart/clear MongoDB so that E2E tests run on clean DB.
   - ./scripts/restart-mongo-with-fresh-db.sh
   # recursively copy coverage results into another folder so the e2e tests don't overwrite them
@@ -89,7 +89,13 @@ before_deploy:
 - ls target/*.jar
   # Copy packaged jar over to deploy dir.
 - cp target/dt-*.jar deploy/
-- cp target/dt-*.jar "deploy/dt-latest-$BRANCH_CLEAN.jar"
+  # Get the first jar file and copy it into a new file that adds the current branch name. During a
+  # merge to master, there are multiple jar files produced, but they're each effectively the same
+  # code (there may be slight differences in the version shown in the `pom.xml`, but that's not
+  # important for the purposes of creating this "latest branch" jar).
+- ALL_JARS=(target/dt-*.jar)
+- FIRST_JAR="${ALL_JARS[0]}"
+- cp "$FIRST_JAR" "deploy/dt-latest-$BRANCH_CLEAN.jar"
 deploy:
   provider: s3
   skip_cleanup: true

configurations/test/env.yml.tmp

@@ -15,5 +15,12 @@ SPARKPOST_EMAIL: [email protected]
 GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue
 # GTFS_DATABASE_USER:
 # GTFS_DATABASE_PASSWORD:
-#MONGO_URI: mongodb://mongo-host:27017
+
+# To configure a remote MongoDB service (such as MongoDB Atlas), provide all
+# Mongo properties below. Otherwise, only a database name is needed (server
+# defaults to mongodb://localhost:27017 with no username/password authentication).
 MONGO_DB_NAME: catalogue
+#MONGO_HOST: cluster1.mongodb.net
+#MONGO_PASSWORD: password
+#MONGO_PROTOCOL: mongodb+srv
+#MONGO_USER: user

pom.xml

@@ -239,18 +239,18 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.12</version>
+            <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
 
-        <!-- Used for loading/fetching/writing GTFS entities. gtfs-lib also provides access to:
+        <!-- Used for loading/fetching/validating/writing GTFS entities. gtfs-lib also provides access to:
          - commons-io - generic utilities
          - AWS S3 SDK - putting/getting objects into/out of S3.
          -->
         <dependency>
             <groupId>com.conveyal</groupId>
             <artifactId>gtfs-lib</artifactId>
-            <version>6.0.3</version>
+            <version>6.1.0</version>
             <!-- Exclusions added in order to silence SLF4J warnings about multiple bindings:
                 http://www.slf4j.org/codes.html#multiple_bindings
             -->
@@ -262,11 +262,11 @@
             </exclusions>
         </dependency>
 
-        <!-- Used for data-tools application database -->
+        <!-- Used for application database -->
         <dependency>
             <groupId>org.mongodb</groupId>
-            <artifactId>mongodb-driver</artifactId>
-            <version>3.5.0</version>
+            <artifactId>mongodb-driver-sync</artifactId>
+            <version>4.0.5</version>
         </dependency>
 
         <!-- Miscellaneous utilities -->
@@ -334,6 +334,13 @@
             <artifactId>gt-api</artifactId>
             <version>${geotools.version}</version>
         </dependency>
+        <!-- gt-epsg-hsql includes coordinate reference libraries that were removed from gtfs-lib in
+             https://github.com/conveyal/gtfs-lib/pull/290 -->
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-epsg-hsql</artifactId>
+            <version>${geotools.version}</version>
+        </dependency>
 
         <!-- Error reporting -->
         <dependency>

src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java

@@ -45,6 +45,7 @@ public abstract class MonitorableJob implements Runnable, Serializable {
 
     public enum JobType {
         UNKNOWN_TYPE,
+        ARBITRARY_FEED_TRANSFORM,
         BUILD_TRANSPORT_NETWORK,
         CREATE_FEEDVERSION_FROM_SNAPSHOT,
         // **** Legacy snapshot jobs
@@ -65,7 +66,8 @@ public enum JobType {
         CONVERT_EDITOR_MAPDB_TO_SQL,
         VALIDATE_ALL_FEEDS,
         MONITOR_SERVER_STATUS,
-        MERGE_FEED_VERSIONS
+        MERGE_FEED_VERSIONS,
+        RECREATE_BUILD_IMAGE
     }
 
     public MonitorableJob(Auth0UserProfile owner, String name, JobType type) {
@@ -216,6 +218,11 @@ public void addNextJob(MonitorableJob ...jobs) {
         }
     }
 
+    /** Convenience wrapper for a {@link List} of jobs. */
+    public void addNextJob(List<MonitorableJob> jobs) {
+        for (MonitorableJob job : jobs) addNextJob(job);
+    }
+
     /**
      * Represents the current status of this job.
      */

src/main/java/com/conveyal/datatools/common/utils/AWSUtils.java

@@ -0,0 +1,29 @@
+package com.conveyal.datatools.common.utils;
+
+/**
+ * A class that holds another variable and keeps track of whether the variable is still considered to be active (ie not
+ * expired)
+ */
+public class ExpiringAsset<T> {
+    public final T asset;
+    private final long expirationTimeMillis;
+
+    public ExpiringAsset(T asset, long validDurationMillis) {
+        this.asset = asset;
+        this.expirationTimeMillis = System.currentTimeMillis() + validDurationMillis;
+    }
+
+    /**
+     * @return true if the asset hasn't yet expired
+     */
+    public boolean isActive() {
+        return expirationTimeMillis > System.currentTimeMillis();
+    }
+
+    /**
+     * @return the amount of time that the asset is still valid for in milliseconds.
+     */
+    public long timeRemainingMillis() {
+        return expirationTimeMillis - System.currentTimeMillis();
+    }
+}