Fix: Improve retrieval of properties from LOI documents

functions/src/common/datastore.ts

@@ -31,7 +31,7 @@
  */
 type pseudoGeoJsonGeometry = {
   type: string;
   coordinates: any;
 };
 
 /**
@@ -165,6 +165,19 @@
       .get();
   }
 
+  fetchPartialLocationsOfInterest(
+    surveyId: string,
+    jobId: string,
+    limit: number
+  ) {
+    return this.db_
+      .collection(lois(surveyId))
+      .where(l.jobId, '==', jobId)
+      .orderBy(FieldPath.documentId())
+      .select('5', '9', '10')
+      .limit(limit);
+  }
+
   fetchMailTemplate(templateId: string) {
     return this.fetchDoc_(mailTemplate(templateId));
   }
@@ -238,7 +251,7 @@
     await loiRef.update({[l.properties]: loiDoc[l.properties]});
   }
 
   static toFirestoreMap(geometry: any) {
     return Object.fromEntries(
       Object.entries(geometry).map(([key, value]) => [
         key,
@@ -247,7 +260,7 @@
     );
   }
 
   static toFirestoreValue(value: any): any {
     if (value === null) {
       return null;
     }
@@ -275,7 +288,7 @@
    *
    * @returns GeoJSON geometry object (with geometry as list of lists)
    */
   static fromFirestoreMap(geoJsonGeometry: any): any {
     const geometryObject = geoJsonGeometry as pseudoGeoJsonGeometry;
     if (!geometryObject) {
       throw new Error(
@@ -290,7 +303,7 @@
     return geometryObject;
   }
 
   static fromFirestoreValue(coordinates: any) {
     if (coordinates instanceof GeoPoint) {
       // Note: GeoJSON coordinates are in lng-lat order.
       return [coordinates.longitude, coordinates.latitude];
@@ -299,7 +312,7 @@
     if (typeof coordinates !== 'object') {
       return coordinates;
     }
     const result = new Array<any>(coordinates.length);
 
     Object.entries(coordinates).map(([i, nestedValue]) => {
       const index = Number.parseInt(i);

functions/src/export-csv.ts

@@ -22,7 +22,7 @@ import {getDatastore} from './common/context';
 import * as HttpStatus from 'http-status-codes';
 import {DecodedIdToken} from 'firebase-admin/auth';
 import {List} from 'immutable';
-import {QuerySnapshot} from 'firebase-admin/firestore';
+// import {QuerySnapshot} from 'firebase-admin/firestore';
 import {timestampToInt, toMessage} from '@ground/lib';
 import {GroundProtos} from '@ground/proto';
 import {toGeoJsonGeometry} from '@ground/lib';
@@ -67,8 +67,44 @@ export async function exportCsvHandler(
   }
   const {name: jobName} = job;
   const tasks = job.tasks.sort((a, b) => a.index! - b.index!);
-  const snapshot = await db.fetchLocationsOfInterest(surveyId, jobId);
-  const loiProperties = createProperySetFromSnapshot(snapshot);
+
+  // let a = 0;
+  const loiProperties = new Set<string>();
+  let query = db.fetchPartialLocationsOfInterest(surveyId, jobId, 1000);
+  let lastVisible = null;
+  do {
+    const snapshot = await query.get();
+    if (snapshot.empty) break;
+    await Promise.all(
+      snapshot.docs.map(async doc => {
+        const loi = doc.data();
+        if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return;
+        Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key));
+      })
+    );
+    // snapshot.forEach(doc => {
+    //   // const loi = toMessage(doc.data(), Pb.LocationOfInterest);
+    //   const loi = doc.data() as {
+    //     5: string;
+    //     9: number;
+    //     10: {[key: string]: any};
+    //   };
+    //   // if (loi instanceof Error) return;
+    //   if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return;
+    //   // if (!isAccessibleLoi(loi, ownerId)) return;
+    //   // const properties = loi.properties;
+    //   // for (const key of Object.keys(properties || {})) {
+    //   //   loiProperties.add(key);
+    //   // }
+    //   Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key));
+    // });
+    // a += 1000;
+    // console.log('Processed: ', a);
+    lastVisible = snapshot.docs[snapshot.docs.length - 1];
+    query = query.startAfter(lastVisible);
+  } while (lastVisible);
+
+  // const loiProperties = createProperySetFromSnapshot(snapshot);
   const headers = getHeaders(tasks, loiProperties);
 
   res.type('text/csv');
@@ -277,22 +313,22 @@ function getFileName(jobName: string | null) {
   return `${fileBase}.csv`;
 }
 
-function createProperySetFromSnapshot(
-  snapshot: QuerySnapshot,
-  ownerId?: string
-): Set<string> {
-  const allKeys = new Set<string>();
-  snapshot.forEach(doc => {
-    const loi = toMessage(doc.data(), Pb.LocationOfInterest);
-    if (loi instanceof Error) return;
-    if (!isAccessibleLoi(loi, ownerId)) return;
-    const properties = loi.properties;
-    for (const key of Object.keys(properties || {})) {
-      allKeys.add(key);
-    }
-  });
-  return allKeys;
-}
+// function createProperySetFromSnapshot(
+//   snapshot: QuerySnapshot,
+//   ownerId?: string
+// ): Set<string> {
+//   const allKeys = new Set<string>();
+//   snapshot.forEach(doc => {
+//     const loi = toMessage(doc.data(), Pb.LocationOfInterest);
+//     if (loi instanceof Error) return;
+//     if (!isAccessibleLoi(loi, ownerId)) return;
+//     const properties = loi.properties;
+//     for (const key of Object.keys(properties || {})) {
+//       allKeys.add(key);
+//     }
+//   });
+//   return allKeys;
+// }
 
 function getPropertiesByName(
   loi: Pb.LocationOfInterest,