From 01ca149df768a5eb84eed7b620d9866ba5f8e40c Mon Sep 17 00:00:00 2001
From: Roberto Fontanarosa <robertofontanarosa@gmail.com>
Date: Fri, 18 Jul 2025 23:06:06 +0200
Subject: [PATCH] improved block to retrieve properties

---
 functions/src/common/datastore.ts | 13 ++++++
 functions/src/export-csv.ts       | 74 +++++++++++++++++++++++--------
 2 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/functions/src/common/datastore.ts b/functions/src/common/datastore.ts
index 9f4b75cc6..2abf86ba6 100644
--- a/functions/src/common/datastore.ts
+++ b/functions/src/common/datastore.ts
@@ -165,6 +165,19 @@ export class Datastore {
       .get();
   }
 
+  fetchPartialLocationsOfInterest(
+    surveyId: string,
+    jobId: string,
+    limit: number
+  ) {
+    return this.db_
+      .collection(lois(surveyId))
+      .where(l.jobId, '==', jobId)
+      .orderBy(FieldPath.documentId())
+      .select('5', '9', '10')
+      .limit(limit);
+  }
+
   fetchMailTemplate(templateId: string) {
     return this.fetchDoc_(mailTemplate(templateId));
   }
diff --git a/functions/src/export-csv.ts b/functions/src/export-csv.ts
index 28fc1385f..20298101a 100644
--- a/functions/src/export-csv.ts
+++ b/functions/src/export-csv.ts
@@ -22,7 +22,7 @@ import {getDatastore} from './common/context';
 import * as HttpStatus from 'http-status-codes';
 import {DecodedIdToken} from 'firebase-admin/auth';
 import {List} from 'immutable';
-import {QuerySnapshot} from 'firebase-admin/firestore';
+// import {QuerySnapshot} from 'firebase-admin/firestore';
 import {timestampToInt, toMessage} from '@ground/lib';
 import {GroundProtos} from '@ground/proto';
 import {toGeoJsonGeometry} from '@ground/lib';
@@ -67,8 +67,44 @@ export async function exportCsvHandler(
   }
   const {name: jobName} = job;
   const tasks = job.tasks.sort((a, b) => a.index! - b.index!);
-  const snapshot = await db.fetchLocationsOfInterest(surveyId, jobId);
-  const loiProperties = createProperySetFromSnapshot(snapshot);
+
+  // let a = 0;
+  const loiProperties = new Set<string>();
+  let query = db.fetchPartialLocationsOfInterest(surveyId, jobId, 1000);
+  let lastVisible = null;
+  do {
+    const snapshot = await query.get();
+    if (snapshot.empty) break;
+    await Promise.all(
+      snapshot.docs.map(async doc => {
+        const loi = doc.data();
+        if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return;
+        Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key));
+      })
+    );
+    // snapshot.forEach(doc => {
+    //   // const loi = toMessage(doc.data(), Pb.LocationOfInterest);
+    //   const loi = doc.data() as {
+    //     5: string;
+    //     9: number;
+    //     10: {[key: string]: any};
+    //   };
+    //   // if (loi instanceof Error) return;
+    //   if (ownerId ? loi[9] === 2 && loi[5] === ownerId : true) return;
+    //   // if (!isAccessibleLoi(loi, ownerId)) return;
+    //   // const properties = loi.properties;
+    //   // for (const key of Object.keys(properties || {})) {
+    //   //   loiProperties.add(key);
+    //   // }
+    //   Object.keys(loi[10] || {}).forEach(key => loiProperties.add(key));
+    // });
+    // a += 1000;
+    // console.log('Processed: ', a);
+    lastVisible = snapshot.docs[snapshot.docs.length - 1];
+    query = query.startAfter(lastVisible);
+  } while (lastVisible);
+
+  // const loiProperties = createProperySetFromSnapshot(snapshot);
   const headers = getHeaders(tasks, loiProperties);
 
   res.type('text/csv');
@@ -277,22 +313,22 @@ function getFileName(jobName: string | null) {
   return `${fileBase}.csv`;
 }
 
-function createProperySetFromSnapshot(
-  snapshot: QuerySnapshot,
-  ownerId?: string
-): Set<string> {
-  const allKeys = new Set<string>();
-  snapshot.forEach(doc => {
-    const loi = toMessage(doc.data(), Pb.LocationOfInterest);
-    if (loi instanceof Error) return;
-    if (!isAccessibleLoi(loi, ownerId)) return;
-    const properties = loi.properties;
-    for (const key of Object.keys(properties || {})) {
-      allKeys.add(key);
-    }
-  });
-  return allKeys;
-}
+// function createProperySetFromSnapshot(
+//   snapshot: QuerySnapshot,
+//   ownerId?: string
+// ): Set<string> {
+//   const allKeys = new Set<string>();
+//   snapshot.forEach(doc => {
+//     const loi = toMessage(doc.data(), Pb.LocationOfInterest);
+//     if (loi instanceof Error) return;
+//     if (!isAccessibleLoi(loi, ownerId)) return;
+//     const properties = loi.properties;
+//     for (const key of Object.keys(properties || {})) {
+//       allKeys.add(key);
+//     }
+//   });
+//   return allKeys;
+// }
 
 function getPropertiesByName(
   loi: Pb.LocationOfInterest,