feat(lba-2522): algo data export (#2358)

* feat: export data for algo

* feat: optimize

Author: kevbarns

server/src/jobs/jobs.ts

@@ -6,6 +6,7 @@ import updateDomainesMetiers from "@/jobs/domainesMetiers/updateDomainesMetiers"
 import { create as createMigration, status as statusMigration, up as upMigration } from "@/jobs/migrations/migrations"
 import { sendMiseEnRelation } from "@/jobs/miseEnRelation/sendMiseEnRelation"
 import { importers } from "@/jobs/offrePartenaire/jobsPartners.importer"
+import { exportFileForAlgo } from "@/jobs/partenaireExport/exportBlacklistAlgo"
 import { exportJobsToS3V2 } from "@/jobs/partenaireExport/exportJobsToS3V2"
 import { updateReferentielCommune } from "@/services/referentiel/commune/commune.referentiel.service"
 import { generateSitemap } from "@/services/sitemap.service"
@@ -183,6 +184,10 @@ export async function setupJobProcessor() {
             handler: syncEtablissementsAndFormations,
             tag: "main",
           },
+          "Export des offres sur S3 v2": {
+            cron_string: "0 3 * * *",
+            handler: () => exportJobsToS3V2(),
+          },
           "Supprime les etablissements dupliqués à cause du parallélisme du job de synchronisation RDVA": {
             cron_string: "30 3 * * *",
             handler: removeDuplicateEtablissements,
@@ -273,19 +278,18 @@ export async function setupJobProcessor() {
             handler: resetInvitationDates,
             tag: "main",
           },
+          "Export des données pour l'algorithme": {
+            cron_string: "0 10 * * FRI",
+            handler: exportFileForAlgo,
+          },
           "Traitement des recruteur LBA par la pipeline jobs partners": {
             cron_string: "0 10 * * SUN",
             handler: () => processRecruteursLba(),
             tag: "main",
           },
-          "Export des offres sur S3 v2": {
-            cron_string: "0 3 * * *",
-            handler: () => exportJobsToS3V2(),
-          },
           "Suppression des contacts Brevo de plus de deux ans": {
             cron_string: "0 8 * * SUN",
             handler: removeBrevoContacts,
-            tag: "main",
           },
         },
     jobs: {

server/src/jobs/partenaireExport/exportBlacklistAlgo.ts

@@ -0,0 +1,41 @@
+import { LBA_ITEM_TYPE } from "shared/constants/lbaitem"
+
+import { s3WriteString } from "@/common/utils/awsUtils"
+import { getDbCollection } from "@/common/utils/mongodbUtils"
+import { notifyToSlack } from "@/common/utils/slackUtils"
+
+const uploadJsonToS3 = async (fileKey: string, data: unknown): Promise<void> => {
+  await s3WriteString("storage", fileKey, {
+    Body: JSON.stringify(data, null, 2),
+    ContentType: "application/json",
+  })
+}
+
+const exportBlacklistAlgo = async (): Promise<void> => {
+  const [reportedCompanies, unsubscribedCompanies] = await Promise.all([
+    getDbCollection("reported_companies")
+      .find({ reason: "Entreprise fermée", type: LBA_ITEM_TYPE.RECRUTEURS_LBA }, { projection: { _id: 0, siret: 1 } })
+      .toArray(),
+    getDbCollection("unsubscribedrecruteurslba")
+      .find({}, { projection: { _id: 0, siret: 1 } })
+      .toArray(),
+  ])
+
+  const uniqueSiret = [...new Set([...reportedCompanies, ...unsubscribedCompanies].map((company) => company.siret))]
+
+  await uploadJsonToS3("blacklist-algo.json", uniqueSiret)
+  await notifyToSlack({ subject: "Fichier de blacklist pour l'algorithme", message: `${uniqueSiret.length} siret exportés` })
+}
+
+const exportApplications = async (): Promise<void> => {
+  const applications = await getDbCollection("applications")
+    .find({ job_origin: LBA_ITEM_TYPE.RECRUTEURS_LBA }, { projection: { _id: 1, company_siret: 1, created_at: 1 } })
+    .toArray()
+
+  await uploadJsonToS3("applications-algo.json", applications)
+  await notifyToSlack({ subject: "Fichier des candidatures pour l'algorithme", message: `${applications.length} candidatures exportées` })
+}
+
+export const exportFileForAlgo = async (): Promise<void> => {
+  await Promise.all([exportBlacklistAlgo(), exportApplications()])
+}

server/src/jobs/simpleJobDefinitions.ts

@@ -22,6 +22,7 @@ import { processRecruteursLba } from "@/jobs/offrePartenaire/recruteur-lba/proce
 import { processRhAlternance } from "@/jobs/offrePartenaire/rh-alternance/processRhAlternance"
 import { analyzeClosedCompanies } from "@/jobs/oneTimeJob/analyzeClosedCompanies"
 import { renvoiMailCreationCompte } from "@/jobs/oneTimeJob/renvoiMailCreationCompte"
+import { exportFileForAlgo } from "@/jobs/partenaireExport/exportBlacklistAlgo"
 import { exportJobsToS3V2 } from "@/jobs/partenaireExport/exportJobsToS3V2"
 import { exportJobsToFranceTravail } from "@/jobs/partenaireExport/exportToFranceTravail"
 import { repriseEnvoiEmailsPRDV } from "@/jobs/rdv/repriseEnvoiPRDV"
@@ -371,4 +372,8 @@ export const simpleJobDefinitions: SimpleJobDefinition[] = [
     fct: removeBrevoContacts,
     description: "Anonymise les contacts Brevo dont la date de creation est supérieure à 2 ans",
   },
+  {
+    fct: exportFileForAlgo,
+    description: "Export des données pour l'algorithme des recruteurs",
+  },
 ]