Index schedules

Classes/Command/SearchableCommandController.php

@@ -6,6 +6,8 @@
 use PAGEmachine\Searchable\IndexManager;
 use PAGEmachine\Searchable\PipelineManager;
 use PAGEmachine\Searchable\Service\ExtconfService;
+use TYPO3\CMS\Core\Registry;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
 use TYPO3\CMS\Extbase\SignalSlot\Dispatcher;
 
@@ -46,17 +48,25 @@ public function injectSignalDispatcher(Dispatcher $signalDispatcher)
     protected $scheduledIndexers = [];
 
     /**
-     * Determines if a full indexing is performed
-     * @var bool
+     * Indexing type, one of full, partial or schedule
+     *
+     * @var string
      */
-    protected $runFullIndexing = false;
+    protected $indexingType;
 
     /**
      * Index type. If null, all indexers are run
      * @var string|null
      */
     protected $type = null;
 
+    /**
+     * Last time schedules where processed for indexing
+     *
+     * @var \DateTime
+     */
+    protected $lastScheduleProcessingDate;
+
     /**
      * Runs all indexers (full)
      * @param  string $type If set, only runs indexing for the given type
@@ -67,7 +77,7 @@ public function indexFullCommand($type = null)
         $this->outputLine();
         $this->checkHealth();
 
-        $this->runFullIndexing = true;
+        $this->indexingType = 'full';
         $this->type = $type;
 
         $this->collectScheduledIndexers();
@@ -84,13 +94,37 @@ public function indexPartialCommand($type = null)
         $this->outputLine();
         $this->checkHealth();
 
-        $this->runFullIndexing = false;
+        $this->indexingType = 'partial';
         $this->type = $type;
 
         $this->collectScheduledIndexers();
         $this->runIndexers();
     }
 
+    /**
+     * Run all indexers (scheduled updates only)
+     *
+     * @param string $type If set, only runs indexing for the given type
+     * @return void
+     */
+    public function indexSchedulesCommand($type = null)
+    {
+        $this->outputLine();
+        $this->checkHealth();
+
+        $this->indexingType = 'schedule';
+        $this->type = $type;
+
+        /** @var \TYPO3\CMS\Core\Registry */
+        $registry = GeneralUtility::makeInstance(Registry::class);
+        $this->lastScheduleProcessingDate = $registry->get('searchable', 'scheduleProcessingDate', new \DateTime('@0'));
+
+        $this->collectScheduledIndexers();
+        $this->runIndexers();
+
+        $registry->set('searchable', 'scheduleProcessingDate', new \DateTime('@' . $GLOBALS['EXEC_TIME']));
+    }
+
     /**
      * Reset index for one or all languages
      *
@@ -279,15 +313,28 @@ protected function runSingleIndexer(IndexerInterface $indexer)
         $this->outputLine("<comment> Type '%s':</comment>", [$indexer->getType()]);
         $this->output->progressStart();
 
-        if ($this->runFullIndexing) {
-            foreach ($indexer->run() as $resultMessage) {
-                $this->output->progressSet($resultMessage);
-            }
-        } else {
-            foreach ($indexer->runUpdate() as $resultMessage) {
-                $this->output->progressSet($resultMessage);
-            }
+        switch ($this->indexingType) {
+            case 'full':
+                $progressValues = $indexer->run();
+                break;
+
+            case 'partial':
+                $progressValues = $indexer->runUpdate();
+                break;
+
+            case 'schedule':
+                $currentExecutionDate = new \DateTime('@' . $GLOBALS['EXEC_TIME']);
+                $progressValues = $indexer->runScheduleUpdate($this->lastScheduleProcessingDate, $currentExecutionDate);
+                break;
+
+            default:
+                throw new \UnexpectedValueException(sprintf('Unexpected indexing type "%s"', $this->indexingType), 1518003689);
         }
+
+        foreach ($progressValues as $progress) {
+            $this->output->progressSet($progress);
+        }
+
         $this->output->progressFinish();
     }
 

Classes/DataCollector/ScheduleAwareDataCollectorInterface.php

@@ -0,0 +1,19 @@
+<?php
+namespace PAGEmachine\Searchable\DataCollector;
+
+/*
+ * This file is part of the PAGEmachine Searchable project.
+ */
+
+interface ScheduleAwareDataCollectorInterface
+{
+     /**
+     * Fetches the list of records scheduled (expired/activated) in a date range
+     *
+     * @param \DateTime $startDate the start of the date range
+     * @param \DateTime $endTime the end of the date range
+     * @param SchedulingType $type the scheduling type
+     * @return \Traversable
+     */
+    public function getScheduledRecords(\DateTime $startDate, \DateTime $endDate, SchedulingType $type);
+}

Classes/DataCollector/SchedulingType.php

@@ -0,0 +1,24 @@
+<?php
+namespace PAGEmachine\Searchable\DataCollector;
+
+/*
+ * This file is part of the Pagemachine Searchable project.
+ */
+
+use TYPO3\CMS\Core\Type\Enumeration;
+
+/**
+ * Enumeration of record scheduling types
+ */
+final class SchedulingType extends Enumeration
+{
+    /**
+     * Records which have been activated: their start date was passed
+     */
+    const ACTIVATED = 1;
+
+    /**
+     * Records which have expired: their end date has passed
+     */
+    const EXPIRED = 2;
+}

Classes/DataCollector/TcaDataCollector.php

@@ -9,6 +9,8 @@
 use TYPO3\CMS\Backend\Utility\BackendUtility;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Database\Query\QueryHelper;
+use TYPO3\CMS\Core\Database\Query\Restriction\EndTimeRestriction;
+use TYPO3\CMS\Core\Database\Query\Restriction\StartTimeRestriction;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /*
@@ -18,7 +20,7 @@
 /**
  * Class for fetching TCA-based data according to the given config
  */
-class TcaDataCollector extends AbstractDataCollector implements DataCollectorInterface
+class TcaDataCollector extends AbstractDataCollector implements DataCollectorInterface, ScheduleAwareDataCollectorInterface
 {
     /**
      *
@@ -214,6 +216,48 @@ public function getUpdatedRecords($updateUidList)
         }
     }
 
+    /**
+     * Fetches the list of records scheduled in a date range
+     *
+     * @param \DateTime $startDate the start of the date range
+     * @param \DateTime $endTime the end of the date range
+     * @param SchedulingType $type the scheduling type
+     * @return \Traversable
+     */
+    public function getScheduledRecords(\DateTime $startDate, \DateTime $endDate, SchedulingType $type)
+    {
+        // Only run if there is a valid TCA available
+        if (!$this->isTcaAvailable()) {
+            yield from [];
+        }
+
+        $tableControl = $this->getTcaConfiguration()['ctrl'];
+        $scheduleField = $type->equals(SchedulingType::ACTIVATED) ? 'starttime' : 'endtime';
+
+        if (empty($tableControl['enablecolumns'][$scheduleField])) {
+            yield from [];
+        }
+
+        $queryBuilder = $this->buildUidListQueryBuilder();
+        $queryBuilder->getRestrictions()
+            ->removeByType(StartTimeRestriction::class)
+            ->add(GeneralUtility::makeInstance(StartTimeRestriction::class, $startDate->getTimestamp()))
+            ->removeByType(EndTimeRestriction::class)
+            ->add(GeneralUtility::makeInstance(EndTimeRestriction::class, $endDate->getTimestamp()));
+
+        $result = $queryBuilder->execute();
+
+        foreach ($result as $record) {
+            if ($type->equals(SchedulingType::ACTIVATED)) {
+                $record = $this->getRecord($record['uid']);
+            }
+
+            yield $record;
+        }
+
+        yield from [];
+    }
+
     /**
      * Fetches a single record
      *

Classes/Indexer/Indexer.php

@@ -2,6 +2,8 @@
 namespace PAGEmachine\Searchable\Indexer;
 
 use PAGEmachine\Searchable\Configuration\DynamicConfigurationInterface;
+use PAGEmachine\Searchable\DataCollector\ScheduleAwareDataCollectorInterface;
+use PAGEmachine\Searchable\DataCollector\SchedulingType;
 use PAGEmachine\Searchable\LinkBuilder\LinkBuilderInterface;
 use PAGEmachine\Searchable\LinkBuilder\PageLinkBuilder;
 use PAGEmachine\Searchable\Preview\DefaultPreviewRenderer;
@@ -350,6 +352,54 @@ public function runUpdate()
         }
     }
 
+    /**
+     * Runs an update for scheduled documents
+     *
+     * @param \DateTime $startDate the start of the date range
+     * @param \DateTime $endDate the end of the date range
+     * @return \Generator
+     */
+    public function runScheduleUpdate(\DateTime $startDate, \DateTime $endDate)
+    {
+        if (!$this->dataCollector instanceof ScheduleAwareDataCollectorInterface) {
+            yield 0;
+
+            return;
+        }
+
+        $bulkSize = $this->config['bulkSize'] ?: 20;
+        $counter = 0;
+        $overallCounter = 0;
+        $records = [];
+
+        foreach ($this->dataCollector->getScheduledRecords($startDate, $endDate, SchedulingType::cast(SchedulingType::ACTIVATED)) as $record) {
+            $records[] = $record;
+            $counter++;
+            $overallCounter++;
+
+            if ($counter >= $bulkSize) {
+                $this->sendBatch($records);
+
+                $records = [];
+                $counter = 0;
+                yield $overallCounter;
+            }
+        }
+
+        if ($counter > 0) {
+            $this->sendBatch($records);
+
+            yield $overallCounter;
+        }
+
+        foreach ($this->dataCollector->getScheduledRecords($startDate, $endDate, SchedulingType::cast(SchedulingType::EXPIRED)) as $record) {
+            $overallCounter++;
+            $this->query->delete($record['uid']);
+
+            yield $overallCounter;
+        }
+    }
+
     /**
      * Sends a batch
      *