diff --git a/modules/redcap/php/config/redcapconfigparser.class.inc b/modules/redcap/php/config/redcapconfigparser.class.inc index ec0bfa1f0bd..d49725ffee5 100644 --- a/modules/redcap/php/config/redcapconfigparser.class.inc +++ b/modules/redcap/php/config/redcapconfigparser.class.inc @@ -448,4 +448,85 @@ class RedcapConfigParser . " (redcap-project-id: {$this->_redcap_project_id}): $message" ); } + + /** + * Get the full REDCap configuration as described in 'config.xml' file. + * Returns a struct ordered by REDCap instance and project. + * Expected structure is: + * [REDCap instance URL => [REDCap Project ID => REDCapConfig object]] + * else null if no configurations are found. + * + * @param \LORIS\LorisInstance $loris the loris instance + * + * @throws \LorisException + * + * @return array[] a REDCap configuration with the form + * [REDCap instance URL => [REDCap Project ID => REDCapConfig object]] + * else null + */ + public static function getConfiguration( + \LORIS\LorisInstance $loris + ): array { + // final REDCap configuration struct + $redcapConfig = []; + + // list of REDCap instances and projects + $rawRedcapConfig = $loris->getConfiguration()->getSetting('redcap'); + if ($rawRedcapConfig === null || empty($rawRedcapConfig)) { + throw new \LorisException("No REDCap configuration."); + } + + // instances, must at least have one + $redcapInstances = $rawRedcapConfig['instance'] ?? null; + if ($redcapInstances === null || empty($redcapInstances)) { + throw new \LorisException( + "No REDCap instance defined in configuration file." + ); + } + + // only one instance, wrap it in an array to be able to iterate + if (!array_key_exists(0, $redcapInstances)) { + $redcapInstances = [$redcapInstances]; + } + + // going throught REDCap defined instances + foreach ($redcapInstances as $instanceStruct) { + // instance URL + $redcapInstanceURL = $instanceStruct['redcap-url']; + + // project of this instance + $redcapProjects = $instanceStruct['project'] ?? null; + if ($redcapProjects === null || empty($redcapProjects)) { + // no project for that instance, next REDCap instance + continue; + } + + // only one project, wrap it in an array to be able to iterate + if (!array_key_exists(0, $redcapProjects)) { + $redcapProjects = [$redcapProjects]; + } + + // add REDCap configuration + foreach ($redcapProjects as $redcapProject) { + // project ID + $redcapProjectID = $redcapProject['redcap-project-id']; + + // parser object + $configParser = new RedcapConfigParser( + $loris, + $redcapInstanceURL, + $redcapProjectID + ); + + // add the parsed REDCap configuration to the final struct + $redcapConfig[$redcapInstanceURL] = [ + ...$redcapConfig[$redcapInstanceURL] ?? [], + $redcapProjectID => $configParser->parse() + ]; + } + } + + // return struct + return empty($redcapConfig) ? null : $redcapConfig; + } } diff --git a/modules/redcap/php/queries.class.inc b/modules/redcap/php/queries.class.inc index 29c4008a56f..b50b701ea2d 100644 --- a/modules/redcap/php/queries.class.inc +++ b/modules/redcap/php/queries.class.inc @@ -16,6 +16,7 @@ namespace LORIS\redcap; use \LORIS\LorisInstance; +use \LORIS\Database\Query; use \LORIS\StudyEntities\Candidate\CandID; /** @@ -283,4 +284,65 @@ class Queries { $this->_db->run("UNLOCK TABLES"); } + + /** + * Get the list of all instrument records to import from REDCap. + * + * @param array $allowedRedcapInstrumentNames authorized redcap instruments + * @param bool $forceUpdate do force update? + * + * @return Query + */ + public function getLORISInstrumentsToImport( + array $allowedRedcapInstrumentNames, + bool $forceUpdate = false + ): Query { + // extract first importable instrument name + $firstInstrumentName = $allowedRedcapInstrumentNames[0]; + $allOtherInstrumentNames = array_slice( + $allowedRedcapInstrumentNames, + 1 + ); + + // build query for exact instrument name match + $selectedInstruments = "AND ("; + $selectedInstruments .= " BINARY f.test_name = '{$firstInstrumentName}'"; + foreach ($allOtherInstrumentNames as $instrumentName) { + $selectedInstruments .= " OR BINARY f.test_name = '{$instrumentName}'"; + } + $selectedInstruments .= ")"; + + // if forced, then we do not filter out administration/data entry already + // set up, they will be overridden + $forceUpdateCondition = ""; + if (!$forceUpdate) { + $forceUpdateCondition = " + AND (f.Data_entry = 'In Progress' OR f.Data_entry IS NULL) + AND f.Administration IS NULL + "; + } + + $query = "SELECT c.PSCID as pscid, + c.CandID as candid, + s.Visit_label as visitLabel, + f.test_name as instrument, + f.CommentID as commentID, + f.Data_entry as dataEntry, + f.Administration as administration + FROM flag f + JOIN session s ON (s.ID = f.sessionID) + JOIN candidate c ON (c.candid = s.candid) + WHERE s.Active = 'Y' + AND c.Active = 'Y' + AND f.CommentID NOT LIKE 'DDE%' + {$forceUpdateCondition} + {$selectedInstruments} + "; + + // + return $this->_db->pselect( + $query, + [] + ); + } } diff --git a/modules/redcap/tools/redcap_bulk_importer.php b/modules/redcap/tools/redcap_bulk_importer.php new file mode 100644 index 00000000000..b7470ef1dff --- /dev/null +++ b/modules/redcap/tools/redcap_bulk_importer.php @@ -0,0 +1,643 @@ +#!/usr/bin/env php + + * @license Loris license + * @link https://www.github.com/aces/Loris/ + */ + +ini_set('display_errors', 1); +ini_set('display_startup_errors', 1); +error_reporting(E_ALL); + +require_once __DIR__ . "/../../../tools/generic_includes.php"; + +// load redcap module +try { + $lorisInstance->getModule('redcap')->registerAutoloader(); +} catch (\LorisNoSuchModuleException $th) { + error_log("[error] no 'redcap' module found."); + exit(1); +} + +use \GuzzleHttp\Client; +use Psr\Http\Message\ResponseInterface; + +use LORIS\StudyEntities\Candidate\CandID; + +use LORIS\redcap\client\models\mappings\RedcapInstrumentEventMap; +use LORIS\redcap\config\RedcapConfigLorisId; +use LORIS\redcap\config\RedcapConfigParser; +use LORIS\redcap\client\RedcapHttpClient; +use LORIS\redcap\Queries; + + +// -------------------------------------- + +/** + * All REDCap connections (http client) for each REDCap instance/project. + * + * @var array + */ +$redcapConnections = []; + +// /** +// * All REDCap unique event names (event + arm) for each REDCap instance/project. +// * +// * @var mixed +// */ +// $redcapEventMap = []; + +// /** +// * All REDCap instruments for each REDCap instance/project. +// * +// * @var mixed +// */ +// $redcapInstrumentMap = []; + +/** + * All REDCap instrument-event map for each REDCap instance/project. + * + * @var array> + */ +$redcapInstrumentEventMap = []; + +/** + * REDCap related queries. + * + * @var Queries + */ +$queries = new Queries($lorisInstance); + + +// -------------------------------------- +// ARGS PARSE +$options = getopt( + "", + [ + "loris-url:", + "redcap-username:", + "force-update", + "verbose", + ] +); + +// +$opts = checkOptions($options); + +// args mapping +$lorisURL = $opts['lorisURL']; +$redcapUsername = $opts['redcapUsername']; +$forceUpdate = $opts['forceUpdate']; +$verbose = $opts['verbose']; + +// arg display +fprintf(STDOUT, "[args:loris_url] {$lorisURL}\n"); +$verboseMsg = $verbose ? 'enabled' : 'disabled'; +fprintf(STDOUT, "[args:verbose] {$verboseMsg}\n"); +$forceUpdateMsg = $forceUpdate ? 'enabled' : 'disabled'; +fprintf(STDOUT, "[args:force_update] {$forceUpdateMsg}\n"); +fprintf(STDOUT, "[args:redcap_username] {$redcapUsername}\n"); + +// -------------------------------------- +// Main process + +// init LORIS client +$lorisClient = new Client( + ['base_uri' => "{$lorisURL}/redcap/notifications"] +); + +// Load all LORIS importable instruments (across all REDCap instances) +$redcapAllowedInstrumentNames = $config->getSetting('redcap_importable_instrument'); +if (empty($redcapAllowedInstrumentNames)) { + fprintf(STDERR, "[redcap:configuration] no instrument authorized.\n"); + exit(3); +} + +// Get LORIS data to import +fprintf(STDOUT, "[loris:data] getting data to import...\n"); +$lorisDataToImport = $queries->getLORISInstrumentsToImport( + $redcapAllowedInstrumentNames, + $forceUpdate +); + +$cLorisData = $lorisDataToImport->count(); +if ($cLorisData === 0) { + fprintf(STDERR, "[loris:data] no data to import.\n"); + exit(0); +} + +$forceMsg = $forceUpdate + ? "(already 'Complete' instruments too)" + : "('In Progress' instruments)"; +fprintf( + STDOUT, + "[loris:data] records to import from REDCap {$forceMsg}: {$cLorisData}\n" +); + +// Load REDCap configuration +fprintf(STDOUT, "[redcap:configuration] getting REDCap configurations...\n"); +try { + $redcapConfiguration = RedcapConfigParser::getConfiguration($lorisInstance); +} catch (\LorisException $th) { + fprintf(STDERR, "[redcap:configuration] {$th->getMessage()}.\n"); + exit(3); +} +// +if ($redcapConfiguration === null) { + fprintf( + STDERR, + "[redcap:configuration] no REDCap configuration in 'config.xml'.\n" + ); + exit(3); +} + +// Load REDCap connections +fprintf(STDOUT, "[redcap:connections] building REDCap connections...\n"); +initREDCapConnections( + $redcapConfiguration, + $redcapConnections, + false +); + +// Test REDCap connections +fprintf(STDOUT, "[redcap:connections] testing REDCap connections...\n"); +testREDCapConnections($redcapConnections); + +// // Load REDCap unique event names +// fprintf(STDOUT, "[redcap:connections] loading REDCap events...\n"); +// initREDCapEventMap( +// $redcapConnections, +// $redcapEventMap +// ); + +// // Load REDCap instruments, only consider importable instruments +// fprintf(STDOUT, "[redcap:connections] loading REDCap instruments...\n"); +// initREDCapInstrumentMap( +// $redcapConnections, +// $redcapAllowedInstrumentNames, +// $redcapInstrumentMap +// ); + +// Load REDCap instrument-event, only consider importable instruments +fprintf(STDOUT, "[redcap:connections] loading REDCap instrument-event mapping...\n"); +initREDCapInstrumentEventMap( + $redcapConnections, + $redcapAllowedInstrumentNames, + $redcapInstrumentEventMap +); + +// error_log( +// print_r( +// $redcapInstrumentEventMap['https://redcap.iths.org/'][98505], +// true +// ) +// ); + +// iterating over all records +fprintf(STDERR, "[loris:redcap_endpoint] triggering notifications and import...\n"); +triggerNotifications( + $lorisClient, + $lorisDataToImport, + $redcapInstrumentEventMap, + $redcapConfiguration, + $redcapUsername +); + +// -------------------------------------- + +/** + * Trigger the selected set of notification to import data in LORIS. + * + * @param GuzzleHttp\Client $lorisClient the loris client + * @param LORIS\Database\Query $lorisDataToImport the list of data to import + * @param mixed[] $redcapInstrumentEventMap the instrument-event map + * @param mixed[] $redcapConfiguration the REDCap conf + * @param string $redcapUsername the text to pass + * + * @return void + */ +function triggerNotifications( + GuzzleHttp\Client $lorisClient, + LORIS\Database\Query $lorisDataToImport, + array $redcapInstrumentEventMap, + array $redcapConfiguration, + string $redcapUsername +): void { + foreach ($lorisDataToImport as $index => $instrumentToQuery) { + // LORIS param + $pscid = $instrumentToQuery['pscid']; + $candid = $instrumentToQuery['candid']; + $visitLabel = $instrumentToQuery['visitLabel']; + $instrumentName = $instrumentToQuery['instrument']; + $commentID = $instrumentToQuery['commentID']; + + // candidate + $candidate = \Candidate::singleton(new CandID("{$candid}")); + + // log + $log = "[{$index}]"; + $log .= "[pscid:{$pscid}|candid:{$candid}]"; + $log .= "[visit:{$visitLabel}]"; + $log .= "[instrument:{$instrumentName}]"; + fprintf(STDOUT, "{$log} \n"); + + // select REDCap instances and projects that have this instrument + // [redcap instance URL => + // [redcap project ID => RedcapInstrumentEventMap object] + // ] + // ideally, there should only be one instance and one project selected + // target event-instrument mapping from REDCap based on instrument name + $redcapTargetedEventInstrument = getTargetedEventInstrument( + $redcapInstrumentEventMap, + $instrumentName + ); + + // count number of selected event-instrument mappings across instances + $nbProjects = array_reduce( + $redcapTargetedEventInstrument, + fn($c, $i) => $c + count($i), + 0 + ); + if ($nbProjects === 0) { + fprintf( + STDERR, + " - no REDCap instances/projects for that instrument.\n" + ); + continue; + } + if ($nbProjects > 1) { + // TODO: what if an instrument name is in different instances/projects? + // TODO: logic needs to be implemented more largely in the rest of the + // TODO: codebase. + fprintf( + STDERR, + " - multiple REDCap instances/projects with that instrument.\n" + ); + continue; + } + + // explicit values + $redcapInstanceURL = array_keys($redcapTargetedEventInstrument)[0]; + $instanceData = $redcapTargetedEventInstrument[$redcapInstanceURL]; + $redcapProjectID = array_keys($instanceData)[0]; + $redcapEventInstrument = $instanceData[$redcapProjectID]; + + fprintf( + STDOUT, + " - instrument found in '{$redcapInstanceURL}', pid:{$redcapProjectID}\n" + ); + + // get this instance/project configuration + // TODO: which candidate ID? PSCID/CANDID? + // TODO: which participant ID? RecordID/surveyID? + // TODO: which visit? visit mapping in config. + $redcapConfig = $redcapConfiguration[$redcapInstanceURL][$redcapProjectID]; + + // candidate ID to use + // TODO: maybe some change here depending on the REDCap way of naming + $redcapRecordID = match ($redcapConfig->candidate_id) { + RedcapConfigLorisId::PscId => $candidate->getPSCID(), + RedcapConfigLorisId::CandId => $candidate->getCandID(), + }; + + // send POST request to LORIS mimicing REDCap notification + try { + $response = sendNotification( + $lorisClient, + $redcapInstanceURL, + $redcapProjectID, + $instrumentName, + $redcapRecordID, + $redcapEventInstrument->unique_event_name, + $redcapUsername + ); + + // get status message + $responseStatusMsg = $response->getStatusCode() != 200 ? "failed" : "ok"; + } catch (GuzzleHttp\Exception\ServerException $ex) { // 500-level + $msg = $ex->getResponse()->getBody()->getContents(); + error_log($msg); + $responseStatusMsg = "[5xx] failed"; + } catch (GuzzleHttp\Exception\ClientException $ex) { // 400-level + $msg = $ex->getResponse()->getBody()->getContents(); + error_log($msg); + $responseStatusMsg = "[4xx] failed"; + } + + fprintf(STDERR, " - {$responseStatusMsg}\n"); + } +} + +/** + * Initialize REDCap connections based on the given configuration. + * + * @param mixed[] $redcapConfig REDCap configuration + * structure + * @param array $redcapConnections REDCap connection + * structure to fill + * @param bool $verbose Verbose mode + * + * @return void + */ +function initREDCapConnections( + array $redcapConfig, + array &$redcapConnections, + bool $verbose = false +): void { + foreach ($redcapConfig as $redcapURL => $redcapInstance) { + foreach ($redcapInstance as $redcapProjectID => $redcapConfig) { + $cleanURL = rtrim($redcapURL, '/'); + $redcapConnections[$redcapURL] = [ + ...$redcapConnections[$redcapURL] ?? [], + $redcapProjectID => new RedcapHttpClient( + "{$cleanURL}/api/", + $redcapConfig->redcap_api_token, + $verbose + ) + ]; + } + } +} + +/** + * Test all REDCap connections based on a given connection structure. + * + * @param array $redcapConnections REDCap connection structure + * + * @return void + */ +function testREDCapConnections( + array $redcapConnections +): void { + foreach ($redcapConnections as $redcapURL => $redcapInstance) { + foreach ($redcapInstance as $redcapProjectID => $redcapClient) { + $redcapClient->checkConnection(); + } + } +} + +// /** +// * Initialize REDCap events based on the given connection. +// * +// * @param array $redcapConnections +// * @param array $redcapEventMap +// * +// * @return void +// */ +// function initREDCapEventMap( +// array $redcapConnections, +// array &$redcapEventMap +// ): void { +// foreach ($redcapConnections as $redcapURL => $redcapInstance) { +// foreach ($redcapInstance as $redcapProjectID => $redcapClient) { +// $redcapEventMap[$redcapURL] = [ +// ...$redcapEventMap[$redcapURL] ?? [], +// $redcapProjectID => $redcapClient->getEvents() +// ]; +// } +// } +// } + +// /** +// * Initialize REDCap instruments based on the given connection. +// * +// * @param array $redcapConnections +// * @param array $redcapAllowedInstrumentNames +// * @param array $redcapInstrumentMap +// * +// * @return void +// */ +// function initREDCapInstrumentMap( +// array $redcapConnections, +// array $redcapAllowedInstrumentNames, +// array &$redcapInstrumentMap +// ): void { +// foreach ($redcapConnections as $redcapURL => $redcapInstance) { +// foreach ($redcapInstance as $redcapProjectID => $redcapClient) { +// // all instruments from a client +// $instruments = $redcapClient->getInstruments(); + +// // filter allowed instruments +// $allowedInstruments = array_filter( +// $instruments, +// fn($i) => in_array( +// $i->name, +// $redcapAllowedInstrumentNames, +// true +// ) +// ); + +// // +// if (!empty($allowedInstruments)) { +// $redcapInstrumentMap[$redcapURL] = [ +// ...$redcapInstrumentMap[$redcapURL] ?? [], +// $redcapProjectID => $allowedInstruments +// ]; +// } +// } +// } +// } + +/** + * Initialize REDCap instrument-event based on the given connection. + * + * @param array $redcapConnections REDCap + * connections + * structure + * @param array $redcapAllowedInstrumentNames Allowed + * instruments + * @param mixed[] $redcapInstrumentEventMap REDCap + * event-instrument + * mapping + * + * @return void + */ +function initREDCapInstrumentEventMap( + array $redcapConnections, + array $redcapAllowedInstrumentNames, + array &$redcapInstrumentEventMap +): void { + foreach ($redcapConnections as $redcapURL => $redcapInstance) { + foreach ($redcapInstance as $redcapProjectID => $redcapClient) { + // instrument-event map from a client + $instrumentEventMap = $redcapClient->getInstrumentEventMapping(); + + // filter allowed instruments + $allowedInstruments = array_filter( + $instrumentEventMap, + fn($i) => in_array( + $i->form_name, + $redcapAllowedInstrumentNames, + true + ) + ); + + // + if (!empty($allowedInstruments)) { + $redcapInstrumentEventMap[$redcapURL] = [ + ...$redcapInstrumentEventMap[$redcapURL] ?? [], + $redcapProjectID => $allowedInstruments + ]; + } + } + } +} + +/** + * Send a REDCap notification to LORIS. + * + * @param GuzzleHttp\Client $lorisClient LORIS client + * @param string $redcapAPIURL the REDCap API URL to use + * @param int $redcapProjectID the REDCap project ID + * @param string $redcapInstrumentName the REDCap instrument name + * @param string $redcapRecordID the REDCap Record ID + * @param string $redcapUniqueEventName the REDCap unique event name + * @param string $redcapUsername the REDCap user + * + * @return ResponseInterface + */ +function sendNotification( + Client $lorisClient, + string $redcapAPIURL, + int $redcapProjectID, + string $redcapInstrumentName, + string $redcapRecordID, + string $redcapUniqueEventName, + string $redcapUsername +): ResponseInterface { + // data to send + $data = [ + "redcap_url" => $redcapAPIURL, + "project_id" => $redcapProjectID, + "project_url" => "", + "instrument" => $redcapInstrumentName, + "record" => $redcapRecordID, + "redcap_event_name" => $redcapUniqueEventName, + "username" => $redcapUsername, + "{$redcapInstrumentName}_complete" => "2", + ]; + + // send + return $lorisClient->request( + 'POST', + '/redcap/notifications', + [ + 'form_params' => $data, + 'debug' => false + ] + ); +} + +/** + * Get a targeted event-instrument mapping. + * This returns a structure + * [redcap instance URL => [redcap project ID => RedcapInstrumentEventMap object]] + * when the given instrument is found in any redcap event-instrument mapping. + * + * @param array $redcapInstrumentEventMap the event-instrument mapping + * @param string $instrumentName the searched instrument name + * + * @return array> an array + */ +function getTargetedEventInstrument( + array $redcapInstrumentEventMap, + string $instrumentName +): array { + $selectedRedcapProject = []; + foreach ($redcapInstrumentEventMap as $redcapURL => $redcapInstance) { + foreach ($redcapInstance as $redcapProjectID => $redcapInstruments) { + // search instrument + $foundInstrumentEventMap = array_filter( + $redcapInstruments, + fn($i) => $i->form_name === $instrumentName + ); + + // get the precise mapping "RedcapInstrumentEventMap" object + if (!empty($foundInstrumentEventMap)) { + $selectedRedcapProject[$redcapURL] = [ + ...$selectedRedcapProject[$redcapURL] ?? [], + $redcapProjectID => array_values($foundInstrumentEventMap)[0] + ]; + } + } + } + return $selectedRedcapProject; +}; + +// -------------------------------------- +// Utility + +/** + * Check arguments passed to this script. + * + * @param array $options the arguments + * + * @return array{ + * lorisURL: string, + * redcapUsername: string, + * forceUpdate: bool, + * verbose: bool + * } clean and valid options. + */ +function checkOptions(array $options): array +{ + // loris URL, mandatory + if (!isset($options['loris-url'])) { + error_log("[error] Required parameter: 'loris-path'."); + showHelp(); + exit(1); + } + $lorisURL = rtrim($options['loris-url'], '/'); + + // redcap-username + $redcapUsername = $options['redcap-username'] ?? 'bulk_import'; + + // force-update + $forceUpdate = isset($options['force-update']); + + // verbose + $verbose = isset($options['verbose']); + + // clean + return [ + 'lorisURL' => $lorisURL, + 'redcapUsername' => $redcapUsername, + 'forceUpdate' => $forceUpdate, + 'verbose' => $verbose, + ]; +} + +/** + * Displays help for this script. + * + * @return void + */ +function showHelp() : void +{ + fprintf( + STDERR, + "Usage:\n" + . " php redcap_bulk_importer.php \n" + . " {--loris-url=LORISURL}\n" + . " [--redcap-username=USER]\n" + . " [--force-update]\n" + . " [--verbose]\n\n" + . "Notes:\n" + . " - required, '--loris-url=LORISURL' " + . "the loris URL\n" + . " - optional, '--redcap-username=USER' " + . "the text that will be in all redcap notification as the REDCap " + . "username. (default: 'bulk_import')\n" + . " - optional, '--force-update' needs to force update on " + . "complete instrument? (default: false)\n" + . " - optional, '--verbose' verbose mode? (default: false)\n\n" + ); +} \ No newline at end of file diff --git a/test/phpstan-loris.neon b/test/phpstan-loris.neon index 8f0ec6edb35..7ca17d7f62d 100644 --- a/test/phpstan-loris.neon +++ b/test/phpstan-loris.neon @@ -2,3 +2,6 @@ parameters: excludes_analyse: - %currentWorkingDirectory%/modules/*/ajax/* - %currentWorkingDirectory%/modules/*/test/* + scanDirectories: + - %currentWorkingDirectory%/modules/redcap/php/client/* + - %currentWorkingDirectory%/modules/redcap/php/config/*