From dba83587a2cb4370de46b255b59b8a2641318cfb Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 27 May 2019 18:48:35 +0300
Subject: [PATCH 01/21] MEPTS-207: Exctracting concept usage details from HTML
 forms and preparing to do the same for other metadata

---
 .gitignore                            |  1 +
 metadataUsage/openmrsMetadataUsage.sh | 83 +++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 .gitignore
 create mode 100755 metadataUsage/openmrsMetadataUsage.sh

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d05c513
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+metadataUsage/data/
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
new file mode 100755
index 0000000..7b3676a
--- /dev/null
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+MYSQL_CMD=$1
+MYSQL_USER=$2
+MYSQL_OPTS=$3
+MYSQL_DB=$4
+form_concept_questions_file="data/form_concept_questions.txt"
+form_concept_answers_file="data/form_concept_answers.txt"
+
+
+## require mysql command or location, user and database
+if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_OPTS" == "" ] || [ "$MYSQL_DB" == "" ]; then
+  printf "\nUsage; mysql_command openmrs_user mysql_opts openmrs_db. Replace any blank values with ''"
+  printf "\nExample; mysql openmrs_user '--socket=/tmp/omrs.sock --max_allowed_packet=96M' openmrs_db"
+  exit 1
+fi
+
+
+## wait for mysql password if any
+echo -n "Enter Mysql password": 
+read -s MYSQL_PASS
+echo
+## initialise data location
+rm -r data;mkdir data
+
+
+## extract form uuids and loop through them
+printf "\nStarting to extract forms from the database\n"
+$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
+do
+  echo -n "."
+  xmlData=`$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
+  ## printf evaluates % character, let us replace it so it's not treated special
+  xmlData="${xmlData//%/PCNT}"
+  ## clean up the xml data file and save each form named after its form uuid
+  printf "$xmlData" > "data/form_$uuid.xml"
+  ## replace back %
+  sed -n -e 's/PCNT/%/g' "data/form_$uuid.xml"
+done
+printf "\nFinished extracting forms from the database\n"
+
+
+## extracting concepts from forms
+printf "\nStarting to extract concepts from forms\n"
+for form_file in data/form_*.xml
+do
+  echo -n "."
+  ## extract concept question ids
+  echo -n 'cat //*/@conceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_questions_file
+  ## extract concept answer ids
+  echo -n 'cat //*/@answerConceptIds' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_answers_file
+done
+printf "\nFinished extracting concepts from forms\n"
+
+
+## first argument; file, second argument; separator
+combineAllLinesFromFileIntoOneWithSeparator () {
+  lines=""
+  while IFS= read -r line
+  do
+    if [ "$line" != "" ] && [ "$line" != "concept_id" ]; then
+      lines="$lines$2$line"
+    fi
+  done < "$1"
+  ## echo or 'return' excluding first trailing character of separator
+  echo "${lines#?}"
+}
+
+
+## generate unique used concept ids from all form's questions and answers
+IFS=',' read -r -a concepts <<< "$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',')"
+concepts=($(printf "%s\n" "${concepts[@]}" | sort -u | tr '\n' ' '))
+usedConceptsIds=$(IFS=$','; echo "${concepts[*]}")
+
+
+## generate unique non used concept ids from all form's questions and answers
+printf "\n\nConcept Ids used in HTML Forms# \n$usedConceptsIds\n"
+nonUsedConceptsIds=$(echo "SELECT concept_id FROM concept WHERE concept_id NOT IN ($usedConceptsIds)" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N)
+nonUsedConceptsIds=`echo "$nonUsedConceptsIds" | tr '\n' ','`
+
+## exclude last trailing ',' character
+nonUsedConceptsIds="${nonUsedConceptsIds%?}"
+printf "\n\nConcept Ids Not used in HTML Forms# \n$nonUsedConceptsIds\n"

From 41838413df12015210c503c44a27cf7d6d392c02 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Tue, 28 May 2019 11:56:26 +0300
Subject: [PATCH 02/21] MEPTS-207: Added programs usage from forms

---
 metadataUsage/openmrsMetadataUsage.sh | 51 ++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 12 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 7b3676a..bdbc6b4 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -6,12 +6,13 @@ MYSQL_OPTS=$3
 MYSQL_DB=$4
 form_concept_questions_file="data/form_concept_questions.txt"
 form_concept_answers_file="data/form_concept_answers.txt"
+form_programs_file="data/form_programs.txt"
 
 
 ## require mysql command or location, user and database
 if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_OPTS" == "" ] || [ "$MYSQL_DB" == "" ]; then
-  printf "\nUsage; mysql_command openmrs_user mysql_opts openmrs_db. Replace any blank values with ''"
-  printf "\nExample; mysql openmrs_user '--socket=/tmp/omrs.sock --max_allowed_packet=96M' openmrs_db"
+  printf "\nUsage;./openmrsMetadataUsage.sh mysql_command_or_location openmrs_user mysql_opts openmrs_db. Replace any blank values with ''"
+  printf "\nExample;./openmrsMetadataUsage.sh mysql openmrs_user '--socket=/tmp/omrs.sock --max_allowed_packet=96M' openmrs_db"
   exit 1
 fi
 
@@ -28,6 +29,7 @@ rm -r data;mkdir data
 printf "\nStarting to extract forms from the database\n"
 $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
 do
+  ## simulate loading or progress bar
   echo -n "."
   xmlData=`$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
   ## printf evaluates % character, let us replace it so it's not treated special
@@ -41,16 +43,19 @@ printf "\nFinished extracting forms from the database\n"
 
 
 ## extracting concepts from forms
-printf "\nStarting to extract concepts from forms\n"
+printf "\nStarting to extract metadata from forms\n"
 for form_file in data/form_*.xml
 do
+  ## simulate loading or progress bar
   echo -n "."
   ## extract concept question ids
   echo -n 'cat //*/@conceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_questions_file
   ## extract concept answer ids
   echo -n 'cat //*/@answerConceptIds' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_answers_file
+  ## extract program ids
+  echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
 done
-printf "\nFinished extracting concepts from forms\n"
+printf "\nFinished extracting metadata from forms\n"
 
 
 ## first argument; file, second argument; separator
@@ -58,7 +63,7 @@ combineAllLinesFromFileIntoOneWithSeparator () {
   lines=""
   while IFS= read -r line
   do
-    if [ "$line" != "" ] && [ "$line" != "concept_id" ]; then
+    if [ "$line" != "" ]; then
       lines="$lines$2$line"
     fi
   done < "$1"
@@ -66,18 +71,40 @@ combineAllLinesFromFileIntoOneWithSeparator () {
   echo "${lines#?}"
 }
 
+## combine all ids into comma separated string
+prepareReadableDisplay () {
+  IFS=',' read -r -a metadata <<< "$1"
+  metadata=($(printf "%s\n" "${metadata[@]}" | sort -u | tr '\n' ' '))
+  metadataIds=$(IFS=$','; echo "${metadata[*]}")
+  echo "$metadataIds"
+}
+
 
-## generate unique used concept ids from all form's questions and answers
-IFS=',' read -r -a concepts <<< "$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',')"
-concepts=($(printf "%s\n" "${concepts[@]}" | sort -u | tr '\n' ' '))
-usedConceptsIds=$(IFS=$','; echo "${concepts[*]}")
+## generate unique used concept ids from all forms' questions and answers
+usedConceptsIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ','))"
+## generate unique program ids from all forms
+usedProgramIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ','))"
 
 
+## clean entire console to only show metadata usage
+clear && printf '\e[3J'
+
 ## generate unique non used concept ids from all form's questions and answers
 printf "\n\nConcept Ids used in HTML Forms# \n$usedConceptsIds\n"
 nonUsedConceptsIds=$(echo "SELECT concept_id FROM concept WHERE concept_id NOT IN ($usedConceptsIds)" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N)
 nonUsedConceptsIds=`echo "$nonUsedConceptsIds" | tr '\n' ','`
-
-## exclude last trailing ',' character
+## exclude last trailing ',' character and show non used conceptIds
 nonUsedConceptsIds="${nonUsedConceptsIds%?}"
-printf "\n\nConcept Ids Not used in HTML Forms# \n$nonUsedConceptsIds\n"
+printf "\nConcept Ids Not used in HTML Forms# \n$nonUsedConceptsIds\n"
+
+## print separator horizontal line
+printf '\n\n%s\n\n' _____________________________________________________________________________________________________
+
+##generate unique non used program ids from all forms
+printf "\n\nProgram Ids used in HTML Forms# \n$usedProgramIds\n"
+nonUsedProgramIds=$(echo "SELECT program_id FROM program WHERE program_id NOT IN ($usedProgramIds)" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N)
+nonUsedProgramIds=`echo "$nonUsedProgramIds" | tr '\n' ','`
+## exclude last trailing ',' character and show non used programIds
+nonUsedProgramIds="${nonUsedProgramIds%?}"
+printf "\nProgram Ids Not used in HTML Forms# \n$nonUsedProgramIds\n"
+

From ea96b8da22286ec7b3df0489400e8ecdb1c83fad Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Thu, 30 May 2019 12:10:43 +0300
Subject: [PATCH 03/21] MEPTS-207: Added indentifier types and person attribute
 types

---
 .gitignore                            |  1 +
 metadataUsage/openmrsMetadataUsage.sh | 95 +++++++++++++++++++++------
 2 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/.gitignore b/.gitignore
index d05c513..b298ec3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 metadataUsage/data/
+metadataUsage/usage/
\ No newline at end of file
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index bdbc6b4..6a7037b 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -1,12 +1,24 @@
 #!/bin/bash
 
+## This openmrs bash script helps extract details about 
+
 MYSQL_CMD=$1
 MYSQL_USER=$2
 MYSQL_OPTS=$3
 MYSQL_DB=$4
+form_concept_grouping_file="data/form_concept_grouping.txt"
 form_concept_questions_file="data/form_concept_questions.txt"
 form_concept_answers_file="data/form_concept_answers.txt"
 form_programs_file="data/form_programs.txt"
+form_expressions_file="data/form_expressions.txt"
+form_concept_used_csv="$PWD/usage/form_used_concepts.csv"
+form_concept_not_used_csv="$PWD/usage/form_not_used_concepts.csv"
+form_programs_used_csv="$PWD/usage/form_used_programs.csv"
+form_programs_not_used_csv="$PWD/usage/form_not_used_programs.csv"
+form_identifiers_used_csv="$PWD/usage/form_used_identifiers.csv"
+form_identifiers_not_used_csv="$PWD/usage/form_not_used_identifiers.csv"
+form_person_attributes_used_csv="$PWD/usage/form_used_person_attributes.csv"
+form_person_attributes_not_used_csv="$PWD/usage/form_not_used_person_attributes.csv"
 
 
 ## require mysql command or location, user and database
@@ -35,9 +47,9 @@ do
   ## printf evaluates % character, let us replace it so it's not treated special
   xmlData="${xmlData//%/PCNT}"
   ## clean up the xml data file and save each form named after its form uuid
-  printf "$xmlData" > "data/form_$uuid.xml"
-  ## replace back %
-  sed -n -e 's/PCNT/%/g' "data/form_$uuid.xml"
+  xmlData=$(printf "$xmlData")
+  ## save xmlData onto a respective form specific file replacing back %
+  echo "${xmlData//PCNT/%}" > "data/form_$uuid.xml"
 done
 printf "\nFinished extracting forms from the database\n"
 
@@ -49,11 +61,15 @@ do
   ## simulate loading or progress bar
   echo -n "."
   ## extract concept question ids
+  echo -n 'cat //*/@groupingConceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_grouping_file
+  ## extract concept question ids
   echo -n 'cat //*/@conceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_questions_file
   ## extract concept answer ids
   echo -n 'cat //*/@answerConceptIds' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_answers_file
   ## extract program ids
   echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
+  ## extract expressions
+  echo -n 'cat //*/@expression' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_expressions_file
 done
 printf "\nFinished extracting metadata from forms\n"
 
@@ -79,32 +95,71 @@ prepareReadableDisplay () {
   echo "$metadataIds"
 }
 
+## extract expression values; first argument;file, second argument; expression prefix, third argument; separator, forth; toBeRemoved
+extractExpressionValues () {
+  lines=""
+  while IFS= read -r line
+  do
+  	## starts with expression prefix
+    if [[ "$line" = "$2"* ]]; then
+      ## get value in between () within expression value
+      value="${line//$2/}"
+      value="${value//$4/}"
+      lines="$lines$3$value"
+    fi
+  done < "$1"
+  ## echo or 'return' excluding first trailing character of separator
+  echo "${lines#?}"
+}
 
 ## generate unique used concept ids from all forms' questions and answers
-usedConceptsIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ','))"
-## generate unique program ids from all forms
+usedConceptIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ','))"
+IFS=',' read -r -a usedConceptIdsArr <<< "$usedConceptIds"
+## generate unique used program ids from all forms
 usedProgramIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ','))"
+IFS=',' read -r -a usedProgramIdsArr <<< "$usedProgramIds"
+## generate unique used identifiers
+usedIdentifierIds="$(prepareReadableDisplay $(extractExpressionValues $form_expressions_file 'patient.getPatientIdentifier(' ',' ')'))"
+IFS=',' read -r -a usedIdentifierIdsArr <<< "$usedIdentifierIds"
+## generate unique used person attributes
+usedPersonAttributes="$(prepareReadableDisplay $(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')'))"
+IFS=',' read -r -a usedPersonAttributesArr <<< "$usedPersonAttributes"
 
 
-## clean entire console to only show metadata usage
-clear && printf '\e[3J'
+## prepare usage output
+rm -r usage;mkdir usage
+
+## TODO support used metadata like concepts in the tree like within programs
 
 ## generate unique non used concept ids from all form's questions and answers
-printf "\n\nConcept Ids used in HTML Forms# \n$usedConceptsIds\n"
-nonUsedConceptsIds=$(echo "SELECT concept_id FROM concept WHERE concept_id NOT IN ($usedConceptsIds)" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N)
-nonUsedConceptsIds=`echo "$nonUsedConceptsIds" | tr '\n' ','`
-## exclude last trailing ',' character and show non used conceptIds
-nonUsedConceptsIds="${nonUsedConceptsIds%?}"
-printf "\nConcept Ids Not used in HTML Forms# \n$nonUsedConceptsIds\n"
+usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN ($usedConceptIds) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+printf "\n${#usedConceptIdsArr[@]} Used Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
+notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN ($usedConceptIds) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
 
 ## print separator horizontal line
-printf '\n\n%s\n\n' _____________________________________________________________________________________________________
+printf '\n%s\n' _________________________________________________________________________________
 
 ##generate unique non used program ids from all forms
-printf "\n\nProgram Ids used in HTML Forms# \n$usedProgramIds\n"
-nonUsedProgramIds=$(echo "SELECT program_id FROM program WHERE program_id NOT IN ($usedProgramIds)" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N)
-nonUsedProgramIds=`echo "$nonUsedProgramIds" | tr '\n' ','`
-## exclude last trailing ',' character and show non used programIds
-nonUsedProgramIds="${nonUsedProgramIds%?}"
-printf "\nProgram Ids Not used in HTML Forms# \n$nonUsedProgramIds\n"
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\n${#usedProgramIdsArr[@]} Used Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________
+
+##generate unique non used identifiers from all forms
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\n${#usedIdentifierIdsArr[@]} Used Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________
 
+##generate unique non used person attribute types from all forms
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\n${#usedPersonAttributesArr[@]} Used person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"

From 7290acc4e28418e44aa464b50928471f81a69f30 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Fri, 31 May 2019 16:20:04 +0300
Subject: [PATCH 04/21] MEPTS-207: Supported used and not used roles

---
 metadataUsage/openmrsMetadataUsage.sh | 47 ++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 6a7037b..2e4b50e 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -6,11 +6,13 @@ MYSQL_CMD=$1
 MYSQL_USER=$2
 MYSQL_OPTS=$3
 MYSQL_DB=$4
-form_concept_grouping_file="data/form_concept_grouping.txt"
-form_concept_questions_file="data/form_concept_questions.txt"
-form_concept_answers_file="data/form_concept_answers.txt"
-form_programs_file="data/form_programs.txt"
-form_expressions_file="data/form_expressions.txt"
+form_concept_grouping_file="$PWD/data/form_concept_grouping.txt"
+form_concept_questions_file="$PWD/data/form_concept_questions.txt"
+form_concept_answers_file="$PWD/data/form_concept_answers.txt"
+form_programs_file="$PWD/data/form_programs.txt"
+form_expressions_file="$PWD/data/form_expressions.txt"
+form_locations_file="$PWD/data/form_locations.txt"
+form_roles_file="$PWD/data/form_roles.txt"
 form_concept_used_csv="$PWD/usage/form_used_concepts.csv"
 form_concept_not_used_csv="$PWD/usage/form_not_used_concepts.csv"
 form_programs_used_csv="$PWD/usage/form_used_programs.csv"
@@ -19,6 +21,8 @@ form_identifiers_used_csv="$PWD/usage/form_used_identifiers.csv"
 form_identifiers_not_used_csv="$PWD/usage/form_not_used_identifiers.csv"
 form_person_attributes_used_csv="$PWD/usage/form_used_person_attributes.csv"
 form_person_attributes_not_used_csv="$PWD/usage/form_not_used_person_attributes.csv"
+form_used_roles_csv="$PWD/usage/form_used_roles.csv"
+form_not_used_roles_csv="$PWD/usage/form_not_used_roles.csv"
 
 
 ## require mysql command or location, user and database
@@ -49,14 +53,14 @@ do
   ## clean up the xml data file and save each form named after its form uuid
   xmlData=$(printf "$xmlData")
   ## save xmlData onto a respective form specific file replacing back %
-  echo "${xmlData//PCNT/%}" > "data/form_$uuid.xml"
+  echo "${xmlData//PCNT/%}" > "$PWD/data/form_$uuid.xml"
 done
 printf "\nFinished extracting forms from the database\n"
 
 
 ## extracting concepts from forms
 printf "\nStarting to extract metadata from forms\n"
-for form_file in data/form_*.xml
+for form_file in $PWD/data/form_*.xml
 do
   ## simulate loading or progress bar
   echo -n "."
@@ -70,24 +74,31 @@ do
   echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
   ## extract expressions
   echo -n 'cat //*/@expression' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_expressions_file
+  
+  ## extract roles
+  echo -n 'cat //*/@role' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_roles_file
 done
 printf "\nFinished extracting metadata from forms\n"
 
 
-## first argument; file, second argument; separator
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________
+
+
+## first argument; file, second argument; separator, third argument; wrapper
 combineAllLinesFromFileIntoOneWithSeparator () {
   lines=""
   while IFS= read -r line
   do
     if [ "$line" != "" ]; then
-      lines="$lines$2$line"
+      lines="$lines$2$3$line$3"
     fi
   done < "$1"
   ## echo or 'return' excluding first trailing character of separator
   echo "${lines#?}"
 }
 
-## combine all ids into comma separated string
+## combine all objects into comma separated string
 prepareReadableDisplay () {
   IFS=',' read -r -a metadata <<< "$1"
   metadata=($(printf "%s\n" "${metadata[@]}" | sort -u | tr '\n' ' '))
@@ -124,7 +135,9 @@ IFS=',' read -r -a usedIdentifierIdsArr <<< "$usedIdentifierIds"
 ## generate unique used person attributes
 usedPersonAttributes="$(prepareReadableDisplay $(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')'))"
 IFS=',' read -r -a usedPersonAttributesArr <<< "$usedPersonAttributes"
-
+## generate unique used roles from all forms
+usedRoles="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"'))"
+IFS=',' read -r -a usedRolesArr <<< "$usedRoles"
 
 ## prepare usage output
 rm -r usage;mkdir usage
@@ -163,3 +176,15 @@ echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_at
 printf "\n${#usedPersonAttributesArr[@]} Used person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________
+
+##generate unique non used person attribute types from all forms
+echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\n${#usedRolesArr[@]} Used roles in HTML Forms saved in# \n$form_used_roles_csv\n"
+echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________

From c7cdcab388e9929e08961ba3bb9ae2ece8df10dc Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Fri, 31 May 2019 18:08:57 +0300
Subject: [PATCH 05/21] MEPTS-207: Added used global properties form usages

---
 metadataUsage/openmrsMetadataUsage.sh | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 2e4b50e..b7e9628 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -13,6 +13,7 @@ form_programs_file="$PWD/data/form_programs.txt"
 form_expressions_file="$PWD/data/form_expressions.txt"
 form_locations_file="$PWD/data/form_locations.txt"
 form_roles_file="$PWD/data/form_roles.txt"
+form_defaults_file="$PWD/data/form_defaults.txt"
 form_concept_used_csv="$PWD/usage/form_used_concepts.csv"
 form_concept_not_used_csv="$PWD/usage/form_not_used_concepts.csv"
 form_programs_used_csv="$PWD/usage/form_used_programs.csv"
@@ -23,6 +24,7 @@ form_person_attributes_used_csv="$PWD/usage/form_used_person_attributes.csv"
 form_person_attributes_not_used_csv="$PWD/usage/form_not_used_person_attributes.csv"
 form_used_roles_csv="$PWD/usage/form_used_roles.csv"
 form_not_used_roles_csv="$PWD/usage/form_not_used_roles.csv"
+form_used_global_properties_csv="$PWD/usage/form_used_global_properties.csv"
 
 
 ## require mysql command or location, user and database
@@ -74,9 +76,10 @@ do
   echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
   ## extract expressions
   echo -n 'cat //*/@expression' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_expressions_file
-  
   ## extract roles
   echo -n 'cat //*/@role' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_roles_file
+  ## extract roles
+  echo -n 'cat //*/@default' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_defaults_file
 done
 printf "\nFinished extracting metadata from forms\n"
 
@@ -106,17 +109,18 @@ prepareReadableDisplay () {
   echo "$metadataIds"
 }
 
-## extract expression values; first argument;file, second argument; expression prefix, third argument; separator, forth; toBeRemoved
+## extract expression values; arguments: first; file, second; expression prefix, third argument; separator, forth; toBeRemoved, fifth: wrapper
 extractExpressionValues () {
   lines=""
   while IFS= read -r line
   do
   	## starts with expression prefix
+  	
     if [[ "$line" = "$2"* ]]; then
       ## get value in between () within expression value
       value="${line//$2/}"
       value="${value//$4/}"
-      lines="$lines$3$value"
+      lines="$lines$3$5$value$5"
     fi
   done < "$1"
   ## echo or 'return' excluding first trailing character of separator
@@ -138,13 +142,15 @@ IFS=',' read -r -a usedPersonAttributesArr <<< "$usedPersonAttributes"
 ## generate unique used roles from all forms
 usedRoles="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"'))"
 IFS=',' read -r -a usedRolesArr <<< "$usedRoles"
+## generate unique used global properties from all forms
+usedGlobalProperties="$(prepareReadableDisplay $(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"'))"
+IFS=',' read -r -a usedGlobalPropertiesArr <<< "$usedGlobalProperties"
 
 ## prepare usage output
 rm -r usage;mkdir usage
 
 ## TODO support used metadata like concepts in the tree like within programs
 
-## generate unique non used concept ids from all form's questions and answers
 usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN ($usedConceptIds) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
 printf "\n${#usedConceptIdsArr[@]} Used Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
 notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN ($usedConceptIds) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
@@ -153,7 +159,6 @@ printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique non used program ids from all forms
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\n${#usedProgramIdsArr[@]} Used Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
@@ -162,7 +167,6 @@ printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_cs
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique non used identifiers from all forms
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\n${#usedIdentifierIdsArr[@]} Used Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
@@ -171,7 +175,6 @@ printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_u
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique non used person attribute types from all forms
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\n${#usedPersonAttributesArr[@]} Used person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
@@ -180,7 +183,6 @@ printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique non used person attribute types from all forms
 echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\n${#usedRolesArr[@]} Used roles in HTML Forms saved in# \n$form_used_roles_csv\n"
 echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
@@ -188,3 +190,10 @@ printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
 
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
+
+##generate unique used global properties from all forms
+echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\n${#usedGlobalPropertiesArr[@]} Used Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________

From 0ccdd576a628735e8368e4f1a67c06f8e147843e Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Thu, 6 Jun 2019 20:58:04 +0300
Subject: [PATCH 06/21] MEPTS-207: accepting uuids as ids for concepts

---
 metadataUsage/openmrsMetadataUsage.sh | 62 ++++++++++++++-------------
 1 file changed, 33 insertions(+), 29 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index b7e9628..1f5b409 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -94,19 +94,24 @@ combineAllLinesFromFileIntoOneWithSeparator () {
   while IFS= read -r line
   do
     if [ "$line" != "" ]; then
-      lines="$lines$2$3$line$3"
+      line="${line//$2/$3$2$3}"
+      if [ "$lines" == "" ]; then
+        lines="$3$line$3"
+      else
+        lines="$lines$2$3$line$3"
+      fi
     fi
   done < "$1"
   ## echo or 'return' excluding first trailing character of separator
-  echo "${lines#?}"
+  echo "$lines"
 }
 
-## combine all objects into comma separated string
+## combine all objects into comma separated string, first argument; string to prepare, two; wrapper
 prepareReadableDisplay () {
-  IFS=',' read -r -a metadata <<< "$1"
-  metadata=($(printf "%s\n" "${metadata[@]}" | sort -u | tr '\n' ' '))
-  metadataIds=$(IFS=$','; echo "${metadata[*]}")
-  echo "$metadataIds"
+  IFS=',' read -r -a meta <<< "$1"
+  meta=($(printf "%s\n" "${meta[@]}" | sort -u | tr '\n' ' '))
+  metaIds=$(IFS=$','; echo "${meta[*]}")
+  echo "$metaIds"
 }
 
 ## extract expression values; arguments: first; file, second; expression prefix, third argument; separator, forth; toBeRemoved, fifth: wrapper
@@ -120,47 +125,46 @@ extractExpressionValues () {
       ## get value in between () within expression value
       value="${line//$2/}"
       value="${value//$4/}"
-      lines="$lines$3$5$value$5"
+      if [ "$lines" == "" ]; then
+        lines="$5$value$5"
+      else
+        lines="$lines$3$5$value$5"
+      fi  
     fi
   done < "$1"
   ## echo or 'return' excluding first trailing character of separator
-  echo "${lines#?}"
+  echo "$lines"
 }
 
 ## generate unique used concept ids from all forms' questions and answers
-usedConceptIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ','),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ','))"
-IFS=',' read -r -a usedConceptIdsArr <<< "$usedConceptIds"
+
+usedConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',' '"')"
 ## generate unique used program ids from all forms
-usedProgramIds="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ','))"
-IFS=',' read -r -a usedProgramIdsArr <<< "$usedProgramIds"
+usedProgramIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ',')"
 ## generate unique used identifiers
-usedIdentifierIds="$(prepareReadableDisplay $(extractExpressionValues $form_expressions_file 'patient.getPatientIdentifier(' ',' ')'))"
-IFS=',' read -r -a usedIdentifierIdsArr <<< "$usedIdentifierIds"
+usedIdentifierIds="$(extractExpressionValues $form_expressions_file 'patient.getPatientIdentifier(' ',' ')')"
 ## generate unique used person attributes
-usedPersonAttributes="$(prepareReadableDisplay $(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')'))"
-IFS=',' read -r -a usedPersonAttributesArr <<< "$usedPersonAttributes"
+usedPersonAttributes="$(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')')"
 ## generate unique used roles from all forms
-usedRoles="$(prepareReadableDisplay $(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"'))"
-IFS=',' read -r -a usedRolesArr <<< "$usedRoles"
+usedRoles="$(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"')"
 ## generate unique used global properties from all forms
-usedGlobalProperties="$(prepareReadableDisplay $(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"'))"
-IFS=',' read -r -a usedGlobalPropertiesArr <<< "$usedGlobalProperties"
+usedGlobalProperties="$(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"')"
 
 ## prepare usage output
 rm -r usage;mkdir usage
 
 ## TODO support used metadata like concepts in the tree like within programs
 
-usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN ($usedConceptIds) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
-printf "\n${#usedConceptIdsArr[@]} Used Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
-notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN ($usedConceptIds) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
+notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
 printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
 
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
 
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\n${#usedProgramIdsArr[@]} Used Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
+printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
 
@@ -168,7 +172,7 @@ printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_cs
 printf '\n%s\n' _________________________________________________________________________________
 
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\n${#usedIdentifierIdsArr[@]} Used Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
+printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
 
@@ -176,7 +180,7 @@ printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_u
 printf '\n%s\n' _________________________________________________________________________________
 
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\n${#usedPersonAttributesArr[@]} Used person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
+printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
 echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
 
@@ -184,7 +188,7 @@ printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person
 printf '\n%s\n' _________________________________________________________________________________
 
 echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\n${#usedRolesArr[@]} Used roles in HTML Forms saved in# \n$form_used_roles_csv\n"
+printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n"
 echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
 printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
 
@@ -193,7 +197,7 @@ printf '\n%s\n' ________________________________________________________________
 
 ##generate unique used global properties from all forms
 echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\n${#usedGlobalPropertiesArr[@]} Used Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
+printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
 
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________

From 70b356f0db4dfe7812a4a0ad8190973be4dc449d Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Tue, 11 Jun 2019 14:49:13 +0300
Subject: [PATCH 07/21] MEPTS-207: added locations usage and introduced failed
 logging to allow manual usage intervention

---
 .gitignore                            |  3 +-
 metadataUsage/openmrsMetadataUsage.sh | 70 +++++++++++++++++++++++++--
 2 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index b298ec3..9be54d0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 metadataUsage/data/
-metadataUsage/usage/
\ No newline at end of file
+metadataUsage/usage/
+metadataUsage/failed/
\ No newline at end of file
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 1f5b409..8ec15fb 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 ## This openmrs bash script helps extract details about 
 
@@ -14,6 +14,8 @@ form_expressions_file="$PWD/data/form_expressions.txt"
 form_locations_file="$PWD/data/form_locations.txt"
 form_roles_file="$PWD/data/form_roles.txt"
 form_defaults_file="$PWD/data/form_defaults.txt"
+##contains form_file: advise to fix it
+form_failed="$PWD/failed/form_failed.txt"
 form_concept_used_csv="$PWD/usage/form_used_concepts.csv"
 form_concept_not_used_csv="$PWD/usage/form_not_used_concepts.csv"
 form_programs_used_csv="$PWD/usage/form_used_programs.csv"
@@ -25,6 +27,7 @@ form_person_attributes_not_used_csv="$PWD/usage/form_not_used_person_attributes.
 form_used_roles_csv="$PWD/usage/form_used_roles.csv"
 form_not_used_roles_csv="$PWD/usage/form_not_used_roles.csv"
 form_used_global_properties_csv="$PWD/usage/form_used_global_properties.csv"
+form_locations_used_csv="$PWD/usage/form_used_locations.csv"
 
 
 ## require mysql command or location, user and database
@@ -37,11 +40,12 @@ fi
 
 ## wait for mysql password if any
 echo -n "Enter Mysql password": 
-read -s MYSQL_PASS
+read -rs MYSQL_PASS
 echo
 ## initialise data location
 rm -r data;mkdir data
-
+mkdir failed
+rm $form_failed
 
 ## extract form uuids and loop through them
 printf "\nStarting to extract forms from the database\n"
@@ -60,9 +64,54 @@ done
 printf "\nFinished extracting forms from the database\n"
 
 
+## extract values from a node, arguments; xml file, node, comma separated attributes whose values to extract
+extractNodeValues () {
+  values=""
+  IFS=',' read -r -a attributes <<< "$3"
+  for a in "${attributes[@]}"
+  do
+    if grep -q $a $1 && grep -q $2 $1; then
+      if [ "$values" != "" ]; then
+        values="$values,"
+      fi
+      node=$(xmllint --xpath '//*/@'$a - <<< "$(find $1 -type f -print | xargs grep $2)")
+      if [ $? -gt 0 ]; then
+  	    ## some error happened, pass this file into failed category
+  	    printf "\nEnsure $2:$a node in data/${1##*/} exist on separate lines in failed/${1##*/}" >> $form_failed
+  	    cp $1 $PWD/failed/${1##*/}
+  	  fi
+  	  values=$values$(cut -d '=' -f 2 <<< $node | sed -e 's/^"//' -e 's/"$//')
+  	fi
+  done
+  echo $values
+}
+
+
+## get values replacing global properties. arguments; values text with gp
+getValuesReplacingGlobalProperties () {
+  finalValues=""
+  if [[ "$1" == *"GlobalProperty:default_location"* ]]; then
+    IFS=',' read -r -a values <<< "$1"
+    for v in "${values[@]}"
+    do
+      if [ "$finalValues" != "" ]; then
+        finalValues="$finalValues,"
+      fi
+      if [[ "$v" = "GlobalProperty:default_location"* ]]; then
+        finalValues=$finalValues$(echo "SELECT GROUP_CONCAT(location_id) FROM location WHERE name = (select property_value from global_property where property = 'default_location')" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s)
+      else
+        finalValues=$finalValues$v
+      fi
+    done
+    echo $finalValues
+  else
+    echo $1
+  fi
+}
+
 ## extracting concepts from forms
 printf "\nStarting to extract metadata from forms\n"
-for form_file in $PWD/data/form_*.xml
+for form_file in $PWD/data/form_*.xml $PWD/failed/form_*.xml
 do
   ## simulate loading or progress bar
   echo -n "."
@@ -80,6 +129,8 @@ do
   echo -n 'cat //*/@role' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_roles_file
   ## extract roles
   echo -n 'cat //*/@default' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_defaults_file
+  ## extract encounterLocations
+  echo $(getValuesReplacingGlobalProperties $(extractNodeValues $form_file 'encounterLocation' 'default,order')) >> $form_locations_file
 done
 printf "\nFinished extracting metadata from forms\n"
 
@@ -121,7 +172,7 @@ extractExpressionValues () {
   do
   	## starts with expression prefix
   	
-    if [[ "$line" = "$2"* ]]; then
+    if [[ "$line" == "$2"* ]]; then
       ## get value in between () within expression value
       value="${line//$2/}"
       value="${value//$4/}"
@@ -149,6 +200,8 @@ usedPersonAttributes="$(extractExpressionValues $form_expressions_file 'patient.
 usedRoles="$(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"')"
 ## generate unique used global properties from all forms
 usedGlobalProperties="$(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"')"
+## generate unique locations from all forms
+usedLocations="$(combineAllLinesFromFileIntoOneWithSeparator $form_locations_file ',')"
 
 ## prepare usage output
 rm -r usage;mkdir usage
@@ -201,3 +254,10 @@ printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_pro
 
 ## print separator horizontal line
 printf '\n%s\n' _________________________________________________________________________________
+
+##generate unique used global properties from all forms
+echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n"
+
+## print separator horizontal line
+printf '\n%s\n' _________________________________________________________________________________

From 07bbc7b7d435b0f7cff4b6280abbd46262bdcbbf Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Tue, 11 Jun 2019 18:20:02 +0300
Subject: [PATCH 08/21] MEPTS-207: added metadataUsage readme

---
 metadataUsage/README.md | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 metadataUsage/README.md

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
new file mode 100644
index 0000000..d77db69
--- /dev/null
+++ b/metadataUsage/README.md
@@ -0,0 +1,18 @@
+# Generate Metadata usage statistics
+> This Script helps to generate csv files including used metadata objects currently; concepts, programs, patient identifier types, person attribute types, roles, global properties, locations
+
+### Running the script
+>  `./openmrsMetadataUsage.sh ~/Projects/jembi/esaude/openmrs-standalone-1.11.5_2/database/bin/./mysql openmrs '--socket=/tmp/esaude3.sock --max_allowed_packet=96M' openmrs`
+Enter Mysql password.
+
+### Results
+> If running it returned no errors quiting execution, then three folders should have been created 3 folders; `data, failed, usage`
+```
+   drwxr-xr-x  ... data
+   drwxr-xr-x  ... failed
+   -rwxr-xr-x@ ... openmrsMetadataUsage.sh
+   drwxr-xr-x  ... usage
+   ```
+   `data`: contains xml forms export from the database as well as metadata fields extracts
+   `failed`: contains failed xml forms which need human intervention, the details of what failed in text files
+   `usage`: contains csv output per exported metadata objects

From 8e429ac25e7beee93e8842a24bb2bf753bcb42a3 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Thu, 13 Jun 2019 12:31:46 +0300
Subject: [PATCH 09/21] MEPTS-207: added external  dependencies to readme

---
 metadataUsage/README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index d77db69..6315ec2 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -1,5 +1,9 @@
 # Generate Metadata usage statistics
-> This Script helps to generate csv files including used metadata objects currently; concepts, programs, patient identifier types, person attribute types, roles, global properties, locations
+> This OpenMRS Script helps to generate csv files including used metadata objects currently used in eSaude environments; concepts, programs, patient identifier types, person attribute types, roles, global properties, locations
+
+
+### Setup
+> This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, please manually install those and proceed
 
 ### Running the script
 >  `./openmrsMetadataUsage.sh ~/Projects/jembi/esaude/openmrs-standalone-1.11.5_2/database/bin/./mysql openmrs '--socket=/tmp/esaude3.sock --max_allowed_packet=96M' openmrs`

From a8faa08c1aa3dd55e612b3e59b514bb2f9974cd4 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 13:53:42 +0300
Subject: [PATCH 10/21] MEPTS-207: preparing for running for more than one
 database

---
 .gitignore                            |   3 +-
 metadataUsage/README.md               |  11 +-
 metadataUsage/openmrsMetadataUsage.sh | 308 +++++++++++++-------------
 metadataUsage/params.json             |  11 +
 4 files changed, 176 insertions(+), 157 deletions(-)
 create mode 100644 metadataUsage/params.json

diff --git a/.gitignore b/.gitignore
index 9be54d0..e17ec37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 metadataUsage/data/
 metadataUsage/usage/
-metadataUsage/failed/
\ No newline at end of file
+metadataUsage/failed/
+metadataUsage/test.sh
\ No newline at end of file
diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index 6315ec2..398672a 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -1,22 +1,21 @@
 # Generate Metadata usage statistics
-> This OpenMRS Script helps to generate csv files including used metadata objects currently used in eSaude environments; concepts, programs, patient identifier types, person attribute types, roles, global properties, locations
+> This OpenMRS Script helps to generate csv files including used metadata objects currently used in eSaude environments; concepts, programs, patient identifier types, person attribute types, roles, global properties, locations. The script can be pointed at-least one database through params.json which contains MySQL connection details from which the separated usages are obtained
 
 
 ### Setup
-> This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, please manually install those and proceed
+> This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, `jq`, `bc` etc, please manually install those and proceed
 
 ### Running the script
->  `./openmrsMetadataUsage.sh ~/Projects/jembi/esaude/openmrs-standalone-1.11.5_2/database/bin/./mysql openmrs '--socket=/tmp/esaude3.sock --max_allowed_packet=96M' openmrs`
-Enter Mysql password.
+>  Configure mysql instances to extract metadata usages from in params.json and run; `./openmrsMetadataUsage.sh`
 
 ### Results
-> If running it returned no errors quiting execution, then three folders should have been created 3 folders; `data, failed, usage`
+> If running it returned no errors while quitting execution, then three folders should have been created 3 folders; `data, failed, usage`
 ```
    drwxr-xr-x  ... data
    drwxr-xr-x  ... failed
    -rwxr-xr-x@ ... openmrsMetadataUsage.sh
    drwxr-xr-x  ... usage
    ```
-   `data`: contains xml forms export from the database as well as metadata fields extracts
+   `data`: contains xml forms export from the database as well as metadata field extracts
    `failed`: contains failed xml forms which need human intervention, the details of what failed in text files
    `usage`: contains csv output per exported metadata objects
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 8ec15fb..46f9664 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -2,69 +2,13 @@
 
 ## This openmrs bash script helps extract details about 
 
-MYSQL_CMD=$1
-MYSQL_USER=$2
-MYSQL_OPTS=$3
-MYSQL_DB=$4
-form_concept_grouping_file="$PWD/data/form_concept_grouping.txt"
-form_concept_questions_file="$PWD/data/form_concept_questions.txt"
-form_concept_answers_file="$PWD/data/form_concept_answers.txt"
-form_programs_file="$PWD/data/form_programs.txt"
-form_expressions_file="$PWD/data/form_expressions.txt"
-form_locations_file="$PWD/data/form_locations.txt"
-form_roles_file="$PWD/data/form_roles.txt"
-form_defaults_file="$PWD/data/form_defaults.txt"
-##contains form_file: advise to fix it
-form_failed="$PWD/failed/form_failed.txt"
-form_concept_used_csv="$PWD/usage/form_used_concepts.csv"
-form_concept_not_used_csv="$PWD/usage/form_not_used_concepts.csv"
-form_programs_used_csv="$PWD/usage/form_used_programs.csv"
-form_programs_not_used_csv="$PWD/usage/form_not_used_programs.csv"
-form_identifiers_used_csv="$PWD/usage/form_used_identifiers.csv"
-form_identifiers_not_used_csv="$PWD/usage/form_not_used_identifiers.csv"
-form_person_attributes_used_csv="$PWD/usage/form_used_person_attributes.csv"
-form_person_attributes_not_used_csv="$PWD/usage/form_not_used_person_attributes.csv"
-form_used_roles_csv="$PWD/usage/form_used_roles.csv"
-form_not_used_roles_csv="$PWD/usage/form_not_used_roles.csv"
-form_used_global_properties_csv="$PWD/usage/form_used_global_properties.csv"
-form_locations_used_csv="$PWD/usage/form_used_locations.csv"
-
-
-## require mysql command or location, user and database
-if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_OPTS" == "" ] || [ "$MYSQL_DB" == "" ]; then
-  printf "\nUsage;./openmrsMetadataUsage.sh mysql_command_or_location openmrs_user mysql_opts openmrs_db. Replace any blank values with ''"
-  printf "\nExample;./openmrsMetadataUsage.sh mysql openmrs_user '--socket=/tmp/omrs.sock --max_allowed_packet=96M' openmrs_db"
-  exit 1
+## ensure params.json exists
+if [ ! -f $PWD/params.json ]; then
+    echo "Ensure $PWD/params.json exists with the right mysql database connection parameters!"
+    exit 1
 fi
 
-
-## wait for mysql password if any
-echo -n "Enter Mysql password": 
-read -rs MYSQL_PASS
-echo
-## initialise data location
-rm -r data;mkdir data
-mkdir failed
-rm $form_failed
-
-## extract form uuids and loop through them
-printf "\nStarting to extract forms from the database\n"
-$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
-do
-  ## simulate loading or progress bar
-  echo -n "."
-  xmlData=`$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
-  ## printf evaluates % character, let us replace it so it's not treated special
-  xmlData="${xmlData//%/PCNT}"
-  ## clean up the xml data file and save each form named after its form uuid
-  xmlData=$(printf "$xmlData")
-  ## save xmlData onto a respective form specific file replacing back %
-  echo "${xmlData//PCNT/%}" > "$PWD/data/form_$uuid.xml"
-done
-printf "\nFinished extracting forms from the database\n"
-
-
-## extract values from a node, arguments; xml file, node, comma separated attributes whose values to extract
+## extract values from a node, arguments; xml file, node, comma separated attributes whose values to extract, form_failed file path
 extractNodeValues () {
   values=""
   IFS=',' read -r -a attributes <<< "$3"
@@ -77,7 +21,7 @@ extractNodeValues () {
       node=$(xmllint --xpath '//*/@'$a - <<< "$(find $1 -type f -print | xargs grep $2)")
       if [ $? -gt 0 ]; then
   	    ## some error happened, pass this file into failed category
-  	    printf "\nEnsure $2:$a node in data/${1##*/} exist on separate lines in failed/${1##*/}" >> $form_failed
+  	    printf "\nEnsure $2:$a node in data/${1##*/} exist on separate lines in failed/${1##*/}" >> $4
   	    cp $1 $PWD/failed/${1##*/}
   	  fi
   	  values=$values$(cut -d '=' -f 2 <<< $node | sed -e 's/^"//' -e 's/"$//')
@@ -86,7 +30,6 @@ extractNodeValues () {
   echo $values
 }
 
-
 ## get values replacing global properties. arguments; values text with gp
 getValuesReplacingGlobalProperties () {
   finalValues=""
@@ -109,36 +52,6 @@ getValuesReplacingGlobalProperties () {
   fi
 }
 
-## extracting concepts from forms
-printf "\nStarting to extract metadata from forms\n"
-for form_file in $PWD/data/form_*.xml $PWD/failed/form_*.xml
-do
-  ## simulate loading or progress bar
-  echo -n "."
-  ## extract concept question ids
-  echo -n 'cat //*/@groupingConceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_grouping_file
-  ## extract concept question ids
-  echo -n 'cat //*/@conceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_questions_file
-  ## extract concept answer ids
-  echo -n 'cat //*/@answerConceptIds' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_answers_file
-  ## extract program ids
-  echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
-  ## extract expressions
-  echo -n 'cat //*/@expression' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_expressions_file
-  ## extract roles
-  echo -n 'cat //*/@role' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_roles_file
-  ## extract roles
-  echo -n 'cat //*/@default' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_defaults_file
-  ## extract encounterLocations
-  echo $(getValuesReplacingGlobalProperties $(extractNodeValues $form_file 'encounterLocation' 'default,order')) >> $form_locations_file
-done
-printf "\nFinished extracting metadata from forms\n"
-
-
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
-
-
 ## first argument; file, second argument; separator, third argument; wrapper
 combineAllLinesFromFileIntoOneWithSeparator () {
   lines=""
@@ -187,77 +100,172 @@ extractExpressionValues () {
   echo "$lines"
 }
 
-## generate unique used concept ids from all forms' questions and answers
 
-usedConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',' '"')"
-## generate unique used program ids from all forms
-usedProgramIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ',')"
-## generate unique used identifiers
-usedIdentifierIds="$(extractExpressionValues $form_expressions_file 'patient.getPatientIdentifier(' ',' ')')"
-## generate unique used person attributes
-usedPersonAttributes="$(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')')"
-## generate unique used roles from all forms
-usedRoles="$(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"')"
-## generate unique used global properties from all forms
-usedGlobalProperties="$(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"')"
-## generate unique locations from all forms
-usedLocations="$(combineAllLinesFromFileIntoOneWithSeparator $form_locations_file ',')"
+## initialise data location
+rm -r data;mkdir data
+mkdir failed
+  
+for ((dbConnIndex=0; dbConnIndex<$(jq ".mysqlDbConnections | length" params.json); dbConnIndex++ ))
+do
+  MYSQL_CMD=$(jq ".mysqlDbConnections[$dbConnIndex].binaryLocation" params.json)
+  MYSQL_USER=$(jq ".mysqlDbConnections[$dbConnIndex].username" params.json)
+  MYSQL_OPTS=$(jq ".mysqlDbConnections[$dbConnIndex].options" params.json)
+  MYSQL_DB=$(jq ".mysqlDbConnections[$dbConnIndex].database" params.json)
+  MYSQL_PASS=$(jq ".mysqlDbConnections[$dbConnIndex].password" params.json)
+  dbConnCount=$(echo "$dbConnIndex + 1" | bc)
+  
+  form_concept_grouping_file="$PWD/data/$dbConnCount_form_concept_grouping.txt"
+  form_concept_questions_file="$PWD/data/$dbConnCount_form_concept_questions.txt"
+  form_concept_answers_file="$PWD/data/$dbConnCount_form_concept_answers.txt"
+  form_programs_file="$PWD/data/$dbConnCount_form_programs.txt"
+  form_expressions_file="$PWD/data/$dbConnCount_form_expressions.txt"
+  form_locations_file="$PWD/data/$dbConnCount_form_locations.txt"
+  form_roles_file="$PWD/data/$dbConnCount_form_roles.txt"
+  form_defaults_file="$PWD/data/$dbConnCount_form_defaults.txt"
+  ##contains form_file: advise to fix it
+  form_failed="$PWD/failed/$dbConnCount_form_failed.txt"
+  form_concept_used_csv="$PWD/usage/$dbConnCount_form_used_concepts.csv"
+  form_concept_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_concepts.csv"
+  form_programs_used_csv="$PWD/usage/$dbConnCount_form_used_programs.csv"
+  form_programs_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_programs.csv"
+  form_identifiers_used_csv="$PWD/usage/$dbConnCount_form_used_identifiers.csv"
+  form_identifiers_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_identifiers.csv"
+  form_person_attributes_used_csv="$PWD/usage/$dbConnCount_form_used_person_attributes.csv"
+  form_person_attributes_not_used_csv="$PWD/$dbConnCount_usage/form_not_used_person_attributes.csv"
+  form_used_roles_csv="$PWD/usage/$dbConnCount_form_used_roles.csv"
+  form_not_used_roles_csv="$PWD/usage/$dbConnCount_form_not_used_roles.csv"
+  form_used_global_properties_csv="$PWD/usage/$dbConnCount_form_used_global_properties.csv"
+  form_locations_used_csv="$PWD/usage/$dbConnCount_form_used_locations.csv"
+
+
+  ## require mysql command or location, user and database
+  if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_OPTS" == "" ] || [ "$MYSQL_DB" == "" ]; then
+    printf "\nUsage; ./openmrsMetadataUsage.sh"
+    exit 1
+  fi
+  
+  
+  rm $form_failed
+
+  ## extract form uuids and loop through them
+  printf "\nStarting to extract forms from the $dbConnCount: $MYSQL_DB database\n"
+  $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
+  do
+    ## simulate loading or progress bar
+    echo -n "."
+    xmlData=`$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
+    ## printf evaluates % character, let us replace it so it's not treated special
+    xmlData="${xmlData//%/PCNT}"
+    ## clean up the xml data file and save each form named after its form uuid
+    xmlData=$(printf "$xmlData")
+    ## save xmlData onto a respective form specific file replacing back %
+    echo "${xmlData//PCNT/%}" > "$PWD/data/$dbConnCount_form_$uuid.xml"
+  done
+  printf "\nFinished extracting forms from the $dbConnCount: $MYSQL_DB database\n"
+
+
+  ## extracting concepts from forms
+  printf "\nStarting to extract metadata from forms\n"
+  for form_file in $PWD/data/$dbConnCount_form_*.xml $PWD/failed/$dbConnCount_form_*.xml
+  do
+    ## simulate loading or progress bar
+    echo -n "."
+    ## extract concept question ids
+    echo -n 'cat //*/@groupingConceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_grouping_file
+    ## extract concept question ids
+    echo -n 'cat //*/@conceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_questions_file
+    ## extract concept answer ids
+    echo -n 'cat //*/@answerConceptIds' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_answers_file
+    ## extract program ids
+    echo -n 'cat //*/@programId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_programs_file
+    ## extract expressions
+    echo -n 'cat //*/@expression' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_expressions_file
+    ## extract roles
+    echo -n 'cat //*/@role' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_roles_file
+    ## extract roles
+    echo -n 'cat //*/@default' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_defaults_file
+    ## extract encounterLocations
+    echo $(getValuesReplacingGlobalProperties $(extractNodeValues $form_file 'encounterLocation' 'default,order' $form_failed)) >> $form_locations_file
+  done
+  printf "\nFinished extracting metadata from forms\n"
+
+
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
+
+
+  ## generate unique used concept ids from all forms' questions and answers
+
+  usedConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',' '"')"
+  ## generate unique used program ids from all forms
+  usedProgramIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ',')"
+  ## generate unique used identifiers
+  usedIdentifierIds="$(extractExpressionValues $form_expressions_file 'patient.getPatientIdentifier(' ',' ')')"
+  ## generate unique used person attributes
+  usedPersonAttributes="$(extractExpressionValues $form_expressions_file 'patient.getAttribute(' ',' ')')"
+  ## generate unique used roles from all forms
+  usedRoles="$(combineAllLinesFromFileIntoOneWithSeparator $form_roles_file ',' '"')"
+  ## generate unique used global properties from all forms
+  usedGlobalProperties="$(extractExpressionValues $form_defaults_file 'GlobalProperty:' ',' '' '"')"
+  ## generate unique locations from all forms
+  usedLocations="$(combineAllLinesFromFileIntoOneWithSeparator $form_locations_file ',')"
 
-## prepare usage output
-rm -r usage;mkdir usage
+  ## prepare usage output
+  rm -r usage;mkdir usage
 
-## TODO support used metadata like concepts in the tree like within programs
+  ## TODO support used metadata like concepts in the tree like within programs
 
-usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
-printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
-notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
-printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
+  usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+  printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
+  notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+  printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n"
-echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
+  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n"
+  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique used global properties from all forms
-echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
+  ##generate unique used global properties from all forms
+  echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
 
-##generate unique used global properties from all forms
-echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
-printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n"
+  ##generate unique used global properties from all forms
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n"
 
-## print separator horizontal line
-printf '\n%s\n' _________________________________________________________________________________
+  ## print separator horizontal line
+  printf '\n%s\n' _________________________________________________________________________________
+done
\ No newline at end of file
diff --git a/metadataUsage/params.json b/metadataUsage/params.json
new file mode 100644
index 0000000..b08624d
--- /dev/null
+++ b/metadataUsage/params.json
@@ -0,0 +1,11 @@
+{
+  "mysqlDbConnections": [
+    {
+      "binaryLocation": "~/Desktop/./mysql",
+      "options": "--socket=/tmp/esaude3.sock --max_allowed_packet=96M",
+      "username": "openmrs",
+      "password": "test",
+      "database": "openmrs"
+    }
+  ]
+}
\ No newline at end of file

From 7a733e4918ceef809bb3474dcfb8d00922671ea9 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 13:59:25 +0300
Subject: [PATCH 11/21] MEPTS-207: added metadataUsage/params.json  to
 gitignore

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index e17ec37..e0e05bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 metadataUsage/data/
 metadataUsage/usage/
 metadataUsage/failed/
-metadataUsage/test.sh
\ No newline at end of file
+metadataUsage/test.sh
+metadataUsage/params.json
\ No newline at end of file

From c8023834410990179f1c886ccef88f268804927d Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 14:04:52 +0300
Subject: [PATCH 12/21] un tracking ignored metadataUsage/params.json

---
 metadataUsage/params.json | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 metadataUsage/params.json

diff --git a/metadataUsage/params.json b/metadataUsage/params.json
deleted file mode 100644
index b08624d..0000000
--- a/metadataUsage/params.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
-  "mysqlDbConnections": [
-    {
-      "binaryLocation": "~/Desktop/./mysql",
-      "options": "--socket=/tmp/esaude3.sock --max_allowed_packet=96M",
-      "username": "openmrs",
-      "password": "test",
-      "database": "openmrs"
-    }
-  ]
-}
\ No newline at end of file

From 5cd30de1ec6e9677a9f1b64067ebfddd269aa650 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 15:43:38 +0300
Subject: [PATCH 13/21] MEPTS-207: fixed previous commit of supporting running
 for more than one database

---
 metadataUsage/openmrsMetadataUsage.sh | 56 +++++++++++++--------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 46f9664..ce0f04f 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -107,35 +107,35 @@ mkdir failed
   
 for ((dbConnIndex=0; dbConnIndex<$(jq ".mysqlDbConnections | length" params.json); dbConnIndex++ ))
 do
-  MYSQL_CMD=$(jq ".mysqlDbConnections[$dbConnIndex].binaryLocation" params.json)
-  MYSQL_USER=$(jq ".mysqlDbConnections[$dbConnIndex].username" params.json)
-  MYSQL_OPTS=$(jq ".mysqlDbConnections[$dbConnIndex].options" params.json)
-  MYSQL_DB=$(jq ".mysqlDbConnections[$dbConnIndex].database" params.json)
-  MYSQL_PASS=$(jq ".mysqlDbConnections[$dbConnIndex].password" params.json)
+  MYSQL_CMD=$(jq -r ".mysqlDbConnections[$dbConnIndex].binaryLocation" params.json)
+  MYSQL_USER=$(jq -r ".mysqlDbConnections[$dbConnIndex].username" params.json)
+  MYSQL_OPTS=$(jq -r ".mysqlDbConnections[$dbConnIndex].options" params.json)
+  MYSQL_DB=$(jq -r ".mysqlDbConnections[$dbConnIndex].database" params.json)
+  MYSQL_PASS=$(jq -r ".mysqlDbConnections[$dbConnIndex].password" params.json)
   dbConnCount=$(echo "$dbConnIndex + 1" | bc)
   
-  form_concept_grouping_file="$PWD/data/$dbConnCount_form_concept_grouping.txt"
-  form_concept_questions_file="$PWD/data/$dbConnCount_form_concept_questions.txt"
-  form_concept_answers_file="$PWD/data/$dbConnCount_form_concept_answers.txt"
-  form_programs_file="$PWD/data/$dbConnCount_form_programs.txt"
-  form_expressions_file="$PWD/data/$dbConnCount_form_expressions.txt"
-  form_locations_file="$PWD/data/$dbConnCount_form_locations.txt"
-  form_roles_file="$PWD/data/$dbConnCount_form_roles.txt"
-  form_defaults_file="$PWD/data/$dbConnCount_form_defaults.txt"
+  form_concept_grouping_file="$PWD/data/$dbConnCount"_form_concept_grouping.txt
+  form_concept_questions_file="$PWD/data/$dbConnCount"_form_concept_questions.txt
+  form_concept_answers_file="$PWD/data/$dbConnCount"_form_concept_answers.txt
+  form_programs_file="$PWD/data/$dbConnCount"_form_programs.txt
+  form_expressions_file="$PWD/data/$dbConnCount"_form_expressions.txt
+  form_locations_file="$PWD/data/$dbConnCount"_form_locations.txt
+  form_roles_file="$PWD/data/$dbConnCount"_form_roles.txt
+  form_defaults_file="$PWD/data/$dbConnCount"_form_defaults.txt
   ##contains form_file: advise to fix it
-  form_failed="$PWD/failed/$dbConnCount_form_failed.txt"
-  form_concept_used_csv="$PWD/usage/$dbConnCount_form_used_concepts.csv"
-  form_concept_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_concepts.csv"
-  form_programs_used_csv="$PWD/usage/$dbConnCount_form_used_programs.csv"
-  form_programs_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_programs.csv"
-  form_identifiers_used_csv="$PWD/usage/$dbConnCount_form_used_identifiers.csv"
-  form_identifiers_not_used_csv="$PWD/usage/$dbConnCount_form_not_used_identifiers.csv"
-  form_person_attributes_used_csv="$PWD/usage/$dbConnCount_form_used_person_attributes.csv"
-  form_person_attributes_not_used_csv="$PWD/$dbConnCount_usage/form_not_used_person_attributes.csv"
-  form_used_roles_csv="$PWD/usage/$dbConnCount_form_used_roles.csv"
-  form_not_used_roles_csv="$PWD/usage/$dbConnCount_form_not_used_roles.csv"
-  form_used_global_properties_csv="$PWD/usage/$dbConnCount_form_used_global_properties.csv"
-  form_locations_used_csv="$PWD/usage/$dbConnCount_form_used_locations.csv"
+  form_failed="$PWD/failed/$dbConnCount"_form_failed.txt
+  form_concept_used_csv="$PWD/usage/$dbConnCount"_form_used_concepts.csv
+  form_concept_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_concepts.csv
+  form_programs_used_csv="$PWD/usage/$dbConnCount"_form_used_programs.csv
+  form_programs_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_programs.csv
+  form_identifiers_used_csv="$PWD/usage/$dbConnCount"_form_used_identifiers.csv
+  form_identifiers_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_identifiers.csv
+  form_person_attributes_used_csv="$PWD/usage/$dbConnCount"_form_used_person_attributes.csv
+  form_person_attributes_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_person_attributes.csv
+  form_used_roles_csv="$PWD/usage/$dbConnCount"_form_used_roles.csv
+  form_not_used_roles_csv="$PWD/usage/$dbConnCount"_form_not_used_roles.csv
+  form_used_global_properties_csv="$PWD/usage/$dbConnCount"_form_used_global_properties.csv
+  form_locations_used_csv="$PWD/usage/$dbConnCount"_form_used_locations.csv
 
 
   ## require mysql command or location, user and database
@@ -159,14 +159,14 @@ do
     ## clean up the xml data file and save each form named after its form uuid
     xmlData=$(printf "$xmlData")
     ## save xmlData onto a respective form specific file replacing back %
-    echo "${xmlData//PCNT/%}" > "$PWD/data/$dbConnCount_form_$uuid.xml"
+    echo "${xmlData//PCNT/%}" > "$PWD/data/$dbConnCount"_form_"$uuid.xml"
   done
   printf "\nFinished extracting forms from the $dbConnCount: $MYSQL_DB database\n"
 
 
   ## extracting concepts from forms
   printf "\nStarting to extract metadata from forms\n"
-  for form_file in $PWD/data/$dbConnCount_form_*.xml $PWD/failed/$dbConnCount_form_*.xml
+  for form_file in "$PWD/data/$dbConnCount"_form_*.xml "$PWD/failed/$dbConnCount"_form_*.xml
   do
     ## simulate loading or progress bar
     echo -n "."

From f54d4c169ca0a7e1e4bbdce46664bf4cabf271a8 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 16:04:13 +0300
Subject: [PATCH 14/21] MEPTS-207: making mysql password optional and adding
 sample params.json into readme

---
 metadataUsage/README.md               | 29 ++++++++++++++++++++++-
 metadataUsage/openmrsMetadataUsage.sh | 34 +++++++++++++++------------
 2 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index 398672a..d6e1dd0 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -6,7 +6,34 @@
 > This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, `jq`, `bc` etc, please manually install those and proceed
 
 ### Running the script
->  Configure mysql instances to extract metadata usages from in params.json and run; `./openmrsMetadataUsage.sh`
+>  Configure mysql instances to extract metadata usages from in `params.json` and run; `./openmrsMetadataUsage.sh`, here's a sample of `params.json` which needs to be in the same folder as the script
+```
+{
+  "mysqlDbConnections": [
+    {
+      "binaryLocation": "mysql",
+      "options": "",
+      "database": "openmrs",
+      "username": "root",
+      "password": ""
+    },
+    {
+      "binaryLocation": "mysql",
+      "options": "",
+      "database": "esaude",
+      "username": "root",
+      "password": ""
+    },
+    {
+      "binaryLocation": "/Users/u/Programs/openmrs-standalone/database/./mysql",
+      "options": "--socket=/tmp/esaude3.sock --max_allowed_packet=96M",
+      "database": "openmrs",
+      "username": "openmrs",
+      "password": "test"
+    }
+  ]
+}
+```
 
 ### Results
 > If running it returned no errors while quitting execution, then three folders should have been created 3 folders; `data, failed, usage`
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index ce0f04f..a963f70 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -41,7 +41,7 @@ getValuesReplacingGlobalProperties () {
         finalValues="$finalValues,"
       fi
       if [[ "$v" = "GlobalProperty:default_location"* ]]; then
-        finalValues=$finalValues$(echo "SELECT GROUP_CONCAT(location_id) FROM location WHERE name = (select property_value from global_property where property = 'default_location')" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s)
+        finalValues=$finalValues$(echo "SELECT GROUP_CONCAT(location_id) FROM location WHERE name = (select property_value from global_property where property = 'default_location')" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s)
       else
         finalValues=$finalValues$v
       fi
@@ -112,6 +112,10 @@ do
   MYSQL_OPTS=$(jq -r ".mysqlDbConnections[$dbConnIndex].options" params.json)
   MYSQL_DB=$(jq -r ".mysqlDbConnections[$dbConnIndex].database" params.json)
   MYSQL_PASS=$(jq -r ".mysqlDbConnections[$dbConnIndex].password" params.json)
+  if [ "$MYSQL_PASS" != "" ]; then
+  do
+    MYSQL_PASS_ENTRY="-p$MYSQL_PASS"
+  fi
   dbConnCount=$(echo "$dbConnIndex + 1" | bc)
   
   form_concept_grouping_file="$PWD/data/$dbConnCount"_form_concept_grouping.txt
@@ -149,11 +153,11 @@ do
 
   ## extract form uuids and loop through them
   printf "\nStarting to extract forms from the $dbConnCount: $MYSQL_DB database\n"
-  $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
+  $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
   do
     ## simulate loading or progress bar
     echo -n "."
-    xmlData=`$MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
+    xmlData=`$MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
     ## printf evaluates % character, let us replace it so it's not treated special
     xmlData="${xmlData//%/PCNT}"
     ## clean up the xml data file and save each form named after its form uuid
@@ -215,55 +219,55 @@ do
 
   ## TODO support used metadata like concepts in the tree like within programs
 
-  usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+  usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
   printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
-  notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB)
+  notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
   printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
-  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n"
-  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
   ##generate unique used global properties from all forms
-  echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
 
   ## print separator horizontal line
   printf '\n%s\n' _________________________________________________________________________________
 
   ##generate unique used global properties from all forms
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER -p$MYSQL_PASS $MYSQL_OPTS $MYSQL_DB
+  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
   printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n"
 
   ## print separator horizontal line

From 8ccb144fe86f578d310c983a26f9a9e19d6c745c Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 16:49:27 +0300
Subject: [PATCH 15/21] MEPTS-207: formatted json in readme

---
 metadataUsage/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index d6e1dd0..ade092f 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -7,7 +7,7 @@
 
 ### Running the script
 >  Configure mysql instances to extract metadata usages from in `params.json` and run; `./openmrsMetadataUsage.sh`, here's a sample of `params.json` which needs to be in the same folder as the script
-```
+```json
 {
   "mysqlDbConnections": [
     {

From 51c4b90390facee8aac38a4392d9e133dde7570a Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Mon, 17 Jun 2019 22:03:31 +0300
Subject: [PATCH 16/21] MEPTS-207: making mysql options optional and accepting
 properties as arguments

---
 metadataUsage/README.md               | 7 +++++++
 metadataUsage/openmrsMetadataUsage.sh | 3 +--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index ade092f..a97d12b 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -30,6 +30,13 @@
       "database": "openmrs",
       "username": "openmrs",
       "password": "test"
+    },
+    {
+      "binaryLocation": "mysql",
+      "options": "",
+      "database": "openmrs",
+      "username": "root",
+      "password": "$MYSQL_ROOT_PASSWORD"
     }
   ]
 }
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index a963f70..c47fe73 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -113,7 +113,6 @@ do
   MYSQL_DB=$(jq -r ".mysqlDbConnections[$dbConnIndex].database" params.json)
   MYSQL_PASS=$(jq -r ".mysqlDbConnections[$dbConnIndex].password" params.json)
   if [ "$MYSQL_PASS" != "" ]; then
-  do
     MYSQL_PASS_ENTRY="-p$MYSQL_PASS"
   fi
   dbConnCount=$(echo "$dbConnIndex + 1" | bc)
@@ -143,7 +142,7 @@ do
 
 
   ## require mysql command or location, user and database
-  if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_OPTS" == "" ] || [ "$MYSQL_DB" == "" ]; then
+  if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_DB" == "" ]; then
     printf "\nUsage; ./openmrsMetadataUsage.sh"
     exit 1
   fi

From 85984668fc5ff07d9cb548ea4eddfcb1e8e68908 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Tue, 18 Jun 2019 10:57:50 +0300
Subject: [PATCH 17/21] MEPTS-207: Added both console and log.out file loggings

---
 metadataUsage/openmrsMetadataUsage.sh | 70 +++++++++++++++------------
 1 file changed, 38 insertions(+), 32 deletions(-)

diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index c47fe73..6c88aa3 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
+LOG_FILE=$PWD/log.out
+
+rm -f $LOG_FILE
+## initialising logging on both console and file
+exec 3>&1 1>>$LOG_FILE 2>&1
+
 ## This openmrs bash script helps extract details about 
 
 ## ensure params.json exists
 if [ ! -f $PWD/params.json ]; then
-    echo "Ensure $PWD/params.json exists with the right mysql database connection parameters!"
+    echo "Ensure $PWD/params.json exists!" | tee /dev/fd/3
     exit 1
 fi
 
@@ -102,8 +108,8 @@ extractExpressionValues () {
 
 
 ## initialise data location
-rm -r data;mkdir data
-mkdir failed
+rm -r data;mkdir -p data
+mkdir -p failed
   
 for ((dbConnIndex=0; dbConnIndex<$(jq ".mysqlDbConnections | length" params.json); dbConnIndex++ ))
 do
@@ -143,19 +149,19 @@ do
 
   ## require mysql command or location, user and database
   if [ "$MYSQL_CMD" == "" ] || [ "$MYSQL_USER" == "" ] || [ "$MYSQL_DB" == "" ]; then
-    printf "\nUsage; ./openmrsMetadataUsage.sh"
+    printf "\nEnsure $PWD/params.json contains the right mysql database connection parameters" | tee /dev/fd/3
     exit 1
   fi
   
   
-  rm $form_failed
+  rm -f $form_failed
 
   ## extract form uuids and loop through them
-  printf "\nStarting to extract forms from the $dbConnCount: $MYSQL_DB database\n"
+  printf "\nStarting to extract forms from the $dbConnCount: $MYSQL_DB database\n" | tee /dev/fd/3
   $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
   do
     ## simulate loading or progress bar
-    echo -n "."
+    echo -n "." | tee /dev/fd/3
     xmlData=`$MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT xml_data FROM htmlformentry_html_form WHERE uuid  = '$uuid'"`
     ## printf evaluates % character, let us replace it so it's not treated special
     xmlData="${xmlData//%/PCNT}"
@@ -164,15 +170,15 @@ do
     ## save xmlData onto a respective form specific file replacing back %
     echo "${xmlData//PCNT/%}" > "$PWD/data/$dbConnCount"_form_"$uuid.xml"
   done
-  printf "\nFinished extracting forms from the $dbConnCount: $MYSQL_DB database\n"
+  printf "\nFinished extracting forms from the $dbConnCount: $MYSQL_DB database\n" | tee /dev/fd/3
 
 
   ## extracting concepts from forms
-  printf "\nStarting to extract metadata from forms\n"
+  printf "\nStarting to extract metadata from forms\n" | tee /dev/fd/3
   for form_file in "$PWD/data/$dbConnCount"_form_*.xml "$PWD/failed/$dbConnCount"_form_*.xml
   do
     ## simulate loading or progress bar
-    echo -n "."
+    echo -n "." | tee /dev/fd/3
     ## extract concept question ids
     echo -n 'cat //*/@groupingConceptId' | xmllint --shell $form_file | awk -F\" 'NR % 2 == 0 { print $2 }' >> $form_concept_grouping_file
     ## extract concept question ids
@@ -190,11 +196,11 @@ do
     ## extract encounterLocations
     echo $(getValuesReplacingGlobalProperties $(extractNodeValues $form_file 'encounterLocation' 'default,order' $form_failed)) >> $form_locations_file
   done
-  printf "\nFinished extracting metadata from forms\n"
+  printf "\nFinished extracting metadata from forms\n" | tee /dev/fd/3
 
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
 
   ## generate unique used concept ids from all forms' questions and answers
@@ -214,61 +220,61 @@ do
   usedLocations="$(combineAllLinesFromFileIntoOneWithSeparator $form_locations_file ',')"
 
   ## prepare usage output
-  rm -r usage;mkdir usage
+  rm -rf usage;mkdir -p usage
 
   ## TODO support used metadata like concepts in the tree like within programs
 
   usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
-  printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n"
+  printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n" | tee /dev/fd/3
   notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
-  printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n"
+  printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n"
+  printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n" | tee /dev/fd/3
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n"
+  printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n"
+  printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n" | tee /dev/fd/3
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n"
+  printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n"
+  printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n" | tee /dev/fd/3
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n"
+  printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n"
+  printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n" | tee /dev/fd/3
   echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n"
+  printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   ##generate unique used global properties from all forms
   echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n"
+  printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 
   ##generate unique used global properties from all forms
   echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n"
+  printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n" | tee /dev/fd/3
 
   ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________
+  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
 done
\ No newline at end of file

From cff808b5dcf5e948516ac4bcde9fe8e3882a107f Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Wed, 19 Jun 2019 13:24:48 +0300
Subject: [PATCH 18/21] MEPTS-207: fixed --secure-file-priv error etc

---
 .gitignore                            |   3 +-
 metadataUsage/README.md               |   3 +-
 metadataUsage/openmrsMetadataUsage.sh | 172 ++++++++++++++++----------
 3 files changed, 112 insertions(+), 66 deletions(-)

diff --git a/.gitignore b/.gitignore
index e0e05bc..2da50a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@ metadataUsage/data/
 metadataUsage/usage/
 metadataUsage/failed/
 metadataUsage/test.sh
-metadataUsage/params.json
\ No newline at end of file
+metadataUsage/params.json
+metadataUsage/log.out
\ No newline at end of file
diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index a97d12b..637b988 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -43,10 +43,11 @@
 ```
 
 ### Results
-> If running it returned no errors while quitting execution, then three folders should have been created 3 folders; `data, failed, usage`
+> If running it returned no errors (logged into `log.out`) while quitting execution, then three folders should have been created 3 folders; `data, failed, usage`
 ```
    drwxr-xr-x  ... data
    drwxr-xr-x  ... failed
+   -rw-r--r--  ... log.out
    -rwxr-xr-x@ ... openmrsMetadataUsage.sh
    drwxr-xr-x  ... usage
    ```
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 6c88aa3..8a6d041 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -105,8 +105,7 @@ extractExpressionValues () {
   ## echo or 'return' excluding first trailing character of separator
   echo "$lines"
 }
-
-
+ 
 ## initialise data location
 rm -r data;mkdir -p data
 mkdir -p failed
@@ -123,6 +122,12 @@ do
   fi
   dbConnCount=$(echo "$dbConnIndex + 1" | bc)
   
+  ## locate where mysql exported cvs are rightly meant to be saved
+  secureFileLocation=$(echo "SELECT @@secure_file_priv" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB | sed -n '2p')
+  if [ "$secureFileLocation" == "NULL" ] || [ "$secureFileLocation" == "" ]; then
+    secureFileLocation="$PWD/usage"
+  fi
+
   form_concept_grouping_file="$PWD/data/$dbConnCount"_form_concept_grouping.txt
   form_concept_questions_file="$PWD/data/$dbConnCount"_form_concept_questions.txt
   form_concept_answers_file="$PWD/data/$dbConnCount"_form_concept_answers.txt
@@ -133,18 +138,18 @@ do
   form_defaults_file="$PWD/data/$dbConnCount"_form_defaults.txt
   ##contains form_file: advise to fix it
   form_failed="$PWD/failed/$dbConnCount"_form_failed.txt
-  form_concept_used_csv="$PWD/usage/$dbConnCount"_form_used_concepts.csv
-  form_concept_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_concepts.csv
-  form_programs_used_csv="$PWD/usage/$dbConnCount"_form_used_programs.csv
-  form_programs_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_programs.csv
-  form_identifiers_used_csv="$PWD/usage/$dbConnCount"_form_used_identifiers.csv
-  form_identifiers_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_identifiers.csv
-  form_person_attributes_used_csv="$PWD/usage/$dbConnCount"_form_used_person_attributes.csv
-  form_person_attributes_not_used_csv="$PWD/usage/$dbConnCount"_form_not_used_person_attributes.csv
-  form_used_roles_csv="$PWD/usage/$dbConnCount"_form_used_roles.csv
-  form_not_used_roles_csv="$PWD/usage/$dbConnCount"_form_not_used_roles.csv
-  form_used_global_properties_csv="$PWD/usage/$dbConnCount"_form_used_global_properties.csv
-  form_locations_used_csv="$PWD/usage/$dbConnCount"_form_used_locations.csv
+  form_concept_used_csv="$secureFileLocation/$dbConnCount"_form_used_concepts.csv
+  form_concept_not_used_csv="$secureFileLocation/$dbConnCount"_form_not_used_concepts.csv
+  form_programs_used_csv="$secureFileLocation/$dbConnCount"_form_used_programs.csv
+  form_programs_not_used_csv="$secureFileLocation/$dbConnCount"_form_not_used_programs.csv
+  form_identifiers_used_csv="$secureFileLocation/$dbConnCount"_form_used_identifiers.csv
+  form_identifiers_not_used_csv="$secureFileLocation/$dbConnCount"_form_not_used_identifiers.csv
+  form_person_attributes_used_csv="$secureFileLocation/$dbConnCount"_form_used_person_attributes.csv
+  form_person_attributes_not_used_csv="$secureFileLocation/$dbConnCount"_form_not_used_person_attributes.csv
+  form_used_roles_csv="$secureFileLocation/$dbConnCount"_form_used_roles.csv
+  form_not_used_roles_csv="$secureFileLocation/$dbConnCount"_form_not_used_roles.csv
+  form_used_global_properties_csv="$secureFileLocation/$dbConnCount"_form_used_global_properties.csv
+  form_locations_used_csv="$secureFileLocation/$dbConnCount"_form_used_locations.csv
 
 
   ## require mysql command or location, user and database
@@ -154,10 +159,14 @@ do
   fi
   
   
+  printf '\n%s\n' ------------------------------------$dbConnCount:$MYSQL_DB------------------------------------ | tee /dev/fd/3
+  
+  
   rm -f $form_failed
+  
 
   ## extract form uuids and loop through them
-  printf "\nStarting to extract forms from the $dbConnCount: $MYSQL_DB database\n" | tee /dev/fd/3
+  printf "\nStarting to extract forms from the database\n" | tee /dev/fd/3
   $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB -s -N -e "SELECT uuid FROM htmlformentry_html_form" | while IFS= read -r uuid
   do
     ## simulate loading or progress bar
@@ -170,7 +179,7 @@ do
     ## save xmlData onto a respective form specific file replacing back %
     echo "${xmlData//PCNT/%}" > "$PWD/data/$dbConnCount"_form_"$uuid.xml"
   done
-  printf "\nFinished extracting forms from the $dbConnCount: $MYSQL_DB database\n" | tee /dev/fd/3
+  printf "\nFinished extracting forms from the database\n" | tee /dev/fd/3
 
 
   ## extracting concepts from forms
@@ -204,8 +213,20 @@ do
 
 
   ## generate unique used concept ids from all forms' questions and answers
-
-  usedConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ',' '"'),$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',' '"')"
+  usedGroupingConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_grouping_file ',' '"')"
+  usedConceptIds="$usedGroupingConceptIds"
+  usedQuestionConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_questions_file ',' '"')"
+  if [ "$usedQuestionConceptIds" != "" ] && [ "$usedConceptIds" != "" ]; then
+    usedConceptIds="$usedConceptIds,$usedQuestionConceptIds"
+  elif [ "$usedQuestionConceptIds" != "" ] && [ "$usedConceptIds" == "" ]; then
+      usedConceptIds="$usedQuestionConceptIds"
+  fi
+  usedAnswerConceptIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_concept_answers_file ',' '"')"
+  if [ "$usedAnswerConceptIds" != "" ] && [ "$usedConceptIds" != "" ]; then
+    usedConceptIds="$usedConceptIds,$usedAnswerConceptIds"
+  elif [ "$usedAnswerConceptIds" != "" ] && [ "$usedConceptIds" == "" ]; then
+      usedConceptIds="$usedAnswerConceptIds"
+  fi
   ## generate unique used program ids from all forms
   usedProgramIds="$(combineAllLinesFromFileIntoOneWithSeparator $form_programs_file ',')"
   ## generate unique used identifiers
@@ -224,57 +245,80 @@ do
 
   ## TODO support used metadata like concepts in the tree like within programs
 
-  usedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
-  printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n" | tee /dev/fd/3
-  notUsedConceptsIdsPut=$(echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB)
-  printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n" | tee /dev/fd/3
-
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
-
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n" | tee /dev/fd/3
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n" | tee /dev/fd/3
-
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
-
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n" | tee /dev/fd/3
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n" | tee /dev/fd/3
-
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
-
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n" | tee /dev/fd/3
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n" | tee /dev/fd/3
+  if [ "$usedConceptIds" != "" ]; then
+    echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed Concepts in HTML Forms saved in# \n$form_concept_used_csv\n" | tee /dev/fd/3
+    echo "SELECT 'UUID', 'ID' UNION ALL SELECT uuid,concept_id FROM concept WHERE concept_id NOT IN (SELECT concept_id FROM concept WHERE (concept_id|uuid) IN ($usedConceptIds)) INTO OUTFILE '$form_concept_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nNot Used Concepts in HTML Forms saved in# \n$form_concept_not_used_csv\n" | tee /dev/fd/3
+    
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
+  
+  if [ "$usedProgramIds" != "" ]; then
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id IN ($usedProgramIds) INTO OUTFILE '$form_programs_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed Programs in HTML Forms saved in# \n$form_programs_used_csv\n" | tee /dev/fd/3
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,program_id,name,description FROM program WHERE program_id NOT IN ($usedProgramIds) INTO OUTFILE '$form_programs_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nNot Used Programs in HTML Forms saved in# \n$form_programs_not_used_csv\n" | tee /dev/fd/3
+
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
 
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  if [ "$usedIdentifierIds" != "" ]; then
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed Identifiers in HTML Forms saved in# \n$form_identifiers_used_csv\n" | tee /dev/fd/3
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,patient_identifier_type_id,name,description FROM patient_identifier_type WHERE patient_identifier_type_id NOT IN ($usedIdentifierIds) INTO OUTFILE '$form_identifiers_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nNot Used Identifiers in HTML Forms saved in# \n$form_identifiers_not_used_csv\n" | tee /dev/fd/3
 
-  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n" | tee /dev/fd/3
-  echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n" | tee /dev/fd/3
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
+  
+  if [ "$usedPersonAttributes" != "" ]; then
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed person attribute types in HTML Forms saved in# \n$form_person_attributes_used_csv\n" | tee /dev/fd/3
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,person_attribute_type_id,name,description FROM person_attribute_type WHERE person_attribute_type_id NOT IN ($usedPersonAttributes) INTO OUTFILE '$form_person_attributes_not_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nNot Used person attribute types in HTML Forms saved in# \n$form_person_attributes_not_used_csv\n" | tee /dev/fd/3
+
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
 
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  if [ "$usedRoles" != "" ]; then
+    echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role IN ($usedRoles) INTO OUTFILE '$form_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed roles in HTML Forms saved in# \n$form_used_roles_csv\n" | tee /dev/fd/3
+    echo "SELECT 'UUID', 'ROLE', 'DESCRIPTION' UNION ALL SELECT uuid,role,description FROM role WHERE role NOT IN ($usedRoles) INTO OUTFILE '$form_not_used_roles_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nNot Used roles in HTML Forms saved in# \n$form_not_used_roles_csv\n" | tee /dev/fd/3
 
-  ##generate unique used global properties from all forms
-  echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n" | tee /dev/fd/3
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
+  
+  if [ "$usedGlobalProperties" != "" ]; then
+    ##generate unique used global properties from all forms
+    echo "SELECT 'UUID', 'PROPERTY', 'VALUE', 'DESCRIPTION' UNION ALL SELECT uuid,property,property_value,description FROM global_property WHERE property IN ($usedGlobalProperties) INTO OUTFILE '$form_used_global_properties_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed Global Properties in HTML Forms saved in# \n$form_used_global_properties_csv\n" | tee /dev/fd/3
 
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
 
-  ##generate unique used global properties from all forms
-  echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
-  printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n" | tee /dev/fd/3
+  if [ "$usedLocations" != "" ]; then
+    ##generate unique used locations from all forms
+    echo "SELECT 'UUID', 'ID', 'NAME', 'DESCRIPTION' UNION ALL SELECT uuid,location_id,name,description FROM location WHERE location_id IN ($usedLocations) INTO OUTFILE '$form_locations_used_csv' FIELDS TERMINATED BY ','" | $MYSQL_CMD -u$MYSQL_USER $MYSQL_PASS_ENTRY $MYSQL_OPTS $MYSQL_DB
+    printf "\nUsed Locations in HTML Forms saved in# \n$form_locations_used_csv\n" | tee /dev/fd/3
 
-  ## print separator horizontal line
-  printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+    ## print separator horizontal line
+    printf '\n%s\n' _________________________________________________________________________________ | tee /dev/fd/3
+  fi
+  
+  ## ensure exports are in usage folder
+  if [[ "$secureFileLocation" != "$PWD/usage" ]]; then
+     printf "\nMoving all *_form_*.csv files from $secureFileLocation into $PWD/usage\n" | tee /dev/fd/3
+    mv "$secureFileLocation/$dbConnCount"_form_*.csv "$PWD/usage"
+  fi
+  
+  printf '\n%s\n' ------------------------------------$dbConnCount:$MYSQL_DB------------------------------------ | tee /dev/fd/3
+  
 done
\ No newline at end of file

From 381dd2a3bdeab9a505096c429ff7d91b7f09b7f6 Mon Sep 17 00:00:00 2001
From: k-joseph <kaweesi@jembi.org>
Date: Wed, 19 Jun 2019 14:22:07 +0300
Subject: [PATCH 19/21] MEPTS-207: Fixed using system variables in params.json

---
 metadataUsage/README.md               | 11 ++++++-----
 metadataUsage/openmrsMetadataUsage.sh | 10 +++++-----
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index 637b988..17b4c68 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -3,10 +3,8 @@
 
 
 ### Setup
-> This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, `jq`, `bc` etc, please manually install those and proceed
-
-### Running the script
->  Configure mysql instances to extract metadata usages from in `params.json` and run; `./openmrsMetadataUsage.sh`, here's a sample of `params.json` which needs to be in the same folder as the script
+> This script has been written against OS X El Capitan 10.11.6 (15G22010), in any case running against another OS or version complains about any external dependencies such as `xmllint`, `jq`, `bc` etc, please manually install those and proceed.
+Configure mysql instances to extract metadata usages within `params.json`, here's a sample content and the this file needs to be in the same folder as the script
 ```json
 {
   "mysqlDbConnections": [
@@ -42,6 +40,9 @@
 }
 ```
 
+### Running the script
+>  Run; `./openmrsMetadataUsage.sh`
+
 ### Results
 > If running it returned no errors (logged into `log.out`) while quitting execution, then three folders should have been created 3 folders; `data, failed, usage`
 ```
@@ -52,5 +53,5 @@
    drwxr-xr-x  ... usage
    ```
    `data`: contains xml forms export from the database as well as metadata field extracts
-   `failed`: contains failed xml forms which need human intervention, the details of what failed in text files
+   `failed`: contains failed xml forms which need human intervention, the details of what failed in text files such as; `*_form_failed.txt`
    `usage`: contains csv output per exported metadata objects
diff --git a/metadataUsage/openmrsMetadataUsage.sh b/metadataUsage/openmrsMetadataUsage.sh
index 8a6d041..8f5fbe3 100755
--- a/metadataUsage/openmrsMetadataUsage.sh
+++ b/metadataUsage/openmrsMetadataUsage.sh
@@ -112,11 +112,11 @@ mkdir -p failed
   
 for ((dbConnIndex=0; dbConnIndex<$(jq ".mysqlDbConnections | length" params.json); dbConnIndex++ ))
 do
-  MYSQL_CMD=$(jq -r ".mysqlDbConnections[$dbConnIndex].binaryLocation" params.json)
-  MYSQL_USER=$(jq -r ".mysqlDbConnections[$dbConnIndex].username" params.json)
-  MYSQL_OPTS=$(jq -r ".mysqlDbConnections[$dbConnIndex].options" params.json)
-  MYSQL_DB=$(jq -r ".mysqlDbConnections[$dbConnIndex].database" params.json)
-  MYSQL_PASS=$(jq -r ".mysqlDbConnections[$dbConnIndex].password" params.json)
+  eval MYSQL_CMD=$(jq -r ".mysqlDbConnections[$dbConnIndex].binaryLocation" params.json)
+  eval MYSQL_USER=$(jq -r ".mysqlDbConnections[$dbConnIndex].username" params.json)
+  eval MYSQL_OPTS=$(jq -r ".mysqlDbConnections[$dbConnIndex].options" params.json)
+  eval MYSQL_DB=$(jq -r ".mysqlDbConnections[$dbConnIndex].database" params.json)
+  eval MYSQL_PASS=$(jq -r ".mysqlDbConnections[$dbConnIndex].password" params.json)
   if [ "$MYSQL_PASS" != "" ]; then
     MYSQL_PASS_ENTRY="-p$MYSQL_PASS"
   fi

From 0b533ab4fffa83e1869d153b8d9e2d7138b6a144 Mon Sep 17 00:00:00 2001
From: cfaife <clerio.faife@gmail.com>
Date: Tue, 3 Sep 2019 16:28:01 +0200
Subject: [PATCH 20/21] MEPTS-207: added troubleshoot section for readme.md
 file

---
 metadataUsage/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index 17b4c68..738c39d 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -55,3 +55,6 @@ Configure mysql instances to extract metadata usages within `params.json`, here'
    `data`: contains xml forms export from the database as well as metadata field extracts
    `failed`: contains failed xml forms which need human intervention, the details of what failed in text files such as; `*_form_failed.txt`
    `usage`: contains csv output per exported metadata objects
+
+### Troubleshooting
+> if genarated usage folder is empty while /var/lib/mysql-files/ contains generated csv files, you can move it manually as root or with sudo typing: `mv /var/lib/mysql-files/*.csv usage/`

From b3a66057b5c29cc12f5ea107f816be36c20dedb4 Mon Sep 17 00:00:00 2001
From: cfaife <clerio.faife@gmail.com>
Date: Tue, 3 Sep 2019 16:34:48 +0200
Subject: [PATCH 21/21] MEPTS-207: formting README.md file

---
 metadataUsage/README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/metadataUsage/README.md b/metadataUsage/README.md
index 738c39d..ff6d1f8 100644
--- a/metadataUsage/README.md
+++ b/metadataUsage/README.md
@@ -57,4 +57,7 @@ Configure mysql instances to extract metadata usages within `params.json`, here'
    `usage`: contains csv output per exported metadata objects
 
 ### Troubleshooting
-> if genarated usage folder is empty while /var/lib/mysql-files/ contains generated csv files, you can move it manually as root or with sudo typing: `mv /var/lib/mysql-files/*.csv usage/`
+> if genarated `usage` folder is empty while `/var/lib/mysql-files/` contains generated csv files, you can move it manually as `root` or with `sudo` typing:
+```
+   `mv /var/lib/mysql-files/*.csv usage/`
+```
\ No newline at end of file