Skip to content

Commit 5a37dc3

Browse files
gujguj
committed
Add profiler/ for parsing and plotting adios json files
1 parent 4582522 commit 5a37dc3

File tree

12 files changed

+1772
-0
lines changed

12 files changed

+1772
-0
lines changed

‎source/utils/profiler/ReadMe.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
These are the script I use to plot json profilers.
2+
In the ADIOS proflier, the important keys are:
3+
PP PDW ES ES_AWD ES_aggregate_info MetaInfoBcast FixedMetaInfoGather transport_0.wbytes
4+
5+
The relationship is
6+
PDW (PerformDataWrite) | PP(PerformPut)
7+
ES: (EndStep)
8+
- ES_AWD
9+
- ES_aggregate_info
10+
- FixedMetaInfoGather
11+
- MetaInfoBcase
12+
Bytes are reported in "transport_0.wbytes"
13+
14+
15+
For how to plot a json file, see test/commands

‎source/utils/profiler/sample_data/t0/t0.json

Lines changed: 514 additions & 0 deletions
Large diffs are not rendered by default.

‎source/utils/profiler/sample_data/t10/t10.json

Lines changed: 514 additions & 0 deletions
Large diffs are not rendered by default.

‎source/utils/profiler/scripts/common.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import matplotlib.pyplot as plt
2+
import parseMe
3+
import os
4+
5+
def saveMe(figName):
6+
parentDir=os.path.dirname(figName)
7+
8+
plt.tight_layout()
9+
#print (figName)
10+
if (os.path.isdir(parentDir)):
11+
plt.savefig(figName)
12+
else:
13+
if (len(parentDir) > 1):
14+
print ("Missing directory! Can not save plot to dir:", parentDir)
15+
else:
16+
plt.savefig(figName)
17+
18+
showFig = parseMe.command_options[parseMe.TAGS["showfig"]]
19+
if (showFig):
20+
plt.show()
21+
22+
23+
24+
25+

‎source/utils/profiler/scripts/draw.sh

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
#!/bin/bash
2+
################################################################
3+
# Usage: ./script.sh <jobID> <scriptsHome> <extractedFilesBase> <timeStep> <aggType>
4+
# Example: ./script.sh 12345 /path/to/scripts /path/to/extracted 0 <aggType>
5+
################################################################
6+
7+
# Validate input parameters
8+
if [ $# -lt 5 ]; then
9+
echo "Usage: $0 <jobID> <scriptsHome> <extractedFilesBase> <timeStep> <aggType>"
10+
return 1
11+
fi
12+
13+
jobID="$1"
14+
scriptsHome="$2"
15+
timeStep="$4"
16+
extractedFilesLoc="$3/${timeStep}"
17+
# TLS EWS etc
18+
aggType=$5
19+
20+
key1="default"
21+
key2="flatten"
22+
key3="joined"
23+
24+
if [ $# -ge 6 ]; then
25+
key1=$6
26+
fi
27+
28+
if [ $# -ge 7 ]; then
29+
key2=$7
30+
fi
31+
32+
if [ $# -ge 8 ]; then
33+
key3=$8
34+
fi
35+
36+
prefix12=${key1}_${key2}
37+
prefix23=${key2}_${key3}
38+
prefix13=${key1}_${key3}
39+
40+
# Create directory structure
41+
mkdir -p "plots" || { echo "Error: Cannot create plots directory"; return 1; }
42+
mkdir -p "plots/${jobID}" || { echo "Error: Cannot create plots/${jobID}"; return 1; }
43+
mkdir -p "plots/${jobID}/${aggType}" || { echo "Error: Cannot create plots/${jobID}/${aggType}"; return 1; }
44+
45+
currPlotDest="plots/${jobID}/${aggType}/${timeStep}"
46+
mkdir -p "${currPlotDest}" || { echo "Error: Cannot create ${currPlotDest}"; return 1; }
47+
48+
49+
detectContent()
50+
{
51+
local key="$1"
52+
local count=$(find "${extractedFilesLoc}" -maxdepth 1 -type f -name "${key}*" | wc -l)
53+
54+
if [ ${count} -gt 0 ]; then
55+
echo "${count} files has ${key}"
56+
true; return
57+
else
58+
false; return
59+
fi
60+
61+
}
62+
63+
compareTwo()
64+
{
65+
local filePrefix=$1
66+
local type1=$2
67+
local type2=$3
68+
69+
#path1=($(ls ${extractedFilesLoc}/${type1}* ))
70+
#path2=($(ls ${extractedFilesLoc}/${type2}* ))
71+
72+
#n=$((${#path1[@]}*${#path2[@]}))
73+
74+
echo "python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=ES"
75+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=ES
76+
77+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=ES_AWD
78+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=ES_AWD logScale=x
79+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=ES_aggregate_info levelAxis=True logScale=x
80+
81+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=FixedMetaInfoGather logScale=xy
82+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=FixedMetaInfoGather logScale=y
83+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=FixedMetaInfoGather
84+
85+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=MetaInfoBcast
86+
87+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=PDW
88+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=PP logScale=x
89+
90+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=transport_0.wbytes whichKind=MB
91+
92+
if [[ $type1 == async* ]] ; then
93+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=BS_WaitOnAsync
94+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=DC_WaitOnAsync1 logScale=x
95+
python3 ${scriptsHome}/plotRanks.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix} jsonAttr=DC_WaitOnAsync2 logScale=x
96+
fi
97+
98+
echo "==> plot all the times spent on rank 0: python3 ${scriptsHome}/plotStack.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} whichRank=0 plotPrefix=${currPlotDest}/${filePrefix}"
99+
python3 ${scriptsHome}/plotStack.py ${type1} ${type2} --set dataDir=${extractedFilesLoc} whichRank=0 plotPrefix=${currPlotDest}/${filePrefix}
100+
101+
#fi
102+
}
103+
104+
105+
compareThree()
106+
{
107+
local filePrefix=$1
108+
local type1=$2
109+
local type2=$3
110+
local type3=$4
111+
112+
echo "==> plot all the times spent on rank 0"
113+
python3 ${scriptsHome}/plotStack.py ${type1} ${type2} ${type3} --set dataDir=${extractedFilesLoc} whichRank=0 plotPrefix=${currPlotDest}/${filePrefix}
114+
115+
echo "==> plot numCalls occurred"
116+
echo "${scriptsHome}/plotCall.py ${type1} ${type2} ${type3} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix}"
117+
python3 ${scriptsHome}/plotCall.py ${type1} ${type2} ${type3} --set dataDir=${extractedFilesLoc} plotPrefix=${currPlotDest}/${filePrefix}
118+
}
119+
120+
checkPossibilities()
121+
{
122+
local prefix=$1
123+
local typeA=${prefix}${key1}
124+
local typeB=${prefix}${key2}
125+
local typeC=${prefix}${key3}
126+
127+
128+
129+
local trueCounter=0;
130+
131+
has_key1=false
132+
has_key2=false
133+
has_key3=false
134+
135+
if detectContent $typeA; then
136+
has_key1=true
137+
trueCounter=$((trueCounter + 1))
138+
fi
139+
140+
if detectContent $typeB; then
141+
has_key2=true
142+
trueCounter=$((trueCounter + 1))
143+
fi
144+
if detectContent $typeC; then
145+
has_key3=true
146+
trueCounter=$((trueCounter + 1))
147+
fi
148+
149+
echo "$prefix counter = $trueCounter, $has_key1 $has_key3 $has_key2"
150+
if [[ trueCounter -ge 2 ]]; then
151+
if ($has_key1 && $has_key3); then
152+
compareTwo $prefix13 $typeA $typeC
153+
fi
154+
if ($has_key1 && $has_key2); then
155+
compareTwo $prefix12 $typeA $typeB
156+
fi
157+
if ($has_key3 && $has_key2); then
158+
compareTwo $prefix23 $typeC $typeB
159+
fi
160+
fi
161+
162+
if [[ trueCounter -eq 3 ]]; then
163+
compareThree djf $typeA $typeC $typeB
164+
fi
165+
166+
if [[ trueCounter -eq 1 ]]; then
167+
validType=$typeA
168+
if ${has_key3}; then
169+
validType=$typeC
170+
elif ${has_key2}; then
171+
validType=$typeB
172+
fi
173+
compareTwo ${validType} ${validType} ## first input is file prefix
174+
fi
175+
}
176+
177+
178+
checkPossibilities ""
179+
checkPossibilities "async"
180+
181+
182+
183+

‎source/utils/profiler/scripts/extract.sh

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/bin/bash
2+
###########################################################################
3+
## sample usage:
4+
## > source extract.sh ES ../jsons/testMe_default ../jsons/testMe_flatten
5+
###########################################################################
6+
outDir=outs
7+
mkdir -p ${outDir}
8+
9+
10+
processFile() {
11+
local filePath="$1"
12+
local attrName="$2"
13+
14+
15+
BASE_NAME=$(basename $filePath | cut -d. -f1)
16+
local key=${BASE_NAME}
17+
if [[ "$filePath" == *"flatten"* ]]; then
18+
key="flatten"
19+
fi
20+
if [[ "$filePath" == *"default"* ]]; then
21+
key="default"
22+
fi
23+
if [[ "$filePath" == *"joined"* ]]; then
24+
key="joined"
25+
fi
26+
local asyncKey=""
27+
if [[ "$filePath" == *"async"* ]]; then
28+
asyncKey="async"
29+
fi
30+
31+
echo "Processing $filePath, $attrName key= ${asyncKey}${key}"
32+
33+
if [[ $attrName == *bytes* ]]; then
34+
jq -r ".[] | .$attrName" "$filePath" | awk '{print $1/1048576}' > "${outDir}/${asyncKey}${key}_MB_${attrName}"
35+
else
36+
local attrMus="${attrName}_mus"
37+
local attrNCalls="${attrName}.nCalls"
38+
39+
#echo "both $attrMus $attrNCalls"
40+
jq -r ".[] | .$attrMus" "$filePath" | awk '{print $1/1000000}' > "${outDir}/${asyncKey}${key}_secs_${attrName}"
41+
42+
## use awk to divide by 1 to make sure nulls becomes 0 (e.g. if PDW is not present, jq returns null, awk makes it 0)
43+
jq -r ".[] | .$attrNCalls" "$filePath" | awk '{print $1/1}'> "outs/${asyncKey}${key}_nCalls_${attrName}"
44+
fi
45+
}
46+
47+
if (( $# < 2 )); then
48+
echo "Expecting: $0 jsonProperty file1 .."
49+
else
50+
numFiles=$(($# - 1))
51+
echo "Num Files: $numFiles"
52+
53+
if [[ $1 == "all" ]]; then
54+
knownAttrs=( 'PP' 'PDW' 'ES' 'ES_AWD' 'ES_aggregate_info' 'MetaInfoBcast' 'FixedMetaInfoGather' 'transport_0.wbytes' )
55+
asyncAttrs=( 'BS_WaitOnAsync' 'DC_WaitOnAsync1' 'DC_WaitOnAsync2' )
56+
else
57+
knownAttrs=("$1")
58+
fi
59+
echo "Attributes: ${knownAttrs[*]}"
60+
61+
for ((i = 2; i <= $#; i++ )); do
62+
currFile=$argv[i]
63+
#currFile="${!i}"
64+
for currAttr in ${knownAttrs[@]}; do
65+
processFile $currFile $currAttr
66+
done
67+
if [[ $currFile == *async* ]]; then
68+
for tmp in "${asyncAttrs[@]}"; do
69+
echo "async file: currFile = $currFile, $tmp"
70+
processFile "$currFile" "$tmp"
71+
done
72+
fi
73+
done
74+
fi

0 commit comments

Comments
 (0)