diff --git a/cgi/analyst.py b/cgi/analyst.py index f4ad283..7fd6841 100644 --- a/cgi/analyst.py +++ b/cgi/analyst.py @@ -4,6 +4,8 @@ from pandas.compat import StringIO import os from pathlib import Path +import subprocess +import re filepath = '/var/www/wou/tmp/' @@ -14,6 +16,11 @@ lastfilename = last_line +cmd_output = subprocess.Popen(['tail', '-n1', '/var/log/apache2/access.log'], stdout=subprocess.PIPE) +out, err = cmd_output.communicate() + +print(out) + file_to_open = filepath + lastfilename.rstrip() f = open(file_to_open, 'r') diff --git a/cgi/analyst_2.py b/cgi/analyst_2.py new file mode 100644 index 0000000..5af6ad8 --- /dev/null +++ b/cgi/analyst_2.py @@ -0,0 +1,78 @@ +#!/usr/bin/python3 + +import pandas as pd +from pandas.compat import StringIO +import os +from pathlib import Path +import subprocess +import re + +filepath = '/var/www/wou/tmp/' + +with open("/var/www/wou/data/filelist.txt", "r") as file: + for last_line in file: + pass + file.close() + +lastfilename = last_line + +rc = subprocess.call('/var/www/wou/cgi/get_info.sh') + +with open("/var/www/wou/data/client_info.txt", "r") as file: + for last_access_line in file: + pass + file.close() + +#r1 = re.findall(r"^[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}\s", last_access_line) + +command_output = os.popen('tail -n1 /var/www/wou/data/client_info.log') +#r1 = re.findall(r"^[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}\s", command_output.read()) +r2 = re.findall(r" [lLwW].{4,11}", command_output.read()) + +file_to_open = filepath + lastfilename.rstrip() + +f = open(file_to_open, 'r') +datainfo = f.readlines(3) + +lines = list(f) +f.close() + +df = pd.read_csv(file_to_open, names=['Time','Severity','Text'], engine='python') +df.to_html('../output.html', justify='center') + +print("Content-Type: text/html\n") +print("\n") +print("""\ + + + + Data Analytic Output + + + + +""") +print("\n") +print("

You are using a {} system

".format(r2)) +print("\n") +print("

Returns of your data filename: {}

".format(lastfilename)) +print("""\ + + + +""".format(lastfilename)) + +print("""\ + +
+ +
+ +
+ + +
+ + + +""") diff --git a/cgi/analyst_3.py b/cgi/analyst_3.py new file mode 100644 index 0000000..cd6a040 --- /dev/null +++ b/cgi/analyst_3.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 + +import pandas as pd +from pandas.compat import StringIO +import os +from pathlib import Path +import subprocess +import re +import matplotlib.pyplot as plt + +filepath = '/var/www/wou/tmp/' + +with open("/var/www/wou/data/filelist.txt", "r") as file: + for last_line in file: + pass + file.close() + +lastfilename = last_line + +rc = subprocess.call('/var/www/wou/cgi/get_info.sh') + +with open("/var/www/wou/data/client_info.txt", "r") as file: + for last_access_line in file: + pass + file.close() + +#r1 = re.findall(r"^[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}\s", last_access_line) + +command_output = os.popen('tail -n1 /var/www/wou/data/client_info.log') +#r1 = re.findall(r"^[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}\s", command_output.read()) + +r2 = re.findall(r" [lLwW][iI][nN].{4,11} ", command_output.read()) + +file_to_open = filepath + lastfilename.rstrip() + +f = open(file_to_open, 'r') +datainfo = f.readlines(3) + +lines = list(f) +f.close() + +Data = {'Tasks': [300,500,700]} +df = DataFrame(Data,columns=['Tasks'],index = ['Tasks Pending','Tasks Ongoing','Tasks Completed']) + +df.plot.pie(y='Tasks',figsize=(5, 5),autopct='%1.1f%%', startangle=90) + +plt.savefig('foo.png') + +df = pd.read_csv(file_to_open, names=['Time','Severity','Text'], engine='python') +df['Severity'].value_counts() + +df.plot.pie(y='Severity',figsize=(10,10),autopct='%1.1f%%', startangle=90) +plt.savefig('../foo.png') + +df.to_html('../output.html', justify='center') + +print("Content-Type: text/html\n") +print("\n") +print("""\ + + + + Data Analytic Output + + + + +""") +print("\n") +print("

You are using a {} system

".format(r2)) +print("\n") +print("

Returns of your data filename: {}

".format(lastfilename)) +print("""\ + + + +""".format(lastfilename)) + +print("""\ + +
+ +
+ +
+ + +
+ + + +""") diff --git a/cgi/analyst_4.py b/cgi/analyst_4.py new file mode 100644 index 0000000..4b3b357 --- /dev/null +++ b/cgi/analyst_4.py @@ -0,0 +1,91 @@ +#!/usr/bin/python3 + +import pandas as pd +from pandas.compat import StringIO +import os +from pathlib import Path +import subprocess +import re +import matplotlib + +matplotlib.use('Agg') + +import matplotlib.pyplot as plt + + +filepath = '/var/www/wou/tmp/' + +with open("/var/www/wou/data/filelist.txt", "r") as file: + for last_line in file: + pass + file.close() + +lastfilename = last_line + +rc = subprocess.call('/var/www/wou/cgi/get_info.sh') + +with open("/var/www/wou/data/client_info.txt", "r") as file: + for last_access_line in file: + pass + file.close() + +command_output = os.popen('tail -n1 /var/www/wou/data/client_info.log') + +r2 = re.findall(r" [lLwW][iI][nN].{4,11} ", command_output.read()) + +file_to_open = filepath + lastfilename.rstrip() + +f = open(file_to_open, 'r') +datainfo = f.readlines(3) + +lines = list(f) +f.close() + +df = pd.read_csv(file_to_open, names=['Time','Severity','Text'], engine='python') + +df.to_html('../output.html', justify='center') + +df.Severity.value_counts().plot.pie(y='Severities', figsize=(5, 5),autopct='%1.1f%%', startangle=90) +plt.savefig('../images/chart_output.png') + + +print("Content-Type: text/html\n") +print("\n") +print("""\ + + + + Data Analytic Output + + + + +""") +print("\n") +print("

You are using a {} system

".format(r2)) +print("\n") +print("

Returns of your data filename: {}

".format(lastfilename)) +print("""\ + + + +""".format(lastfilename)) + +print("""\ + +
+ +
+ +
+ +
+ +
+ + +
+ + + +""") diff --git a/cgi/check.py b/cgi/check.py new file mode 100644 index 0000000..9e10a57 --- /dev/null +++ b/cgi/check.py @@ -0,0 +1,45 @@ +from pandas import DataFrame + +import matplotlib + +matplotlib.use('Agg') + +import matplotlib.pyplot as plt +import pandas as pd + +# Data = {'Tasks': [300,500,700]} +# df = DataFrame(Data,columns=['Tasks'],index = ['Tasks Pending','Tasks Ongoing','Tasks Completed']) + +sevs = {} + +df = pd.read_csv('../tmp/log.txt', names=['Time','Severity','Text'], engine='python') + +print(df) + +sd = df['Severity'] + +cf = pd.value_counts(df['Severity'].values) + + +print(cf) + + +df.Severity.value_counts().plot.pie(y='Severities', figsize=(5, 5),autopct='%1.1f%%', startangle=90) +plt.savefig('foo_1.png') + +#print(sd) +# plt.title('severities dataset') +# plt.axis('severities') + + + +# df.plot.pie(y='Severity') + +# plt.show() + + +#sd.plot.pie(y='Severity',figsize=(5, 5),autopct='%1.1f%%', startangle=90) + +# # df.plot.pie(y='Tasks',figsize=(5, 5),autopct='%1.1f%%', startangle=90) +# # #plt.show() +# plt.savefig('foo_1.png') \ No newline at end of file diff --git a/cgi/foo.png b/cgi/foo.png new file mode 100644 index 0000000..a9e24cd Binary files /dev/null and b/cgi/foo.png differ diff --git a/cgi/foo_1.png b/cgi/foo_1.png new file mode 100644 index 0000000..5fca001 Binary files /dev/null and b/cgi/foo_1.png differ diff --git a/cgi/upload.py b/cgi/upload.py index 8c0301e..0a719b0 100644 --- a/cgi/upload.py +++ b/cgi/upload.py @@ -2,6 +2,7 @@ import cgi, os import cgitb; cgitb.enable() +import webbrowser form = cgi.FieldStorage() @@ -20,14 +21,27 @@ ap_file.close() message = 'The file "' + fn + '" was uploaded successfully' - + +elif not (fn == ('.txt') or ('.log') or ('.csv')): + message = 'uploaded file type is not supported' + else: message = 'No file was uploaded' + +with open("/var/log/apache2/access.log", "r") as file: + for last_access_line in file: + pass + file.close() + + print """\ Content-Type: text/html\n + + +

%s

@@ -36,8 +50,8 @@
-Back To Main Page -""" % (message,) +""" % (message,) \ No newline at end of file diff --git a/cgi/upload_1.py b/cgi/upload_1.py new file mode 100644 index 0000000..369ebff --- /dev/null +++ b/cgi/upload_1.py @@ -0,0 +1,69 @@ +#!/usr/bin/python + +import cgi, os +import cgitb; cgitb.enable() +import webbrowser + +form = cgi.FieldStorage() + +# Get filename here. +fileitem = form['filename'] + +# Test if the file was uploaded + +if fileitem.filename: + # strip leading path from file name to avoid + # directory traversal attacks + fn = os.path.basename(fileitem.filename.replace("\\", "/" )) + + if not (fn.split('.')[1] == 'txt' or fn.endswith == 'log' or fn.endswith == 'csv'): + message = 'uploaded file type is not supported' + + print """\ + Content-Type: text/html\n + + +

%s

+ + + + """ % (message,) + + else: + open('/var/www/wou/tmp/' + fn, 'wb').write(fileitem.file.read()) + + ap_file = open('/var/www/wou/data/filelist.txt', 'a') + ap_file.write(fn + "\n") + ap_file.close() + + message = 'The file "' + fn + '" was uploaded successfully' + + print """\ + Content-Type: text/html\n + + +

%s

+
+
+ +
+
+ + + """ % (message,) + +else: + message = 'No file was uploaded' + +print """\ + Content-Type: text/html\n + + +

%s

+ + + + """ % (message,) diff --git a/cgi/upload_2.py b/cgi/upload_2.py new file mode 100644 index 0000000..998f077 --- /dev/null +++ b/cgi/upload_2.py @@ -0,0 +1,71 @@ +#!/usr/bin/python + +import cgi, os +import cgitb; cgitb.enable() +import webbrowser + +form = cgi.FieldStorage() + +# Get filename here. +fileitem = form['filename'] + +# Test if the file was uploaded + +if fileitem.filename: + # strip leading path from file name to avoid + # directory traversal attacks + fn = os.path.basename(fileitem.filename.replace("\\", "/" )) + + # check for supported file type + if not (fn.split('.')[1] == 'txt' or fn.endswith == 'log' or fn.endswith == 'csv'): + message = 'uploaded file type is not supported' + + print """\ + Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) + + else: + open('/var/www/wou/tmp/' + fn, 'wb').write(fileitem.file.read()) + + ap_file = open('/var/www/wou/data/filelist.txt', 'a') + ap_file.write(fn + "\n") + ap_file.close() + + message = 'The file "' + fn + '" was uploaded successfully' + + print """\ + Content-Type: text/html\n + + +

%s

+ +
+
+ +
+
+ + + + """ % (message,) + +else: + message = 'No file was uploaded' + + print """\ + Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) diff --git a/cgi/upload_3.py b/cgi/upload_3.py new file mode 100644 index 0000000..7450f05 --- /dev/null +++ b/cgi/upload_3.py @@ -0,0 +1,75 @@ +#!/usr/bin/python + +import cgi, os +import cgitb; cgitb.enable() + +form = cgi.FieldStorage() + +# Get filename here. +fileitem = form['filename'] + +# Test if the file was uploaded + +if fileitem.filename: + # strip leading path from file name to avoid + # directory traversal attacks + fn = os.path.basename(fileitem.filename.replace("\\", "/" )) + + # check for supported file type + if not (fn.split('.')[1] == 'txt' or fn.endswith == 'log' or fn.endswith == 'csv'): + message = 'uploaded file type is not supported' + + print """\ +Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) + + exit() + + # open file directory and upload file + open('/var/www/wou/tmp/' + fn, 'wb').write(fileitem.file.read()) + + # store filename in filelist + ap_file = open('/var/www/wou/data/filelist.txt', 'a') + ap_file.write(fn + "\n") + ap_file.close() + + message = 'The file ' + fn + ' was uploaded successfully' + + print """\ +Content-Type: text/html\n + + +

%s

+ +
+
+ +
+
+ + + + """ % (message,) + +else: + + # no file submitted + message = 'No file was uploaded' + + print """\ +Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) diff --git a/cgi/upload_4.py b/cgi/upload_4.py new file mode 100644 index 0000000..1797cad --- /dev/null +++ b/cgi/upload_4.py @@ -0,0 +1,70 @@ +#!/usr/bin/python + +import cgi, os +import cgitb; cgitb.enable() + +form = cgi.FieldStorage() + +# Get filename here. +fileitem = form['filename'] + +# Test if the file was uploaded + +if fileitem.filename: + # strip leading path from file name to avoid + # directory traversal attacks + fn = os.path.basename(fileitem.filename.replace("\\", "/" )) + + if not (fn.split('.')[1] == 'txt' or fn.endswith == 'log' or fn.endswith == 'csv'): + message = 'uploaded file type is not supported' + + print """\ +Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) + + exit() + + open('/var/www/wou/tmp/' + fn, 'wb').write(fileitem.file.read()) + + ap_file = open('/var/www/wou/data/filelist.txt', 'a') + ap_file.write(fn + "\n") + ap_file.close() + + message = 'The file ' + fn + ' was uploaded successfully' + + print """\ +Content-Type: text/html\n + + +

%s

+ +
+
+ +
+
+ + + + """ % (message,) + +else: + message = 'No file was uploaded' + + print """\ +Content-Type: text/html\n + + +

%s

+ + + + + """ % (message,) diff --git a/css/styles_1.css b/css/styles_1.css new file mode 100644 index 0000000..072442c --- /dev/null +++ b/css/styles_1.css @@ -0,0 +1,23 @@ +.div_1 { + position: relative; + width: 400px; + heigh: 200px; +} + +form { + position: absolute; + top: 0; +} + +.box-1 { + position: static; +} + +.box-2 { + position: relative; + left: 10px; +} +img { + position: relative; +} + diff --git a/data/filelist.txt b/data/filelist.txt new file mode 100644 index 0000000..008a58e --- /dev/null +++ b/data/filelist.txt @@ -0,0 +1 @@ +data_info_v2.txt \ No newline at end of file diff --git a/index.html b/index.html index e229de9..a58f926 100644 --- a/index.html +++ b/index.html @@ -18,7 +18,6 @@

Log File Upload

Please upload log file for data analysis

-
@@ -28,5 +27,4 @@

Log File Upload

- - + \ No newline at end of file diff --git a/index_1.html b/index_1.html new file mode 100644 index 0000000..e229de9 --- /dev/null +++ b/index_1.html @@ -0,0 +1,32 @@ + + + + + + + Data Analytic Site + + + + + + + + + +

Log File Upload

+ +

Please upload log file for data analysis

+ + +
+ + + + +
+ + + + + diff --git a/log_analytics/README.md b/log_analytics/README.md new file mode 100644 index 0000000..c17ef85 --- /dev/null +++ b/log_analytics/README.md @@ -0,0 +1,2 @@ +# log_analytics +syslog analytic tool diff --git a/log_analytics/cgi/analyst.py b/log_analytics/cgi/analyst.py new file mode 100644 index 0000000..aa33b26 --- /dev/null +++ b/log_analytics/cgi/analyst.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 + +import pandas as pd +from pandas.compat import StringIO +import os +from pathlib import Path + +filepath = '/var/www/wou/tmp/' + +with open("/var/www/wou/data/filelist.txt", "r") as file: + for last_line in file: + pass + file.close() + +lastfilename = last_line + +file_to_open = filepath + lastfilename.rstrip() + +f = open(file_to_open, 'r') +datainfo = f.read() +f.close() + +df = pd.read_csv(file_to_open, names=['Time','Severity','Text'], engine='python') +df.to_html('../output.html', justify='center') + +print("Content-Type: text/html\n") +print("\n") +print("""\ + + + + Data Analytic Output + + + + +""") +print("\n") +print("\n") +print("""\ + +
+ +
+ +
+ + +
+ + + +""") diff --git a/log_analytics/cgi/analyst_1.py b/log_analytics/cgi/analyst_1.py new file mode 100644 index 0000000..589e39b --- /dev/null +++ b/log_analytics/cgi/analyst_1.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 + +import pandas as pd +from pandas.compat import StringIO +import os +from pathlib import Path + +filepath = '/var/www/wou/tmp/' + +with open("/var/www/wou/data/filelist.txt", "r") as file: + for last_line in file: + pass + file.close() + +lastfilename = last_line + +file_to_open = filepath + lastfilename.rstrip() + +f = open(file_to_open, 'r') +datainfo = f.readlines(3) + +lines = list(f) +f.close() + +df = pd.read_csv(file_to_open, names=['Time','Severity','Text'], engine='python') +df.to_html('../output.html', justify='center') + +print("Content-Type: text/html\n") +print("\n") +print("""\ + + + + Data Analytic Output + + + + +""") +print("\n") +print("\n") +print("

Returns of your data filename: {}

".format(lastfilename)) +print("""\ + + + +""".format(lastfilename)) + +print("""\ + +
+ +
+ +
+ + +
+ + + +""") diff --git a/log_analytics/cgi/upload.py b/log_analytics/cgi/upload.py new file mode 100644 index 0000000..d0d18cf --- /dev/null +++ b/log_analytics/cgi/upload.py @@ -0,0 +1,43 @@ +#!/usr/bin/python + +import cgi, os +import cgitb; cgitb.enable() + +form = cgi.FieldStorage() + +# Get filename here. +fileitem = form['filename'] + +# Test if the file was uploaded +if fileitem.filename: + # strip leading path from file name to avoid + # directory traversal attacks + fn = os.path.basename(fileitem.filename.replace("\\", "/" )) + open('/var/www/wou/tmp/' + fn, 'wb').write(fileitem.file.read()) + + ap_file = open('/var/www/wou/data/filelist.txt', 'a') + ap_file.write(fn + "\n") + ap_file.close() + + message = 'The file "' + fn + '" was uploaded successfully' + +else: + message = 'No file was uploaded' + +print """\ +Content-Type: text/html\n + + +

%s

+ +
+
+ +
+
+ + + +""" % (message,) diff --git a/log_analytics/css/styles.css b/log_analytics/css/styles.css new file mode 100644 index 0000000..e112de8 --- /dev/null +++ b/log_analytics/css/styles.css @@ -0,0 +1,20 @@ +.div_1 { + position: relative; + width: 400px; + heigh: 200px; +} + +form { + position: absolute; + top: 0; +} + +.box-1 { + position: static; +} + +.box-2 { + position: relative; + top: 1200px; + left: 30px; +} diff --git a/log_analytics/index.html b/log_analytics/index.html new file mode 100644 index 0000000..42a1fd9 --- /dev/null +++ b/log_analytics/index.html @@ -0,0 +1,32 @@ + + + + + + + Data Analytic Site + + + + + + + + + +

Log File Upload

+ +

Please upload log file for data analysis

+ + +
+
+ + +
+
+ + + + + diff --git a/tmp/log.txt b/tmp/log.txt new file mode 100644 index 0000000..0b5e83d --- /dev/null +++ b/tmp/log.txt @@ -0,0 +1,11 @@ +Nov 1 2019 23:56:27, INFO, "kernel:Initializing cgroup subsys cpuset" +Nov 1 2019 23:56:28, INFO, "kernel:Initializing cgroup subsys cpu" +Nov 1 2019 23:56:29, WARN, "kernel:Initializing cgroup subsys cpuacct" +Nov 1 2019 23:33:57, INFO, kernel:initializing cgroup subsys cpuset +Nov 1 2019 23:34:57, INFO, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:35:57, INFO, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:36:57, WARN, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:37:57, INFO, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:38:57, ERR, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:39:57, INFO, kernel:initializing cgroup subsys cpuset +Nov 1 2019 22:40:57, INFO, kernel:initializing cgroup subsys cpuset \ No newline at end of file