commit 4a183d6f90a9a075eeae3a3b0a2763b55787b5f7 Author: Ignace Date: Mon Mar 24 21:33:34 2025 +0000 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..09173cc --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.rrd diff --git a/__pycache__/addEvent.cpython-310.pyc b/__pycache__/addEvent.cpython-310.pyc new file mode 100644 index 0000000..cbb0391 Binary files /dev/null and b/__pycache__/addEvent.cpython-310.pyc differ diff --git a/__pycache__/addEvent.cpython-312.pyc b/__pycache__/addEvent.cpython-312.pyc new file mode 100644 index 0000000..6f193eb Binary files /dev/null and b/__pycache__/addEvent.cpython-312.pyc differ diff --git a/__pycache__/addEvent.cpython-36.pyc b/__pycache__/addEvent.cpython-36.pyc new file mode 100644 index 0000000..ce2da6e Binary files /dev/null and b/__pycache__/addEvent.cpython-36.pyc differ diff --git a/__pycache__/sendAuthMail.cpython-310.pyc b/__pycache__/sendAuthMail.cpython-310.pyc new file mode 100644 index 0000000..457f21c Binary files /dev/null and b/__pycache__/sendAuthMail.cpython-310.pyc differ diff --git a/__pycache__/sendAuthMail.cpython-312.pyc b/__pycache__/sendAuthMail.cpython-312.pyc new file mode 100644 index 0000000..6877cd4 Binary files /dev/null and b/__pycache__/sendAuthMail.cpython-312.pyc differ diff --git a/__pycache__/sendAuthMail.cpython-36.pyc b/__pycache__/sendAuthMail.cpython-36.pyc new file mode 100644 index 0000000..0c2a5f9 Binary files /dev/null and b/__pycache__/sendAuthMail.cpython-36.pyc differ diff --git a/addEvent.py b/addEvent.py new file mode 100644 index 0000000..2957503 --- /dev/null +++ b/addEvent.py @@ -0,0 +1,26 @@ +''' + send a text to the local event-log + usage as library or from the CLI: + python3 addEvent.py "ERROR,key,timeout,count, text" +''' +import sys +import socket + + +def send_event(text): + ''' + text must be a str + ''' + + host = "localhost" + port = 1234 # The same port as used by the server + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((host, port)) + s.sendall(bytes(text, 'utf-8')) + + # data = s.recv(1024) # nothing should be returned.... + s.close() + +if __name__ == "__main__": + if len(sys.argv) > 1: + send_event(sys.argv[1].encode('utf-8')) diff --git a/addEvent.sh b/addEvent.sh new file mode 100755 index 0000000..2d853ca --- /dev/null +++ b/addEvent.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# add an event to the event log +# ./addEvent.sh "ERROR,key,timeout,count, text" + +echo "$1" | telnet localhost 1234 >/dev/null 2>&1 diff --git a/checkMeteoService b/checkMeteoService new file mode 100755 index 0000000..5cbc98a --- /dev/null +++ b/checkMeteoService @@ -0,0 +1,6 @@ +#!/bin/bash + + +if [[ $(tail -200 /tmp/meteoo.err | grep -c Traceback) -gt 0 ]]; then + ./addEvent.sh "ERROR,"meteoo",1,1,Meteoo service programming error detected" +fi diff --git a/checkProcesses b/checkProcesses new file mode 100755 index 0000000..417fceb --- /dev/null +++ b/checkProcesses @@ -0,0 +1,11 @@ +#!/bin/bash + +if [[ $(ps ax | grep 'eventService.py' | grep -v grep | wc -l) -eq 0 ]] ; then + python3 sendAuthMail.py "Service -eventService daemon NOT running - event-alerting disabled" +fi + +cat processesToCheck.txt | while read -r LINE; do + if [[ $(ps ax | grep "${LINE}" | grep -v grep | wc -l) -eq 0 ]] ; then + ./addEvent.sh "ERROR,${LINE},3,10,Service ${LINE} NOT running anymore" + fi +done diff --git a/createGraphs b/createGraphs new file mode 100755 index 0000000..06bea0e --- /dev/null +++ b/createGraphs @@ -0,0 +1,90 @@ +#!/bin/bash + +# this script is started: +# - manual, from the monitor folder +# - cronwise, from the project-HOME folder +# usage: +# ./createGraphs 1d # or 1h or 2w or 1y +cd monitor >/dev/null 2>&1 # this only works if we are in the HOME folder, else its ignored with an error + +# the output of this script can only be visualized under a webserver +FOLDER='/var/www/suy.nl/m' + +function graph { + P=${1} + mkdir -p ${FOLDER}/${P} + rrdtool graph ${FOLDER}/${P}/disk.png \ + --imgformat PNG \ + --width 640 \ + --height 240 \ + --upper-limit 100 \ + --lower-limit 0 \ + --start -${P} \ + --watermark "$(date)" \ + --vertical-label 'Percentage' \ + --title 'Usage last '${P} \ + 'DEF:root=disk.rrd:root:AVERAGE' \ + 'DEF:memt=mem.rrd:total:AVERAGE' \ + 'DEF:memu=mem.rrd:used:AVERAGE' \ + 'CDEF:memp=memu,memt,/' \ + 'CDEF:memq=memp,100,*' \ + 'AREA:root#0000FF:"disk /"' \ + 'LINE2:memq#00FF00:"Memory"' + + rrdtool graph ${FOLDER}/${P}/load.png \ + --imgformat PNG \ + --width 640 \ + --height 240 \ + --upper-limit 1 \ + --lower-limit 0 \ + --start -${P} \ + --watermark "$(date)" \ + --title 'processor load in '${P} \ + --vertical-label 'Processors used' \ + 'DEF:min1=load.rrd:min1:AVERAGE' \ + 'DEF:min5=load.rrd:min5:AVERAGE' \ + 'DEF:min15=load.rrd:min15:AVERAGE' \ + 'AREA:min1#FF0000:"1 minute"' \ + 'AREA:min5#FFA500:"5 minutes"' \ + 'LINE1:min5#FF0000:' \ + 'AREA:min15#FFD801:"15 minutes"' \ + 'LINE1:min15#FFA500:' + + rrdtool graph ${FOLDER}/${P}/cpu.png \ + --imgformat PNG \ + --width 640 \ + --height 240 \ + --upper-limit 10 \ + --lower-limit 0 \ + --start -${P} \ + --watermark "$(date)" \ + --vertical-label 'Percentage' \ + --title 'CPU usage in '${P} \ + 'DEF:us=cpu.rrd:us:AVERAGE' \ + 'DEF:sy=cpu.rrd:sy:AVERAGE' \ + 'DEF:ni=cpu.rrd:ni:AVERAGE' \ + 'AREA:sy#00FF00:"sy"' \ + 'STACK:us#0000FF:"us"' \ + 'STACK:ni#FF0000:"ni"' + + rrdtool graph ${FOLDER}/${P}/network.png \ + --imgformat PNG \ + --width 640 \ + --height 240 \ + --start -${P} \ + --watermark "$(date)" \ + --vertical-label 'Bytes' \ + --upper-limit 10000 \ + --lower-limit 0 \ + --title 'Network last '${P} \ + --lower-limit 0 \ + 'DEF:in=network.rrd:in:AVERAGE' \ + 'DEF:out=network.rrd:out:AVERAGE' \ + 'AREA:out#00FF00:"out"' \ + 'LINE2:in#0000FF:"in"' +} + +for var in "$@" +do + graph "$var" +done diff --git a/createRRDs.sh b/createRRDs.sh new file mode 100644 index 0000000..dd128dc --- /dev/null +++ b/createRRDs.sh @@ -0,0 +1,72 @@ +# create RRDs: +# 600 samples of 5 minutes (2 days and 2 hours) +# 700 samples of 30 minutes (2 days and 2 hours, plus 12.5 days) +# 775 samples of 2 hours (above + 50 days) +# 797 samples of 1 day (above + 732 days, rounded up to 797) + +rrdtool create disk.rrd \ + DS:root:GAUGE:600:U:U \ + DS:boot:GAUGE:600:U:U \ + DS:home:GAUGE:600:U:U \ + DS:opt:GAUGE:600:U:U \ + DS:tmp:GAUGE:600:U:U \ + DS:var:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 + +rrdtool create load.rrd \ + DS:min1:GAUGE:600:U:U \ + DS:min5:GAUGE:600:U:U \ + DS:min15:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 + +rrdtool create cpu.rrd \ + DS:us:GAUGE:600:U:U \ + DS:sy:GAUGE:600:U:U \ + DS:ni:GAUGE:600:U:U \ + DS:id:GAUGE:600:U:U \ + DS:wa:GAUGE:600:U:U \ + DS:hi:GAUGE:600:U:U \ + DS:si:GAUGE:600:U:U \ + DS:st:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 + +rrdtool create mem.rrd \ + DS:total:GAUGE:600:U:U \ + DS:free:GAUGE:600:U:U \ + DS:used:GAUGE:600:U:U \ + DS:cached:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 + +rrdtool create swap.rrd \ + DS:total:GAUGE:600:U:U \ + DS:free:GAUGE:600:U:U \ + DS:used:GAUGE:600:U:U \ + DS:avail:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 + +rrdtool create network.rrd \ + DS:in:GAUGE:600:U:U \ + DS:out:GAUGE:600:U:U \ + RRA:AVERAGE:0.5:1:600 \ + RRA:AVERAGE:0.5:6:700 \ + RRA:AVERAGE:0.5:24:775 \ + RRA:AVERAGE:0.5:288:797 \ + RRA:MAX:0.5:1:600 \ + RRA:MAX:0.5:6:700 \ + RRA:MAX:0.5:24:775 \ + RRA:MAX:0.5:288:797 diff --git a/emailEscalationMessage.txt b/emailEscalationMessage.txt new file mode 100644 index 0000000..5ddd944 --- /dev/null +++ b/emailEscalationMessage.txt @@ -0,0 +1,8 @@ +Hmmm. +Ignace heeft al bijna 4 dagen geen email gelezen. +Dat is enigzins ongewoon. +Als je weet dat daar een goede reden voor is (en alles goed is verder), hoef je niets meer met deze mail. +Als er iets mis is, ga dan naar https://www.suy.nl/cloud/ , login als katja, met haar geboorte-datum DDMMYYYY. + +Noot. Zolang als die email niet wordt gelezen, krijg je dit bericht met enige regelmaat. :-( + diff --git a/eventService.py b/eventService.py new file mode 100755 index 0000000..4696b68 --- /dev/null +++ b/eventService.py @@ -0,0 +1,86 @@ +#!/usr/bin/python3 + +import os +from twisted.internet import protocol, reactor, endpoints +from datetime import datetime +from sendAuthMail import SendOff + +# HOMEIP='213.46.222.164' +with open('/root/SAVEIPS') as f: + SAVEIPS = f.read().splitlines() + +# run this as a daemon +# # nohup python3 eventService.py & +# start-stop-daemon -SbCv -x /opt/Monitor/monitor/eventService.py + +def log_event(text): + with open('/tmp/event.log', 'a') as f1: + f1.write(text) + + +class Event(protocol.Protocol): + + events = {} + illegal_events = 0 + + def dataReceived(self, data): + ''' + data contains the event, which a 1-liner like: + "type,key,repetition-treshold,time-out,event-text" + type: ERROR, BLOCK, WARNING + key: error-id, incse of BLOCK this is an IP-address + repetition-treshold and time-out: if 5 times in 10 minutes, take action (ERROR: sendmail, BLOCK: block thre ip-address) + ''' + event = str(data, 'utf-8').strip() + now = datetime.now() + now_label = now.strftime('%d.%H:%M:%S') + + # put ther raw event in the log + log_event("{} {}\n".format(now_label, event)) + + # saving and dealing with the event comes here.... + if not (event.count(',') == 4 and (event.startswith('ERROR') or event.startswith('WARN') or event.startswith('BLOCK'))): + event="ERROR,ILLEGAL,3,10,Misformed event has been sent to the logger" + + (evtype, evkey, evrep, evtimeout, evtext) = event.split(',') + if evtype == 'ERROR' or evtype == 'BLOCK': + new_timestamp_list = [] + if evkey in self.events.keys(): + timestamp_list = self.events[evkey] + for timestamp in timestamp_list: + # if timestamp is younger than now - evtimeout, add it to the new list + delta_time = now - timestamp + if delta_time.total_seconds() < (60.0 * int(evtimeout)): + new_timestamp_list.append(timestamp) + new_timestamp_list.append(now) + self.events[evkey] = new_timestamp_list + + # print(self.events) + + if len(new_timestamp_list) >= int(evrep): + # delete the event in the key-list + del self.events[evkey] + # now send a message or block an ip + if evtype == 'ERROR': + # send an email + log_event("***** Previous line caused an alert *****\n") + s = SendOff(event) + s.ignace() + elif evtype == 'BLOCK': + # evkey is the IP address, check if its not my own + if not evkey in SAVEIPS: + log_event("***** Blocking IP {} full access *****\n".format(evkey)) + # block the IP address + os.system("ufw insert 1 deny from {}".format(evkey)) + # os.system("ufw insert 1 deny log from {}".format(evkey)) + + + +class EventFactory(protocol.Factory): + def buildProtocol(self, addr): + return Event() + + +endpoints.serverFromString(reactor, "tcp:1234").listen(EventFactory()) +reactor.run() # pylint: disable=no-member + diff --git a/monitor b/monitor new file mode 100755 index 0000000..ea714c1 --- /dev/null +++ b/monitor @@ -0,0 +1,3 @@ +#!/bin/sh +df | /opt/monitor/monitorDisks.py +top -b -n 5 -p0 | tail -9 | /opt/monitor/monitorTop.py diff --git a/monitorCpu.py b/monitorCpu.py new file mode 100755 index 0000000..4a08c19 --- /dev/null +++ b/monitorCpu.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import sys +import re +import datetime +import os +from rrdtool import update as rrd_update # pylint: disable=no-name-in-module + +SUBFOLDER = '' +#print(datetime.datetime.now().strftime("%V")) + +# root@plex:/opt/monitor# mpstat +# Linux 5.4.0-125-generic (plex) 01/16/2024 _x86_64_ (4 CPU) +# 03:57:00 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %gnice %idle +# 03:57:00 PM all 0.72 0.01 0.14 0.28 0.00 0.03 0.00 0.00 0.00 98.82 + +re2 = re.compile(r"\d+\.\d+") + +def save(filename, string): + fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a") + fo.writelines( string + "\n" ) + fo.close() + +for line in sys.stdin: + if 'all' in line: + cpus = re2.findall(line) + # print(";".join(cpus)) + ret = rrd_update(os.path.join(SUBFOLDER, 'cpu.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}:{6}:{7}'.format( cpus[0], cpus[2], cpus[1], 0,0,0,0,0) ) + if float(cpus[0]) > 5.0: + send_event("ERROR,cpus,3,7,Unexpected high cpu for the last few minutes: {}".format(cpus[0])) + diff --git a/monitorDisks.py b/monitorDisks.py new file mode 100755 index 0000000..74e5424 --- /dev/null +++ b/monitorDisks.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 +import sys +import re +import os +from rrdtool import update as rrd_update # pylint: disable=no-name-in-module +from addEvent import send_event + +SUBFOLDER = '' +# print(datetime.datetime.now().strftime("%V")) + +re1 = re.compile(r"load average:\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2})") +re2 = re.compile(r"\d+\.\d+") +re3 = re.compile(r"\d+") + +root = 0 +boot = 0 +home = 0 +opt = 0 +tmp = 0 +var = 0 + +for line in sys.stdin: + if '/' in line: + words = line.split() + if words[-1] == '/': + root = words[-2].replace('%', '') + if words[-1] == '/mnt/hdrive': + boot = words[-2].replace('%', '') + if words[-1] == '/home': + home = words[-2].replace('%', '') + if words[-1] == '/opt': + opt = words[-2].replace('%', '') + if words[-1] == '/tmp': + tmp = words[-2].replace('%', '') + if words[-1] == '/var': + var = words[-2].replace('%', '') + +# print("{0}:{1}:{2}:{3}:{4}:{5}".format(root, boot, home, opt, tmp, var)) +ret = rrd_update(os.path.join(SUBFOLDER, 'disk.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}'.format(root, boot, home, opt, tmp, var)) + +if float(root) > 90.0: + send_event("ERROR,disk,5,30,Unexpected high disk-usagefor the last few minutes: {}".format(root)) diff --git a/monitorEmails.py b/monitorEmails.py new file mode 100755 index 0000000..3f4622b --- /dev/null +++ b/monitorEmails.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 + +import imaplib +import email +import datetime +import os +from pathlib import Path + +## constants +UNREADDAYS = 3 +FILE_SMS = '/tmp/emailcheck.sms' +FILE_WARNING = '/tmp/emailcheck.warning' +FILE_ESCALATE = '/tmp/emailcheck.ESCALATE' + +def write_file(path, text): + file1 = open(path, "w") + file1.write(text) + file1.close() + + +# calculate date 3 days ago +threedaysago = datetime.datetime.now() - datetime.timedelta(days = UNREADDAYS) +threedaysago_str = threedaysago.strftime('(SINCE "%d-%b-%Y")') +# print(threedaysago_str) + +# Connect to inbox +imap_server = imaplib.IMAP4_SSL(host='imap.strato.com') +imap_server.login('ignace@suy.nl', 'Ghdjh&653gh*') +imap_server.select() # Default is `INBOX` + +#Search for all UIDs first +result1, data1 = imap_server.uid('search', threedaysago_str) +mails_total = len(data1[0].split()) + +result2, data2 = imap_server.uid('search', threedaysago_str, 'UNSEEN') +mails_unseen = len(data2[0].split()) + +print("{} mails {}, {}=={}".format(datetime.datetime.now(), threedaysago_str, mails_total, mails_unseen)) + +# print("{} {}".format(mails_total, mails_unseen)) +if (mails_total > mails_unseen) or (mails_total == 0): + # some mail has been read in the past days, all is well + # delete sms file, warning file + if os.path.exists(FILE_SMS): + os.remove(FILE_SMS) + if os.path.exists(FILE_WARNING): + os.remove(FILE_WARNING) + if os.path.exists(FILE_ESCALATE): + os.remove(FILE_ESCALATE) + print(" ->Ok") +else: + # no mails have been read in the past days + # check if a earlier warning file has been created + if os.path.exists(FILE_WARNING): + if (not os.path.exists(FILE_ESCALATE)): + # serious shit, escalate + write_file(FILE_ESCALATE, "Escalation has taken place") + write_file(FILE_SMS, "SUY-CLOUD: You have received a mail from cloudserver@suy.nl - please check!.") + print(" ->Escalate") + else: + # create a warning file + write_file(FILE_WARNING, "In another half day, if no email is read, escalation will take place") + write_file(FILE_SMS, "SUY-CLOUD: You have not read mail for 3 days, do this immediately pls or escalate.") + print(" ->Warning") + diff --git a/monitorLoad.py b/monitorLoad.py new file mode 100755 index 0000000..01028b1 --- /dev/null +++ b/monitorLoad.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +import sys +import re +import datetime +import os +from rrdtool import update as rrd_update # pylint: disable=no-name-in-module + +SUBFOLDER = '' +#print(datetime.datetime.now().strftime("%V")) +# root@plex:/opt/monitor# uptime +# 16:05:39 up 422 days, 1:05, 1 user, load average: 0.01, 0.03, 0.01 + +re2 = re.compile(r"\d+\.\d+") + +def save(filename, string): + fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a") + fo.writelines( string + "\n" ) + fo.close() + +for line in sys.stdin: + if 'load average' in line: + avgs = re2.findall(line) + # print(";".join(avgs)) + ret = rrd_update(os.path.join(SUBFOLDER, 'load.rrd'), 'N:{0}:{1}:{2}'.format(avgs[0],avgs[1],avgs[2])) + if float(avgs[2]) > 5.0: + send_event("ERROR,avgs,3,7,Unexpected high average load15 for the last minutes: {}".format(avgs[2])) diff --git a/monitorMem.py b/monitorMem.py new file mode 100755 index 0000000..9f22ad0 --- /dev/null +++ b/monitorMem.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import sys +import re +import datetime +import os +from rrdtool import update as rrd_update # pylint: disable=no-name-in-module + +SUBFOLDER = '' +#print(datetime.datetime.now().strftime("%V")) + +# root@plex:/opt/monitor# free +# total used free shared buff/cache available +# Mem: 5904284 353696 190204 3524 5360384 5238048 +# Swap: 2097148 248064 1849084 + + +re3 = re.compile(r"\d+") + +def save(filename, string): + fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a") + fo.writelines( string + "\n" ) + fo.close() + +for line in sys.stdin: + if 'Mem:' in line: + mems = re3.findall(line) + # save('mem_', ";".join(mems)) + print('N:{0}:{1}:{2}:{3}'.format( mems[0], mems[2], mems[1], int(mems[0])-int(mems[5]))) + ret = rrd_update(os.path.join(SUBFOLDER, 'mem.rrd'), 'N:{0}:{1}:{2}:{3}'.format( mems[0], mems[2], mems[1], int(mems[0])-int(mems[5])) ) + diff --git a/monitorNet.sh b/monitorNet.sh new file mode 100755 index 0000000..40b7f56 --- /dev/null +++ b/monitorNet.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +LOAD=$( ifstat 120 1 | tail -1 ) + +IN=$( echo ${LOAD} | awk -F' ' '{print $1}' ) +IN=$( bc <<< "scale=0; (${IN} * 1000)/1" ) + +OUT=$( echo ${LOAD} | awk -F' ' '{print $2}') +OUT=$( bc <<< "scale=0; (${OUT} * 1000)/1" ) + +# echo " Net: ${LOAD} ${IN} ${OUT}" +rrdtool update network.rrd N:${IN}:${OUT} diff --git a/monitorTop.py b/monitorTop.py new file mode 100755 index 0000000..6660efa --- /dev/null +++ b/monitorTop.py @@ -0,0 +1,30 @@ +#!/usr/bin/python3 +import sys +import re +import datetime +import os +from rrdtool import update as rrd_update # pylint: disable=no-name-in-module +from addEvent import send_event + +SUBFOLDER = '' +#print(datetime.datetime.now().strftime("%V")) + +# top - 13:35:26 up 2 days, 23:46, 2 users, load average: 0,00, 0,03, 0,050 +# top - 13:44:48 up 187 days, 38 min, 1 user, load average: 0.02, 0.08, 0.24 +re1 = re.compile('load average:\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2})') +re2 = re.compile("\d+\.\d+") +re3 = re.compile("\d+") + +def save(filename, string): + fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a") + fo.writelines( string + "\n" ) + fo.close() + +for line in sys.stdin: + if 'Cpu(s):' in line: + cpus = re2.findall(line) + # print(";".join(cpus)) + ret = rrd_update(os.path.join(SUBFOLDER, 'cpu.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}:{6}:{7}'.format( cpus[0], cpus[1], cpus[2], cpus[3], cpus[4], cpus[5], cpus[6], cpus[7]) ) + if float(cpus[0]) > 20.0: # 20 of 100% + send_event("ERROR,cpus,3,7,Unexpected high cpu for the last few minutes: {}".format(cpus[0])) + diff --git a/monitorTop.sh b/monitorTop.sh new file mode 100755 index 0000000..19755a4 --- /dev/null +++ b/monitorTop.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# rrd_update(os.path.join(SUBFOLDER, 'load.rrd'), 'N:{0}:{1}:{2}'.f + +IFS=' ' read -r -a 'HH' <<< "$(cat /proc/loadavg)" +rrdtool update load.rrd N:${HH[0]}:${HH[1]}:${HH[2]} + diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..8502fb1 --- /dev/null +++ b/nohup.out @@ -0,0 +1,53 @@ +Unhandled Error +Traceback (most recent call last): + File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 103, in callWithLogger + return callWithContext({"system": lp}, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 86, in callWithContext + return context.call({ILogContext: newCtx}, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 122, in callWithContext + return self.currentContext().callWithContext(ctx, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 85, in callWithContext + return func(*args,**kw) +--- --- + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite + why = selectable.doRead() + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 243, in doRead + return self._dataReceived(data) + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 249, in _dataReceived + rval = self.protocol.dataReceived(data) + File "eventService.py", line 46, in dataReceived + new_list.append(e) +builtins.MemoryError: + +Unhandled Error +Traceback (most recent call last): + File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 103, in callWithLogger + return callWithContext({"system": lp}, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 86, in callWithContext + return context.call({ILogContext: newCtx}, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 122, in callWithContext + return self.currentContext().callWithContext(ctx, func, *args, **kw) + File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 85, in callWithContext + return func(*args,**kw) +--- --- + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite + why = selectable.doRead() + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 243, in doRead + return self._dataReceived(data) + File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 249, in _dataReceived + rval = self.protocol.dataReceived(data) + File "eventService.py", line 48, in dataReceived + new_list.append(new_event) +builtins.MemoryError: + +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted +Rule inserted diff --git a/processesToCheck.txt b/processesToCheck.txt new file mode 100644 index 0000000..1696bd8 --- /dev/null +++ b/processesToCheck.txt @@ -0,0 +1,5 @@ +/usr/sbin/mysqld +/usr/sbin/apache2 +/usr/local/bin/gitea +/usr/lib/plexmediaserver/Plex +/usr/sbin/sshd diff --git a/run5 b/run5 new file mode 100755 index 0000000..71986d0 --- /dev/null +++ b/run5 @@ -0,0 +1,26 @@ +#!/bin/bash +# runs every 2 minutes + +echo "$( date ) monitor run5" + +# log the busiest processes, for trouble shooting +echo "----------- $(date) --------------------------------------------" >>/tmp/processes.log +ps -eo pcpu,pid,user,args | sort -k 1 -r | head -5 >>/tmp/processes.log + +# check the most relevant services: +# ./checkProcesses + +# all this is started from cron, met HOME folder -> the root of this project +cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored + +# monitorTop.sh +df | ./monitorDisks.py +free | ./monitorMem.py +# top -b -n 5 -p0 | tail -9 | ./monitorTop.py +uptime | ./monitorLoad.py +tail -5 /tmp/mpstat.log | grep all | tail -1 | ./monitorCpu.py + +./monitorNet.sh & + +./createGraphs 3h + diff --git a/run60 b/run60 new file mode 100755 index 0000000..4b222f0 --- /dev/null +++ b/run60 @@ -0,0 +1,12 @@ +#!/bin/bash +# runs every 5 minutes + +echo "$( date ) monitor run60" +# all this is started from cron, met HOME folder -> the root of this project +cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored + +./createGraphs 1d 2w 1y + +./checkMeteoService + +nethogs -v1 -d30 -c2 -t diff --git a/send2Mail.py b/send2Mail.py new file mode 100755 index 0000000..642e605 --- /dev/null +++ b/send2Mail.py @@ -0,0 +1,51 @@ +# call this program with: +# echo "blah" | python3 sendAuthMail.py +# cat | python3 sendAuthMail.py +# python3 sendAuthMail.py 'blah' + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib +import sys + + +class SendOff(): + def __init__(self, message): + self.message = message + + def ignace(self): + # Step 2 - Create message object instance + msg = MIMEMultipart() + + smtp_ssl_host = 'mail.mijndomein.nl' # smtp.mail.yahoo.com + smtp_ssl_port = 465 + + # Step 4 - Declare SMTP credentials + password = "Cloud%67HgDd4569djs5" + username = "cloudserver@suy.nu" + sender = 'cloudserver@suy.nu' + targets = ['ignace@suy.nu'] + + # msg = MIMEText('Hi, how are you today?') + msg.attach(MIMEText(self.message, 'plain')) + msg['Subject'] = "www.suy.nu - system message" + msg['From'] = sender + msg['To'] = ', '.join(targets) + + server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port) + server.login(username, password) + server.sendmail(sender, targets, msg.as_string()) + + server.quit() + + +if __name__ == "__main__": + body = '' + if len(sys.argv) > 1: + body = sys.argv[1] + else: + for line in sys.stdin: + body += line + if len(body) > 0: + s = SendOff(body) + s.ignace() diff --git a/sendAuthMail.py b/sendAuthMail.py new file mode 100755 index 0000000..fd07073 --- /dev/null +++ b/sendAuthMail.py @@ -0,0 +1,51 @@ +# call this program with: +# echo "blah" | python3 sendAuthMail.py +# cat | python3 sendAuthMail.py +# python3 sendAuthMail.py 'blah' + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib +import sys + + +class SendOff(): + def __init__(self, message): + self.message = message + + def ignace(self): + # Step 2 - Create message object instance + msg = MIMEMultipart() + + smtp_ssl_host = 'smtp.strato.com' # smtp.mail.yahoo.com + smtp_ssl_port = 465 + + # Step 4 - Declare SMTP credentials + password = "Cloud%67HgDd4569djs5" + username = "cloudserver@suy.nl" + sender = 'cloudserver@suy.nl' + targets = ['ignace@suy.nl'] + + # msg = MIMEText('Hi, how are you today?') + msg.attach(MIMEText(self.message, 'plain')) + msg['Subject'] = "www.suy.nl - system message" + msg['From'] = sender + msg['To'] = ', '.join(targets) + + server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port) + server.login(username, password) + server.sendmail(sender, targets, msg.as_string()) + + server.quit() + + +if __name__ == "__main__": + body = '' + if len(sys.argv) > 1: + body = sys.argv[1] + else: + for line in sys.stdin: + body += line + if len(body) > 0: + s = SendOff(body) + s.ignace() diff --git a/sendAuthMail.py.2019 b/sendAuthMail.py.2019 new file mode 100755 index 0000000..50f467a --- /dev/null +++ b/sendAuthMail.py.2019 @@ -0,0 +1,58 @@ +# call this program with: +# echo "blah" | python3 sendAuthMail.py +# cat | python3 sendAuthMail.py +# python3 sendAuthMail.py 'blah' + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib +import sys + + +class SendOff(): + def __init__(self, message): + self.message = message + + def ignace(self): + # Step 2 - Create message object instance + msg = MIMEMultipart() + + # Step 4 - Declare SMTP credentials + password = "Cloud%67HgDd4569djs5" + username = "cloudserver@suy.nu" + smtphost = "mail.mijndomein.nl" + + # Step 5 - Declare message elements + msg['From'] = "cloudserver@suy.nu" + msg['To'] = "ignace@suy.nu" + msg['Subject'] = "www.suy.nu - system message" + + # Step 6 - Add the message body to the object instance + msg.attach(MIMEText(self.message, 'plain')) + + # Step 7 - Create the server connection + server = smtplib.SMTP(smtphost) + + # Step 8 - Switch the connection over to TLS encryption + server.starttls() + + # Step 9 - Authenticate with the server + server.login(username, password) + + # Step 10 - Send the message + server.sendmail(msg['From'], msg['To'], msg.as_string()) + + # Step 11 - Disconnect + server.quit() + + +if __name__ == "__main__": + body = '' + if len(sys.argv) > 1: + body = sys.argv[1] + else: + for line in sys.stdin: + body += line + if len(body) > 0: + s = SendOff(body) + s.ignace() diff --git a/sendEscalationMail.py b/sendEscalationMail.py new file mode 100755 index 0000000..8052f1e --- /dev/null +++ b/sendEscalationMail.py @@ -0,0 +1,59 @@ +# call this program with: +# echo "blah" | python3 sendAuthMail.py +# cat | python3 sendAuthMail.py +# python3 sendAuthMail.py 'blah' + +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib +import sys + +TEXTM = ''' +Hmmm. +Ignace heeft al bijna 4 dagen geen email gelezen. +Dat is enigzins ongewoon. +Als je weet dat daar een goede reden voor is (en alles goed is verder), hoef je niets meer met deze mail. +Als er iets mis is, ga dan naar https://www.suy.nl/cloud/ , login als katja, met haar geboorte-datum DDMMYYYY. + +Noot. Zolang als die email niet wordt gelezen, krijg je dit bericht met enige regelmaat. :-( +''' + + +class SendOff(): + def __init__(self, message): + self.message = message + + def ignace(self): + # Step 2 - Create message object instance + msg = MIMEMultipart() + + smtp_ssl_host = 'smtp.strato.com' # smtp.mail.yahoo.com + smtp_ssl_port = 465 + + # Step 4 - Declare SMTP credentials + password = "Cloud%67HgDd4569djs5" + username = "cloudserver@suy.nl" + sender = 'cloudserver@suy.nl' + # targets = ['ignace@suy.nl','janine@demaat.info', 'xander@suy.nl', 'querijn@suy.nl'] + targets = ['ignace@suy.nl','ignace.suy@gmail.com'] + + # msg = MIMEText('Hi, how are you today?') + msg.attach(MIMEText(self.message, 'plain')) + msg['Subject'] = "www.suy.nl - Escalation message" + msg['From'] = sender + msg['To'] = ', '.join(targets) + + server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port) + server.login(username, password) + server.sendmail(sender, targets, msg.as_string()) + + server.quit() + + +if __name__ == "__main__": + body = TEXTM + if len(sys.argv) > 1: + body = sys.argv[1] + if len(body) > 0: + s = SendOff(body) + s.ignace() diff --git a/updateRRDs b/updateRRDs new file mode 100755 index 0000000..088441c --- /dev/null +++ b/updateRRDs @@ -0,0 +1,12 @@ +#!/bin/bash +# runs every 5 minutes + +echo "$( date ) updateRRDs monitor" +# all this is started from cron, met HOME folder -> the root of this project +cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored + +df | ./monitorDisks.py +top -b -n 5 -p0 | tail -9 | ./monitorTop.py +# top -b -n 1 | tee /tmp/top/$(date +"%H%M").log | ./monitorTop.py + +# ./monitorNet.sh diff --git a/workspace.code-workspace b/workspace.code-workspace new file mode 100644 index 0000000..bab1b7f --- /dev/null +++ b/workspace.code-workspace @@ -0,0 +1,8 @@ +{ + "folders": [ + { + "path": ".." + } + ], + "settings": {} +} \ No newline at end of file