first commit

.gitignore

@@ -0,0 +1 @@
+*.rrd

addEvent.py

@@ -0,0 +1,26 @@
+'''
+    send a text to the local event-log
+    usage as library or from the CLI:
+      python3 addEvent.py "ERROR,key,timeout,count, text"
+'''
+import sys
+import socket
+
+
+def send_event(text):
+    '''
+    text must be a str
+    '''
+
+    host = "localhost"
+    port = 1234                   # The same port as used by the server
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.connect((host, port))
+    s.sendall(bytes(text, 'utf-8'))
+
+    # data = s.recv(1024)  # nothing should be returned....
+    s.close()
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        send_event(sys.argv[1].encode('utf-8'))

addEvent.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# add an event to the event log
+#   ./addEvent.sh "ERROR,key,timeout,count, text"
+
+echo "$1" | telnet localhost 1234 >/dev/null 2>&1

checkMeteoService

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+
+if [[ $(tail -200 /tmp/meteoo.err | grep -c Traceback) -gt 0 ]]; then
+    ./addEvent.sh "ERROR,"meteoo",1,1,Meteoo service programming error detected"
+fi

checkProcesses

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ $(ps ax | grep 'eventService.py' | grep -v grep | wc -l) -eq 0 ]] ; then
+    python3 sendAuthMail.py "Service -eventService daemon NOT running - event-alerting disabled"
+fi
+
+cat processesToCheck.txt | while read -r LINE; do
+    if [[ $(ps ax | grep "${LINE}" | grep -v grep | wc -l) -eq 0 ]] ; then
+        ./addEvent.sh "ERROR,${LINE},3,10,Service ${LINE} NOT running anymore"
+    fi
+done

createGraphs

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# this script is started:
+#  - manual, from the monitor folder
+#  - cronwise, from the project-HOME folder
+# usage:
+#  ./createGraphs 1d #  or 1h or 2w or 1y
+cd monitor >/dev/null 2>&1 # this only works if we are in the HOME folder, else its ignored with an error
+
+# the output of this script can only be visualized under a webserver
+FOLDER='/var/www/suy.nl/m'
+
+function graph {
+  P=${1}
+  mkdir -p ${FOLDER}/${P}
+  rrdtool graph ${FOLDER}/${P}/disk.png        \
+	--imgformat PNG                        \
+	--width 640                            \
+	--height 240                           \
+        --upper-limit 100                      \
+        --lower-limit 0                        \
+	--start -${P}                          \
+        --watermark "$(date)"                  \
+	--vertical-label 'Percentage'          \
+	--title 'Usage last '${P}              \
+	'DEF:root=disk.rrd:root:AVERAGE'       \
+	'DEF:memt=mem.rrd:total:AVERAGE'       \
+	'DEF:memu=mem.rrd:used:AVERAGE'        \
+	'CDEF:memp=memu,memt,/'                \
+	'CDEF:memq=memp,100,*'                 \
+	'AREA:root#0000FF:"disk /"'            \
+	'LINE2:memq#00FF00:"Memory"'
+	
+  rrdtool graph ${FOLDER}/${P}/load.png        \
+	--imgformat PNG                        \
+	--width 640                            \
+	--height 240                           \
+        --upper-limit 1                        \
+        --lower-limit 0                        \
+	--start -${P}                          \
+        --watermark "$(date)"                  \
+	--title 'processor load in '${P}       \
+	--vertical-label 'Processors used'     \
+	'DEF:min1=load.rrd:min1:AVERAGE'       \
+	'DEF:min5=load.rrd:min5:AVERAGE'       \
+	'DEF:min15=load.rrd:min15:AVERAGE'     \
+	'AREA:min1#FF0000:"1 minute"'          \
+	'AREA:min5#FFA500:"5 minutes"'         \
+	'LINE1:min5#FF0000:'                   \
+	'AREA:min15#FFD801:"15 minutes"'       \
+	'LINE1:min15#FFA500:'
+	
+  rrdtool graph ${FOLDER}/${P}/cpu.png         \
+	--imgformat PNG                        \
+	--width 640                            \
+	--height 240                           \
+        --upper-limit 10                       \
+        --lower-limit 0                        \
+	--start -${P}                          \
+        --watermark "$(date)"                  \
+	--vertical-label 'Percentage'          \
+	--title 'CPU usage in '${P}            \
+	'DEF:us=cpu.rrd:us:AVERAGE'            \
+	'DEF:sy=cpu.rrd:sy:AVERAGE'            \
+	'DEF:ni=cpu.rrd:ni:AVERAGE'            \
+	'AREA:sy#00FF00:"sy"'                  \
+	'STACK:us#0000FF:"us"'                 \
+	'STACK:ni#FF0000:"ni"'
+ 
+    rrdtool graph ${FOLDER}/${P}/network.png   \
+        --imgformat PNG                        \
+        --width 640                            \
+        --height 240                           \
+        --start -${P}                          \
+        --watermark "$(date)"                  \
+        --vertical-label 'Bytes'               \
+        --upper-limit 10000                    \
+        --lower-limit 0                        \
+        --title 'Network last '${P}            \
+        --lower-limit 0                        \
+        'DEF:in=network.rrd:in:AVERAGE'        \
+        'DEF:out=network.rrd:out:AVERAGE'      \
+        'AREA:out#00FF00:"out"'                \
+        'LINE2:in#0000FF:"in"'
+}
+
+for var in "$@"
+do
+    graph "$var"
+done

createRRDs.sh

@@ -0,0 +1,72 @@
+# create RRDs:
+#  600 samples of 5 minutes  (2 days and 2 hours)
+#  700 samples of 30 minutes (2 days and 2 hours, plus 12.5 days)
+#  775 samples of 2 hours    (above + 50 days)
+#  797 samples of 1 day      (above + 732 days, rounded up to 797)
+
+rrdtool create disk.rrd             \
+            DS:root:GAUGE:600:U:U   \
+            DS:boot:GAUGE:600:U:U   \
+            DS:home:GAUGE:600:U:U   \
+            DS:opt:GAUGE:600:U:U    \
+            DS:tmp:GAUGE:600:U:U    \
+            DS:var:GAUGE:600:U:U    \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797
+
+rrdtool create load.rrd             \
+            DS:min1:GAUGE:600:U:U   \
+            DS:min5:GAUGE:600:U:U   \
+            DS:min15:GAUGE:600:U:U  \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797            
+
+rrdtool create cpu.rrd              \
+            DS:us:GAUGE:600:U:U     \
+            DS:sy:GAUGE:600:U:U     \
+            DS:ni:GAUGE:600:U:U     \
+            DS:id:GAUGE:600:U:U     \
+            DS:wa:GAUGE:600:U:U     \
+            DS:hi:GAUGE:600:U:U     \
+            DS:si:GAUGE:600:U:U     \
+            DS:st:GAUGE:600:U:U     \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797            
+
+rrdtool create mem.rrd              \
+            DS:total:GAUGE:600:U:U   \
+            DS:free:GAUGE:600:U:U   \
+            DS:used:GAUGE:600:U:U   \
+            DS:cached:GAUGE:600:U:U   \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797            
+
+rrdtool create swap.rrd             \
+            DS:total:GAUGE:600:U:U   \
+            DS:free:GAUGE:600:U:U   \
+            DS:used:GAUGE:600:U:U   \
+            DS:avail:GAUGE:600:U:U   \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797            
+
+rrdtool create network.rrd          \
+            DS:in:GAUGE:600:U:U   \
+            DS:out:GAUGE:600:U:U  \
+            RRA:AVERAGE:0.5:1:600   \
+            RRA:AVERAGE:0.5:6:700   \
+            RRA:AVERAGE:0.5:24:775  \
+            RRA:AVERAGE:0.5:288:797 \
+            RRA:MAX:0.5:1:600       \
+            RRA:MAX:0.5:6:700       \
+            RRA:MAX:0.5:24:775      \
+            RRA:MAX:0.5:288:797 

emailEscalationMessage.txt

@@ -0,0 +1,8 @@
+Hmmm.
+Ignace heeft al bijna 4 dagen geen email gelezen.
+Dat is enigzins ongewoon.
+Als je weet dat daar een goede reden voor is (en alles goed is verder), hoef je niets meer met deze mail.
+Als er iets mis is, ga dan naar https://www.suy.nl/cloud/ , login als katja, met haar geboorte-datum DDMMYYYY.
+
+Noot. Zolang als die email niet wordt gelezen, krijg je dit bericht met enige regelmaat. :-(
+

eventService.py

@@ -0,0 +1,86 @@
+#!/usr/bin/python3
+
+import os
+from twisted.internet import protocol, reactor, endpoints
+from datetime import datetime
+from sendAuthMail import SendOff
+
+# HOMEIP='213.46.222.164'
+with open('/root/SAVEIPS') as f:
+    SAVEIPS = f.read().splitlines()
+
+# run this as a daemon
+#   # nohup python3 eventService.py &
+#   start-stop-daemon -SbCv -x /opt/Monitor/monitor/eventService.py
+
+def log_event(text):
+    with open('/tmp/event.log', 'a') as f1:
+        f1.write(text)
+
+
+class Event(protocol.Protocol):
+
+    events = {}
+    illegal_events = 0
+
+    def dataReceived(self, data):
+        '''
+            data contains the event, which a 1-liner like:
+            "type,key,repetition-treshold,time-out,event-text"
+            type: ERROR, BLOCK, WARNING
+            key: error-id, incse of BLOCK this is an IP-address
+            repetition-treshold and time-out: if 5 times in 10 minutes, take action (ERROR: sendmail, BLOCK: block thre ip-address)
+        '''
+        event = str(data, 'utf-8').strip()
+        now = datetime.now()
+        now_label = now.strftime('%d.%H:%M:%S')
+
+        # put ther raw event in the log
+        log_event("{} {}\n".format(now_label, event))
+
+        # saving and dealing with the event comes here....
+        if not (event.count(',') == 4 and (event.startswith('ERROR') or event.startswith('WARN') or event.startswith('BLOCK'))):
+            event="ERROR,ILLEGAL,3,10,Misformed event has been sent to the logger"
+
+        (evtype, evkey, evrep, evtimeout, evtext) = event.split(',')
+        if evtype == 'ERROR' or evtype == 'BLOCK':
+            new_timestamp_list = []
+            if evkey in self.events.keys():
+                timestamp_list = self.events[evkey]
+                for timestamp in timestamp_list:
+                    # if timestamp is younger than now - evtimeout, add it to the new list 
+                    delta_time = now - timestamp
+                    if delta_time.total_seconds() < (60.0 * int(evtimeout)):
+                        new_timestamp_list.append(timestamp)
+            new_timestamp_list.append(now)    
+            self.events[evkey] = new_timestamp_list
+
+            #  print(self.events)
+
+            if len(new_timestamp_list) >= int(evrep):
+                # delete the event in the key-list
+                del self.events[evkey]
+                # now send a message or block an ip
+                if evtype == 'ERROR':
+                    # send an email
+                    log_event("***** Previous line caused an alert *****\n")
+                    s = SendOff(event)
+                    s.ignace()
+                elif evtype == 'BLOCK':
+                    # evkey is the IP address, check if its not my own
+                    if not evkey in SAVEIPS:
+                        log_event("***** Blocking IP {} full access  *****\n".format(evkey))
+                        # block the IP address
+                        os.system("ufw insert 1 deny from {}".format(evkey))
+                        # os.system("ufw insert 1 deny log from {}".format(evkey))
+
+
+
+class EventFactory(protocol.Factory):
+    def buildProtocol(self, addr):
+        return Event()
+
+
+endpoints.serverFromString(reactor, "tcp:1234").listen(EventFactory())
+reactor.run()    # pylint: disable=no-member
+

monitor

@@ -0,0 +1,3 @@
+#!/bin/sh
+df | /opt/monitor/monitorDisks.py
+top -b -n 5 -p0 | tail -9 | /opt/monitor/monitorTop.py

monitorCpu.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+import sys
+import re
+import datetime
+import os
+from rrdtool import update as rrd_update # pylint: disable=no-name-in-module
+
+SUBFOLDER = ''
+#print(datetime.datetime.now().strftime("%V"))
+
+# root@plex:/opt/monitor# mpstat
+# Linux 5.4.0-125-generic (plex) 	01/16/2024 	_x86_64_	(4 CPU)
+# 03:57:00 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
+# 03:57:00 PM  all    0.72    0.01    0.14    0.28    0.00    0.03    0.00    0.00    0.00   98.82
+
+re2 = re.compile(r"\d+\.\d+")
+
+def save(filename, string):
+    fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a")
+    fo.writelines( string + "\n" )
+    fo.close()
+
+for line in sys.stdin:
+    if 'all' in line:
+        cpus = re2.findall(line)
+        # print(";".join(cpus))
+        ret = rrd_update(os.path.join(SUBFOLDER, 'cpu.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}:{6}:{7}'.format( cpus[0], cpus[2], cpus[1], 0,0,0,0,0) )
+        if float(cpus[0]) > 5.0:
+            send_event("ERROR,cpus,3,7,Unexpected high cpu for the last few minutes: {}".format(cpus[0])) 
+

monitorDisks.py

@@ -0,0 +1,42 @@
+#!/usr/bin/python3
+import sys
+import re
+import os
+from rrdtool import update as rrd_update # pylint: disable=no-name-in-module
+from addEvent import send_event
+
+SUBFOLDER = ''
+# print(datetime.datetime.now().strftime("%V"))
+
+re1 = re.compile(r"load average:\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2})")
+re2 = re.compile(r"\d+\.\d+")
+re3 = re.compile(r"\d+")
+
+root = 0
+boot = 0
+home = 0
+opt = 0
+tmp = 0
+var = 0
+
+for line in sys.stdin:
+    if '/' in line:
+        words = line.split()
+        if words[-1] == '/':
+            root = words[-2].replace('%', '')
+        if words[-1] == '/mnt/hdrive':
+            boot = words[-2].replace('%', '')
+        if words[-1] == '/home':
+            home = words[-2].replace('%', '')
+        if words[-1] == '/opt':
+            opt = words[-2].replace('%', '')
+        if words[-1] == '/tmp':
+            tmp = words[-2].replace('%', '')
+        if words[-1] == '/var':
+            var = words[-2].replace('%', '')
+
+# print("{0}:{1}:{2}:{3}:{4}:{5}".format(root, boot, home, opt, tmp, var))
+ret = rrd_update(os.path.join(SUBFOLDER, 'disk.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}'.format(root, boot, home, opt, tmp, var))
+
+if float(root) > 90.0:
+    send_event("ERROR,disk,5,30,Unexpected high disk-usagefor the last few minutes: {}".format(root))

monitorEmails.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+
+import imaplib
+import email
+import datetime
+import os
+from pathlib import Path
+
+##  constants
+UNREADDAYS = 3
+FILE_SMS = '/tmp/emailcheck.sms'
+FILE_WARNING = '/tmp/emailcheck.warning'
+FILE_ESCALATE = '/tmp/emailcheck.ESCALATE'
+
+def write_file(path, text):
+    file1 = open(path, "w")
+    file1.write(text)
+    file1.close()
+
+
+# calculate date 3 days ago
+threedaysago = datetime.datetime.now() - datetime.timedelta(days = UNREADDAYS)
+threedaysago_str = threedaysago.strftime('(SINCE "%d-%b-%Y")')
+# print(threedaysago_str)
+
+# Connect to inbox
+imap_server = imaplib.IMAP4_SSL(host='imap.strato.com')
+imap_server.login('ignace@suy.nl', 'Ghdjh&653gh*')
+imap_server.select()  # Default is `INBOX`
+
+#Search for all UIDs first
+result1, data1 = imap_server.uid('search', threedaysago_str)
+mails_total = len(data1[0].split())
+
+result2, data2 = imap_server.uid('search', threedaysago_str, 'UNSEEN')
+mails_unseen = len(data2[0].split())
+
+print("{} mails {}, {}=={}".format(datetime.datetime.now(), threedaysago_str, mails_total, mails_unseen))
+
+# print("{} {}".format(mails_total, mails_unseen))
+if (mails_total > mails_unseen) or (mails_total == 0):
+    # some mail has been read in the past days, all is well
+    # delete sms file, warning file 
+    if os.path.exists(FILE_SMS):
+        os.remove(FILE_SMS)
+    if os.path.exists(FILE_WARNING):
+        os.remove(FILE_WARNING)
+    if os.path.exists(FILE_ESCALATE):
+        os.remove(FILE_ESCALATE)
+    print("  ->Ok")
+else:
+    # no mails have been read in the past days
+    # check if a earlier warning file has been created
+    if os.path.exists(FILE_WARNING):
+        if (not os.path.exists(FILE_ESCALATE)):
+            # serious shit, escalate
+            write_file(FILE_ESCALATE, "Escalation has taken place")
+            write_file(FILE_SMS, "SUY-CLOUD: You have received a mail from cloudserver@suy.nl - please check!.")
+            print("  ->Escalate")
+    else:
+        # create a warning file
+        write_file(FILE_WARNING, "In another half day, if no email is read, escalation will take place")
+        write_file(FILE_SMS, "SUY-CLOUD: You have not read mail for 3 days, do this immediately pls or escalate.")
+        print("  ->Warning")
+

monitorLoad.py

@@ -0,0 +1,26 @@
+#!/usr/bin/python3
+import sys
+import re
+import datetime
+import os
+from rrdtool import update as rrd_update # pylint: disable=no-name-in-module
+
+SUBFOLDER = ''
+#print(datetime.datetime.now().strftime("%V"))
+# root@plex:/opt/monitor# uptime
+#  16:05:39 up 422 days,  1:05,  1 user,  load average: 0.01, 0.03, 0.01
+
+re2 = re.compile(r"\d+\.\d+")
+
+def save(filename, string):
+    fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a")
+    fo.writelines( string + "\n" )
+    fo.close()
+
+for line in sys.stdin:
+    if 'load average' in line:
+        avgs = re2.findall(line)
+        # print(";".join(avgs))
+        ret = rrd_update(os.path.join(SUBFOLDER, 'load.rrd'), 'N:{0}:{1}:{2}'.format(avgs[0],avgs[1],avgs[2]))
+        if float(avgs[2]) > 5.0:
+            send_event("ERROR,avgs,3,7,Unexpected high average load15 for the last minutes: {}".format(avgs[2]))

monitorMem.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+import sys
+import re
+import datetime
+import os
+from rrdtool import update as rrd_update # pylint: disable=no-name-in-module
+
+SUBFOLDER = ''
+#print(datetime.datetime.now().strftime("%V"))
+
+# root@plex:/opt/monitor# free
+#               total        used        free      shared  buff/cache   available
+# Mem:        5904284      353696      190204        3524     5360384     5238048
+# Swap:       2097148      248064     1849084
+
+
+re3 = re.compile(r"\d+")
+
+def save(filename, string):
+    fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a")
+    fo.writelines( string + "\n" )
+    fo.close()
+
+for line in sys.stdin:
+    if 'Mem:' in line:
+        mems = re3.findall(line)
+        # save('mem_', ";".join(mems))
+        print('N:{0}:{1}:{2}:{3}'.format( mems[0], mems[2], mems[1], int(mems[0])-int(mems[5])))
+        ret = rrd_update(os.path.join(SUBFOLDER, 'mem.rrd'), 'N:{0}:{1}:{2}:{3}'.format( mems[0], mems[2], mems[1], int(mems[0])-int(mems[5])) )
+

monitorNet.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+LOAD=$( ifstat 120 1 | tail -1 )
+
+IN=$( echo ${LOAD} | awk -F' ' '{print $1}' )
+IN=$( bc <<< "scale=0; (${IN} * 1000)/1" )
+
+OUT=$( echo ${LOAD} | awk -F' ' '{print $2}')
+OUT=$( bc <<< "scale=0; (${OUT} * 1000)/1" )
+
+# echo " Net: ${LOAD} ${IN} ${OUT}"
+rrdtool update network.rrd N:${IN}:${OUT}

monitorTop.py

@@ -0,0 +1,30 @@
+#!/usr/bin/python3
+import sys
+import re
+import datetime
+import os
+from rrdtool import update as rrd_update # pylint: disable=no-name-in-module
+from addEvent import send_event
+
+SUBFOLDER = ''
+#print(datetime.datetime.now().strftime("%V"))
+
+# top - 13:35:26 up 2 days, 23:46,  2 users,  load average: 0,00, 0,03, 0,050
+# top - 13:44:48 up 187 days, 38 min,  1 user,  load average: 0.02, 0.08, 0.24
+re1 = re.compile('load average:\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2}),\s?(\d{1,2}\.\d{1,2})')
+re2 = re.compile("\d+\.\d+")
+re3 = re.compile("\d+")
+
+def save(filename, string):
+    fo = open(os.path.join(SUBFOLDER, filename+'_'+datetime.datetime.now().strftime("%V"))+'.log', "a")
+    fo.writelines( string + "\n" )
+    fo.close()
+
+for line in sys.stdin:
+    if 'Cpu(s):' in line:
+        cpus = re2.findall(line)
+        # print(";".join(cpus))
+        ret = rrd_update(os.path.join(SUBFOLDER, 'cpu.rrd'), 'N:{0}:{1}:{2}:{3}:{4}:{5}:{6}:{7}'.format( cpus[0], cpus[1], cpus[2], cpus[3], cpus[4], cpus[5], cpus[6], cpus[7]) )
+        if float(cpus[0]) > 20.0:   # 20 of 100%
+            send_event("ERROR,cpus,3,7,Unexpected high cpu for the last few minutes: {}".format(cpus[0])) 
+

monitorTop.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# rrd_update(os.path.join(SUBFOLDER, 'load.rrd'), 'N:{0}:{1}:{2}'.f
+
+IFS=' ' read -r -a 'HH' <<< "$(cat /proc/loadavg)"
+rrdtool update load.rrd N:${HH[0]}:${HH[1]}:${HH[2]}
+

nohup.out

@@ -0,0 +1,53 @@
+Unhandled Error
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 103, in callWithLogger
+    return callWithContext({"system": lp}, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 86, in callWithContext
+    return context.call({ILogContext: newCtx}, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 122, in callWithContext
+    return self.currentContext().callWithContext(ctx, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 85, in callWithContext
+    return func(*args,**kw)
+--- <exception caught here> ---
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
+    why = selectable.doRead()
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 243, in doRead
+    return self._dataReceived(data)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 249, in _dataReceived
+    rval = self.protocol.dataReceived(data)
+  File "eventService.py", line 46, in dataReceived
+    new_list.append(e)
+builtins.MemoryError: 
+
+Unhandled Error
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 103, in callWithLogger
+    return callWithContext({"system": lp}, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/log.py", line 86, in callWithContext
+    return context.call({ILogContext: newCtx}, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 122, in callWithContext
+    return self.currentContext().callWithContext(ctx, func, *args, **kw)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/python/context.py", line 85, in callWithContext
+    return func(*args,**kw)
+--- <exception caught here> ---
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
+    why = selectable.doRead()
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 243, in doRead
+    return self._dataReceived(data)
+  File "/usr/local/lib/python3.6/dist-packages/twisted/internet/tcp.py", line 249, in _dataReceived
+    rval = self.protocol.dataReceived(data)
+  File "eventService.py", line 48, in dataReceived
+    new_list.append(new_event)
+builtins.MemoryError: 
+
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted
+Rule inserted

processesToCheck.txt

@@ -0,0 +1,5 @@
+/usr/sbin/mysqld
+/usr/sbin/apache2
+/usr/local/bin/gitea
+/usr/lib/plexmediaserver/Plex
+/usr/sbin/sshd

run5

@@ -0,0 +1,26 @@
+#!/bin/bash
+# runs every 2 minutes
+
+echo "$( date ) monitor run5"
+
+# log the busiest processes, for trouble shooting
+echo "----------- $(date) --------------------------------------------" >>/tmp/processes.log
+ps -eo pcpu,pid,user,args | sort -k 1 -r | head -5 >>/tmp/processes.log
+
+# check the most relevant services:
+# ./checkProcesses
+
+# all this is started from cron, met HOME folder -> the root of this project
+cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored
+
+# monitorTop.sh
+df | ./monitorDisks.py
+free | ./monitorMem.py
+# top -b -n 5 -p0 | tail -9 | ./monitorTop.py
+uptime | ./monitorLoad.py
+tail -5 /tmp/mpstat.log | grep all | tail -1 | ./monitorCpu.py
+
+./monitorNet.sh &
+
+./createGraphs 3h
+

run60

@@ -0,0 +1,12 @@
+#!/bin/bash
+# runs every 5 minutes
+
+echo "$( date ) monitor run60"
+# all this is started from cron, met HOME folder -> the root of this project
+cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored
+
+./createGraphs 1d 2w 1y
+
+./checkMeteoService
+
+nethogs -v1 -d30 -c2 -t

send2Mail.py

@@ -0,0 +1,51 @@
+# call this program with:
+#   echo "blah" | python3 sendAuthMail.py
+#   cat <filename.txt> | python3 sendAuthMail.py
+#   python3 sendAuthMail.py 'blah'
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import smtplib
+import sys
+
+
+class SendOff():
+    def __init__(self, message):
+        self.message = message
+       
+    def ignace(self):
+        # Step 2 - Create message object instance
+        msg = MIMEMultipart()
+
+        smtp_ssl_host = 'mail.mijndomein.nl'  # smtp.mail.yahoo.com
+        smtp_ssl_port = 465
+         
+        # Step 4 - Declare SMTP credentials
+        password = "Cloud%67HgDd4569djs5"
+        username = "cloudserver@suy.nu"
+        sender = 'cloudserver@suy.nu'
+        targets = ['ignace@suy.nu']
+
+        # msg = MIMEText('Hi, how are you today?')
+        msg.attach(MIMEText(self.message, 'plain'))
+        msg['Subject'] = "www.suy.nu - system message"
+        msg['From'] = sender
+        msg['To'] = ', '.join(targets)
+
+        server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port)
+        server.login(username, password)
+        server.sendmail(sender, targets, msg.as_string())
+
+        server.quit()
+
+ 
+if __name__ == "__main__":
+    body = ''
+    if len(sys.argv) > 1:
+        body = sys.argv[1]
+    else:
+        for line in sys.stdin: 
+            body += line
+    if len(body) > 0:
+        s = SendOff(body)
+        s.ignace()

sendAuthMail.py

@@ -0,0 +1,51 @@
+# call this program with:
+#   echo "blah" | python3 sendAuthMail.py
+#   cat <filename.txt> | python3 sendAuthMail.py
+#   python3 sendAuthMail.py 'blah'
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import smtplib
+import sys
+
+
+class SendOff():
+    def __init__(self, message):
+        self.message = message
+       
+    def ignace(self):
+        # Step 2 - Create message object instance
+        msg = MIMEMultipart()
+
+        smtp_ssl_host = 'smtp.strato.com'  # smtp.mail.yahoo.com
+        smtp_ssl_port = 465
+         
+        # Step 4 - Declare SMTP credentials
+        password = "Cloud%67HgDd4569djs5"
+        username = "cloudserver@suy.nl"
+        sender = 'cloudserver@suy.nl'
+        targets = ['ignace@suy.nl']
+
+        # msg = MIMEText('Hi, how are you today?')
+        msg.attach(MIMEText(self.message, 'plain'))
+        msg['Subject'] = "www.suy.nl - system message"
+        msg['From'] = sender
+        msg['To'] = ', '.join(targets)
+
+        server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port)
+        server.login(username, password)
+        server.sendmail(sender, targets, msg.as_string())
+
+        server.quit()
+
+ 
+if __name__ == "__main__":
+    body = ''
+    if len(sys.argv) > 1:
+        body = sys.argv[1]
+    else:
+        for line in sys.stdin: 
+            body += line
+    if len(body) > 0:
+        s = SendOff(body)
+        s.ignace()

sendAuthMail.py.2019

@@ -0,0 +1,58 @@
+# call this program with:
+#   echo "blah" | python3 sendAuthMail.py
+#   cat <filename.txt> | python3 sendAuthMail.py
+#   python3 sendAuthMail.py 'blah'
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import smtplib
+import sys
+
+
+class SendOff():
+    def __init__(self, message):
+        self.message = message
+       
+    def ignace(self):
+        # Step 2 - Create message object instance
+        msg = MIMEMultipart()
+         
+        # Step 4 - Declare SMTP credentials
+        password = "Cloud%67HgDd4569djs5"
+        username = "cloudserver@suy.nu"
+        smtphost = "mail.mijndomein.nl"
+
+        # Step 5 - Declare message elements
+        msg['From'] = "cloudserver@suy.nu"
+        msg['To'] = "ignace@suy.nu"
+        msg['Subject'] = "www.suy.nu - system message"
+         
+        # Step 6 - Add the message body to the object instance
+        msg.attach(MIMEText(self.message, 'plain'))
+         
+        # Step 7 - Create the server connection
+        server = smtplib.SMTP(smtphost)
+         
+        # Step 8 - Switch the connection over to TLS encryption
+        server.starttls()
+         
+        # Step 9 - Authenticate with the server
+        server.login(username, password)
+         
+        # Step 10 - Send the message
+        server.sendmail(msg['From'], msg['To'], msg.as_string())
+
+        # Step 11 - Disconnect
+        server.quit()
+
+ 
+if __name__ == "__main__":
+    body = ''
+    if len(sys.argv) > 1:
+        body = sys.argv[1]
+    else:
+        for line in sys.stdin: 
+            body += line
+    if len(body) > 0:
+        s = SendOff(body)
+        s.ignace()

sendEscalationMail.py

@@ -0,0 +1,59 @@
+# call this program with:
+#   echo "blah" | python3 sendAuthMail.py
+#   cat <filename.txt> | python3 sendAuthMail.py
+#   python3 sendAuthMail.py 'blah'
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import smtplib
+import sys
+
+TEXTM = '''
+Hmmm.
+Ignace heeft al bijna 4 dagen geen email gelezen.
+Dat is enigzins ongewoon.
+Als je weet dat daar een goede reden voor is (en alles goed is verder), hoef je niets meer met deze mail.
+Als er iets mis is, ga dan naar https://www.suy.nl/cloud/ , login als katja, met haar geboorte-datum DDMMYYYY.
+
+Noot. Zolang als die email niet wordt gelezen, krijg je dit bericht met enige regelmaat. :-(
+'''
+
+
+class SendOff():
+    def __init__(self, message):
+        self.message = message
+       
+    def ignace(self):
+        # Step 2 - Create message object instance
+        msg = MIMEMultipart()
+
+        smtp_ssl_host = 'smtp.strato.com'  # smtp.mail.yahoo.com
+        smtp_ssl_port = 465
+         
+        # Step 4 - Declare SMTP credentials
+        password = "Cloud%67HgDd4569djs5"
+        username = "cloudserver@suy.nl"
+        sender = 'cloudserver@suy.nl'
+        # targets = ['ignace@suy.nl','janine@demaat.info', 'xander@suy.nl', 'querijn@suy.nl']
+        targets = ['ignace@suy.nl','ignace.suy@gmail.com']
+
+        # msg = MIMEText('Hi, how are you today?')
+        msg.attach(MIMEText(self.message, 'plain'))
+        msg['Subject'] = "www.suy.nl -  Escalation message"
+        msg['From'] = sender
+        msg['To'] = ', '.join(targets)
+
+        server = smtplib.SMTP_SSL(smtp_ssl_host, smtp_ssl_port)
+        server.login(username, password)
+        server.sendmail(sender, targets, msg.as_string())
+
+        server.quit()
+
+ 
+if __name__ == "__main__":
+    body = TEXTM
+    if len(sys.argv) > 1:
+        body = sys.argv[1]
+    if len(body) > 0:
+        s = SendOff(body)
+        s.ignace()

updateRRDs

@@ -0,0 +1,12 @@
+#!/bin/bash
+# runs every 5 minutes
+
+echo "$( date ) updateRRDs monitor"
+# all this is started from cron, met HOME folder -> the root of this project
+cd monitor >/dev/null 2>&1 # this only works if we start it from CRON, else ignored
+
+df | ./monitorDisks.py
+top -b -n 5 -p0 | tail -9 | ./monitorTop.py
+# top -b -n 1 | tee /tmp/top/$(date +"%H%M").log | ./monitorTop.py
+
+# ./monitorNet.sh

workspace.code-workspace

@@ -0,0 +1,8 @@
+{
+	"folders": [
+		{
+			"path": ".."
+		}
+	],
+	"settings": {}
+}