added async logger

logcomponents/zmqlogserver.py

+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr  3 10:24:19 2018
+
+@author: Magnus Rentsch Ersdal (magnus.ersdal@uib.no)
+
+Call zmq_ctx_new() once at the start of a process, and zmq_ctx_destroy() once at the end.
+
+ for 2e7 (20 M) items: 
+ operation completed in 1 minutes 17.879222 seconds for 100_000 lines
+ operation completed in 1 minutes 4.791010 seconds for 1_000_000 lines
+ operation completed in 1 minutes 3.716582 seconds for 20_000_000 lines
+ approx 300_000 per second. 
+ 
+ https://pyzmq.readthedocs.io/en/latest/api/zmq.html#basic-classes
+"""
+#import time
+
+import zmq
+import os
+from filever.filever import fileversion
+from stopwatch import Stopwatch
+
+LOG_BUFFER_SIZE = 10_000_000 #1_000_000 # lines
+logstrs = []
+filename = fileversion("svf_log",None,".txt")
+#
+#try:
+#    os.remove(filename)
+#except OSError:
+#    pass
+
+
+
+class Server():
+    def __init__(self):
+        self.i = 0
+        self.ctx = zmq.Context()
+        self.receiver = self.ctx.socket(zmq.PULL)   #context.socket(zmq.PULL)
+        self.receiver.bind("tcp://*:5557")
+        self.receiver.setsockopt(zmq.RCVTIMEO, 1000)
+    def appendlog(self, logstrs: list, newdata : str, line_end = ""):
+        logstrs.append(newdata + line_end)
+        return logstrs
+    def get_log_str(self, handle = None):
+        self.i += 1
+        try:
+            s = self.receiver.recv_string()
+        except zmq.ZMQError:
+            print(".", end ="")
+            s = None
+        return s #"faker logstring {}".format(self.i)
+    def write_file(self, logstrs: list, filename: str):
+        status = 0
+        f = open(filename,"a+")
+        assert f
+        f.writelines(logstrs)
+        f.close()
+        return status
+    def close(self):
+        status = 0
+        return status
+    def __enter__(self):
+        return self
+    def __exit__(self, type, value, traceback):
+        s = self.close()
+
+""" usage:"""
+with Server() as mys:
+    try:
+        print("starting server")
+    #    mys = Server()
+        i = 0
+        sw1 = Stopwatch()
+        while True:
+    #    for i in range(int(2e7)):
+            newd = mys.get_log_str()
+            if newd:
+                logstrs = mys.appendlog(logstrs, newd, "\n")
+            else:
+                pass
+
+            if len(logstrs) >= LOG_BUFFER_SIZE:
+                E = mys.write_file(logstrs, filename)
+                logstrs = []
+                i += 1
+                print(" {} x ".format(i), end="")
+
+
+    except KeyboardInterrupt:
+        print("----- C-C detected! -----")
+    finally:
+        if len(logstrs) > 0:
+            print("writing final log entries")
+            E = mys.write_file(logstrs, filename)
+            logstrs = []
+
+    sw1.stop()
+"""some stuff"""

logcomponents/zmqsender.py

+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr  3 13:57:14 2018
+
+@author: Magnus Rentsch Ersdal (magnus.ersdal@uib.no)
+sending to server_with_buffer.py
+operation completed in 0 minutes 14.143425 seconds with no windows antivirus and tcp with the string int(i)
+16 seconds with a longer string.  
+1e6/14 = 71 000
+"""
+
+import zmq
+from zmqsender_config import Errlvl
+from zmqsender_config import severity_report, ecompare
+
+class Sender():
+    def __init__(self):
+        self.i = 0
+        self.ctx = zmq.Context()
+        self.sck = self.ctx.socket(zmq.PUSH)   #context.socket(zmq.PULL)
+        self.sck.connect("tcp://localhost:5557")
+    def log(self, logstring: str, errlvl = Errlvl.LOW):
+        if ecompare(severity_report, errlvl):
+            self.sck.send_string(logstring)
+    def close(self):
+        status = 0
+        return status
+    def __enter__(self):
+        return self
+    def __exit__(self, type, value, traceback):
+        s = self.close()
+""" usage:         
+with Sender() as snd:
+    try:
+        print("starting sender")
+        time.sleep(1)
+        snd.log("start of log at {}".format(time.asctime()))
+        sw1 = Stopwatch()
+        for i in range(int(1e6)):
+            snd.log("a longer logstring" + str(i))
+    except KeyboardInterrupt:
+        print("----- C-C detected! -----")
+    finally:
+        snd.log("end of log at {}".format(time.asctime()))
+        sw1.stop()
+"""
\ No newline at end of file

zmqsender_config.py

+# -*- coding: utf-8 -*-
+"""
+Created on Wed Apr  4 12:13:39 2018
+
+@author: Magnus Rentsch Ersdal (magnus.ersdal@uib.no)
+"""
+
+
+
+from enum import Enum
+
+class Errlvl(Enum):
+    LOW = 0
+    MEDIUM = 1
+    HIGH = 2
+    CRITICAL = 3
+    INFO = 99
+    DEBUG0 = 0
+    DEBUG1 = 1
+    DEBUG2 = 2
+    DEBUG3 = 3
+    
+severity_report = Errlvl.MEDIUM    
+    
+def ecompare(report,actual):
+    return report.value <= actual.value
\ No newline at end of file