From a20a734a97cde7046c75fa39144406ca8746c1d7 Mon Sep 17 00:00:00 2001 From: nugenjs Date: Thu, 11 Jun 2026 17:34:46 -0500 Subject: [PATCH 1/5] added logger module --- KeyMaster.ini | 10 +++++- KeyMaster.ini.cabinet | 12 +++++-- KeyMaster.template.ini | 20 +++++++---- drivers/Log/KibanaLog.py | 72 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 drivers/Log/KibanaLog.py diff --git a/KeyMaster.ini b/KeyMaster.ini index ffb6496..eaea126 100644 --- a/KeyMaster.ini +++ b/KeyMaster.ini @@ -5,7 +5,8 @@ auth = ADApiAuth rfid = StdInRFID -log = FileLog +log = KibanaLog +# log = FileLog relay = Relay relay_interface = PiFaceInterface currentSense = BinaryCurrentSense @@ -37,5 +38,12 @@ interface_position_red = 8 interface_position_green = 7 interface_position_blue = 6 +[KibanaLog] +url = http://your-elasticsearch-host:9200 +index = rfid-keymaster +filename = KeyMaster.log +log_level = debug +# device_name = large-machine-01 + [FileLog] filename = KeyMaster.log diff --git a/KeyMaster.ini.cabinet b/KeyMaster.ini.cabinet index 4d8cc01..b0fef36 100644 --- a/KeyMaster.ini.cabinet +++ b/KeyMaster.ini.cabinet @@ -7,7 +7,8 @@ auth = ADCommonAPIAuth rfid = StdInRFID -log = FileLog +log = KibanaLog +# log = FileLog controller_io_interface = PiGpioInterface @@ -33,7 +34,14 @@ interface_position_red = GPIO22, non-inverting interface_position_green = GPIO23, non-inverting interface_position_blue = GPIO24, non-inverting +[KibanaLog] +url = http://your-elasticsearch-host:9200 +index = rfid-keymaster +filename = KeyMaster.log +log_level = debug +# device_name = cabinet-01 + [FileLog] filename = KeyMaster.log -loglevel = debug +log_level = debug diff --git a/KeyMaster.template.ini b/KeyMaster.template.ini index f945d32..d478fc6 100644 --- a/KeyMaster.template.ini +++ b/KeyMaster.template.ini @@ -9,7 +9,8 @@ controller = LargeMachineController auth = ADCommonAPIAuth #rfid = KeyboardRFID rfid = StdInRFID -log = FileLog +log = KibanaLog +# log = FileLog relay = Relay relay_interface = PiGpioInterface # relay_interface = PiFaceInterface @@ -95,13 +96,20 @@ interface_position_blue = GPIO24, non-inverting # in blinks per second, 50% duty cycle. # blink_rate = -[FileLog] +[KibanaLog] +url = http://your-elasticsearch-host:9200 +index = rfid-keymaster filename = KeyMaster.log -# format = -# date_format = -# log_level= debug|info|error +log_level = debug +# device_name = my-machine-01 + +# [FileLog] +# filename = KeyMaster.log +# format = +# date_format = +# log_level = debug|info|error # [SysLogger] # not yet implemented -# log_level' = error|info|debug +# log_level = error|info|debug #[Buzzer] # not yet implemented diff --git a/drivers/Log/KibanaLog.py b/drivers/Log/KibanaLog.py new file mode 100644 index 0000000..b22962d --- /dev/null +++ b/drivers/Log/KibanaLog.py @@ -0,0 +1,72 @@ +from drivers.Log.Log import Log +import logging +import requests +import datetime +import socket + +class KibanaLog(Log): + def __init__(self, config, loader): + super().__init__(config, loader) + + if 'url' not in config: + raise Exception("KibanaLog requires 'url' (Elasticsearch endpoint)") + + self.es_url = config['url'].rstrip('/') + self.index = config.get('index', 'rfid-keymaster') + self.device = config.get('device_name', socket.gethostname()) + + level_str = config.get('log_level', 'debug').lower() + loglevel = {'debug': logging.DEBUG, + 'info': logging.INFO, + 'error': logging.ERROR}.get(level_str, logging.DEBUG) + + fmt = config.get('format', '%(asctime)-15s %(message)s') + datefmt = config.get('date_format', '%Y-%m-%d %H:%M:%S') + + if 'filename' in config: + logging.basicConfig(filename=config['filename'], format=fmt, + level=loglevel, datefmt=datefmt) + else: + logging.basicConfig(format=fmt, level=loglevel, datefmt=datefmt) + + def _send(self, level, message, extra=None): + doc = { + '@timestamp': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', + 'level': level, + 'message': message, + 'device': self.device, + 'service': 'rfid-keymaster', + } + if extra: + doc.update(extra) + try: + requests.post(f'{self.es_url}/{self.index}/_doc', + json=doc, + headers={'Content-Type': 'application/json'}, + timeout=3) + except Exception: + pass + + def auth(self, user): + rfid_id = user.get('id') if isinstance(user, dict) else str(user) + authorized = user.get('authorized') if isinstance(user, dict) else None + msg = f"Auth: {user}" + logging.info(msg) + self._send('auth', msg, {'rfid_id': rfid_id, 'authorized': authorized}) + + def engaged(self, status): + msg = f"Engaged: {status}" + logging.info(msg) + self._send('info', msg, {'engaged': status}) + + def debug(self, message): + logging.debug(message) + self._send('debug', message) + + def info(self, message): + logging.info(message) + self._send('info', message) + + def error(self, message): + logging.error(message) + self._send('error', message) From ee6e920cf6acd908e3ab860e89481ca9ef24cd3b Mon Sep 17 00:00:00 2001 From: nugenjs Date: Sun, 5 Jul 2026 23:15:13 -0500 Subject: [PATCH 2/5] Added queueing and attach stdlib logger with handler --- KeyMaster.ini | 7 +- KeyMaster.ini.cabinet | 7 +- KeyMaster.template.ini | 5 +- .../CabinetStapleLatchController.py | 2 +- drivers/Controller/LargeMachineController.py | 2 +- drivers/Log/KibanaLog.py | 97 +++++++++++++------ 6 files changed, 82 insertions(+), 38 deletions(-) diff --git a/KeyMaster.ini b/KeyMaster.ini index eaea126..b57859e 100644 --- a/KeyMaster.ini +++ b/KeyMaster.ini @@ -39,11 +39,12 @@ interface_position_green = 7 interface_position_blue = 6 [KibanaLog] -url = http://your-elasticsearch-host:9200 -index = rfid-keymaster +url = https://192.168.0.150:9200 +index = not-john-index filename = KeyMaster.log log_level = debug -# device_name = large-machine-01 +device_name = +api_key = [FileLog] filename = KeyMaster.log diff --git a/KeyMaster.ini.cabinet b/KeyMaster.ini.cabinet index b0fef36..c77ce37 100644 --- a/KeyMaster.ini.cabinet +++ b/KeyMaster.ini.cabinet @@ -35,11 +35,12 @@ interface_position_green = GPIO23, non-inverting interface_position_blue = GPIO24, non-inverting [KibanaLog] -url = http://your-elasticsearch-host:9200 -index = rfid-keymaster +url = https://192.168.0.150:9200 +index = not-john-index filename = KeyMaster.log log_level = debug -# device_name = cabinet-01 +device_name = +api_key = [FileLog] filename = KeyMaster.log diff --git a/KeyMaster.template.ini b/KeyMaster.template.ini index d478fc6..525313a 100644 --- a/KeyMaster.template.ini +++ b/KeyMaster.template.ini @@ -98,10 +98,11 @@ interface_position_blue = GPIO24, non-inverting [KibanaLog] url = http://your-elasticsearch-host:9200 -index = rfid-keymaster +index = elastic-index filename = KeyMaster.log log_level = debug -# device_name = my-machine-01 +device_name = +api_key = # [FileLog] # filename = KeyMaster.log diff --git a/drivers/Controller/CabinetStapleLatchController.py b/drivers/Controller/CabinetStapleLatchController.py index 8e70b3c..b1c109a 100644 --- a/drivers/Controller/CabinetStapleLatchController.py +++ b/drivers/Controller/CabinetStapleLatchController.py @@ -56,7 +56,7 @@ def setup(self): if 'latch_control_interface' in self.config: self.latch_control_interface = (self.config['latch_control_interface']) else: - log.debug("latch_control_interface not specified, aborting") + logging.debug("latch_control_interface not specified, aborting") # Defaults diff --git a/drivers/Controller/LargeMachineController.py b/drivers/Controller/LargeMachineController.py index 74b4cff..fb78f3f 100644 --- a/drivers/Controller/LargeMachineController.py +++ b/drivers/Controller/LargeMachineController.py @@ -168,7 +168,7 @@ def run(self): # relay off self.relay.off() - logging.notice("Initial Startup Current Detected, shutting off") + logging.info("Initial Startup Current Detected, shutting off") # red LED blinking self.light(self.LIGHT_ERROR) diff --git a/drivers/Log/KibanaLog.py b/drivers/Log/KibanaLog.py index b22962d..7696e07 100644 --- a/drivers/Log/KibanaLog.py +++ b/drivers/Log/KibanaLog.py @@ -1,8 +1,17 @@ from drivers.Log.Log import Log import logging +import queue +import threading +import traceback import requests import datetime import socket +import urllib3 + +# KibanaLog is used to stand in for FileLog +# It will write logs to a file (KeyMaster.log) +# It will add itself as a logging handler to python stdlib logger +# It will also send those logs to ELK endpoint asynchronously class KibanaLog(Log): def __init__(self, config, loader): @@ -13,7 +22,12 @@ def __init__(self, config, loader): self.es_url = config['url'].rstrip('/') self.index = config.get('index', 'rfid-keymaster') - self.device = config.get('device_name', socket.gethostname()) + self.host = config.get('device_name', socket.gethostname()) + self.api_key = config.get('api_key', None) + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # urllib3 log only warnings and errors + logging.getLogger('urllib3').setLevel(logging.WARNING) level_str = config.get('log_level', 'debug').lower() loglevel = {'debug': logging.DEBUG, @@ -24,49 +38,76 @@ def __init__(self, config, loader): datefmt = config.get('date_format', '%Y-%m-%d %H:%M:%S') if 'filename' in config: - logging.basicConfig(filename=config['filename'], format=fmt, - level=loglevel, datefmt=datefmt) + logging.basicConfig(filename=config['filename'], format=fmt, level=loglevel, datefmt=datefmt) else: logging.basicConfig(format=fmt, level=loglevel, datefmt=datefmt) - def _send(self, level, message, extra=None): + # Add messages to queue to send to Kibana asynchronously + self.send_queue = queue.Queue(maxsize=1000) + threading.Thread(target=self._send_forever, daemon=True).start() + + # Attach kibana as a logging handler to python stdlib logger + logging.getLogger().addHandler(_KibanaHandler(self)) + + def _queue(self, level, message): + try: + self.send_queue.put_nowait((level, message)) + except queue.Full: + # if queue is full, drop message + pass + + def _send_forever(self): + while True: + level, message = self.send_queue.get() + self._send(level, message) + + def _send(self, level, message): doc = { - '@timestamp': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', + '@timestamp': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', 'level': level, - 'message': message, - 'device': self.device, + 'message': str(message), + 'host': self.host, 'service': 'rfid-keymaster', } - if extra: - doc.update(extra) + headers = {'Content-Type': 'application/json'} + if self.api_key: + headers['Authorization'] = f'ApiKey {self.api_key}' try: - requests.post(f'{self.es_url}/{self.index}/_doc', - json=doc, - headers={'Content-Type': 'application/json'}, - timeout=3) - except Exception: - pass + resp = requests.post(f'{self.es_url}/{self.index}/_doc', + json=doc, + headers=headers, + timeout=3, + verify=False) + print(f"[KibanaLog] {resp.status_code} {resp.text}") + except Exception as e: + print(f"[KibanaLog] ERROR: {e}") def auth(self, user): - rfid_id = user.get('id') if isinstance(user, dict) else str(user) - authorized = user.get('authorized') if isinstance(user, dict) else None - msg = f"Auth: {user}" - logging.info(msg) - self._send('auth', msg, {'rfid_id': rfid_id, 'authorized': authorized}) + logging.info("Auth: " + str(user)) def engaged(self, status): - msg = f"Engaged: {status}" - logging.info(msg) - self._send('info', msg, {'engaged': status}) + logging.info("Engaged: " + str(status)) def debug(self, message): logging.debug(message) - self._send('debug', message) def info(self, message): logging.info(message) - self._send('info', message) - def error(self, message): - logging.error(message) - self._send('error', message) + def error(self, message, exc_info=False): + logging.error(message, exc_info=exc_info) + + +class _KibanaHandler(logging.Handler): + def __init__(self, kibana): + super().__init__() + self.kibana = kibana + + def emit(self, record): + # Filter out logs that are not from the root logger to prevent infinite loops + if record.name != 'root': + return + message = record.getMessage() + if record.exc_info: + message += "\n" + "".join(traceback.format_exception(*record.exc_info)) + self.kibana._queue(record.levelname.lower(), message) From b845fd8206003546c38d8384017b50124b6aa07d Mon Sep 17 00:00:00 2001 From: Rich Osman Date: Thu, 18 Jun 2026 13:52:27 -0500 Subject: [PATCH 3/5] Update LargeMachineController.py to allow fractional risetime --- drivers/Controller/LargeMachineController.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/Controller/LargeMachineController.py b/drivers/Controller/LargeMachineController.py index fb78f3f..3f0e387 100644 --- a/drivers/Controller/LargeMachineController.py +++ b/drivers/Controller/LargeMachineController.py @@ -39,7 +39,7 @@ def setup(self): self.timer = None if 'rise_time' in self.config: - self.rise_time = int(self.config['rise_time']) + self.rise_time = float(self.config['rise_time']) if 'timeout_time' in self.config: self.timeout_time = int(self.config['timeout_time']) From 7c1042055cfb1db46bd90c19ac4060e89bb9ef39 Mon Sep 17 00:00:00 2001 From: nugenjs Date: Mon, 6 Jul 2026 12:37:54 -0500 Subject: [PATCH 4/5] clean up for pr --- KeyMaster.ini | 2 +- KeyMaster.ini.cabinet | 4 ++-- KeyMaster.template.ini | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/KeyMaster.ini b/KeyMaster.ini index b57859e..7d91e99 100644 --- a/KeyMaster.ini +++ b/KeyMaster.ini @@ -5,8 +5,8 @@ auth = ADApiAuth rfid = StdInRFID -log = KibanaLog # log = FileLog +log = KibanaLog relay = Relay relay_interface = PiFaceInterface currentSense = BinaryCurrentSense diff --git a/KeyMaster.ini.cabinet b/KeyMaster.ini.cabinet index c77ce37..1008b7b 100644 --- a/KeyMaster.ini.cabinet +++ b/KeyMaster.ini.cabinet @@ -7,8 +7,8 @@ auth = ADCommonAPIAuth rfid = StdInRFID -log = KibanaLog # log = FileLog +log = KibanaLog controller_io_interface = PiGpioInterface @@ -44,5 +44,5 @@ api_key = [FileLog] filename = KeyMaster.log -log_level = debug +loglevel = debug diff --git a/KeyMaster.template.ini b/KeyMaster.template.ini index 525313a..29ca157 100644 --- a/KeyMaster.template.ini +++ b/KeyMaster.template.ini @@ -9,8 +9,8 @@ controller = LargeMachineController auth = ADCommonAPIAuth #rfid = KeyboardRFID rfid = StdInRFID -log = KibanaLog # log = FileLog +log = KibanaLog relay = Relay relay_interface = PiGpioInterface # relay_interface = PiFaceInterface @@ -104,13 +104,13 @@ log_level = debug device_name = api_key = -# [FileLog] -# filename = KeyMaster.log +[FileLog] +filename = KeyMaster.log # format = # date_format = # log_level = debug|info|error # [SysLogger] # not yet implemented -# log_level = error|info|debug +# log_level' = error|info|debug #[Buzzer] # not yet implemented From faa15392499ef18099f29a51b4aabc055cdcbfb6 Mon Sep 17 00:00:00 2001 From: nugenjs Date: Mon, 6 Jul 2026 12:40:36 -0500 Subject: [PATCH 5/5] clean up for pr --- KeyMaster.template.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/KeyMaster.template.ini b/KeyMaster.template.ini index 29ca157..619552e 100644 --- a/KeyMaster.template.ini +++ b/KeyMaster.template.ini @@ -106,9 +106,9 @@ api_key = [FileLog] filename = KeyMaster.log -# format = -# date_format = -# log_level = debug|info|error +# format = +# date_format = +# log_level= debug|info|error # [SysLogger] # not yet implemented # log_level' = error|info|debug