Commit 31a8159d authored by Paul Graydon's avatar Paul Graydon

wendelin_telecom_base: Update data chunk size limits in KPI calculation process

- Reduce maximum data chunk size for new data
- Add a maximum data chunk size for the previous data to bound computing time
parent 10d9f4d0
import json import json
import numpy as np import numpy as np
MAX_CHUNK_SIZE = 10000000 # bytes MAX_PREV_CHUNK_SIZE = 1024 * 1024 # bytes
MAX_NEW_CHUNK_SIZE = 1024 * 1024 # bytes
HISTORY_LINE_COUNT = 64 HISTORY_LINE_COUNT = 64
T_PERIOD = 30 # seconds T_PERIOD = 30 # seconds
QCI_COUNT = 256 QCI_COUNT = 256
...@@ -9,8 +10,8 @@ QCI_COUNT = 256 ...@@ -9,8 +10,8 @@ QCI_COUNT = 256
in_data_stream = in_stream['Data Stream'] in_data_stream = in_stream['Data Stream']
progress_indicator = in_stream['Progress Indicator'] progress_indicator = in_stream['Progress Indicator']
offset_index = progress_indicator.getIntOffsetIndex() offset_index = progress_indicator.getIntOffsetIndex()
start = 0 start = max(0, offset_index - MAX_PREV_CHUNK_SIZE)
end = min(in_data_stream.getSize(), offset_index + MAX_CHUNK_SIZE) end = min(in_data_stream.getSize(), offset_index + MAX_NEW_CHUNK_SIZE)
# No new data to process # No new data to process
if offset_index >= end: if offset_index >= end:
...@@ -22,7 +23,11 @@ new_log_data = ''.join(in_data_stream.readChunkList(offset_index, end)) ...@@ -22,7 +23,11 @@ new_log_data = ''.join(in_data_stream.readChunkList(offset_index, end))
previous_log_data_line_list = previous_log_data.splitlines() previous_log_data_line_list = previous_log_data.splitlines()
new_log_data_line_list = new_log_data.splitlines() new_log_data_line_list = new_log_data.splitlines()
# Last new log line may not be valid JSON due to MAX_CHUNK_SIZE: # First previous log line may not be valid JSON due to MAX_PREV_CHUNK_SIZE:
# Simply discard it
previous_log_data_line_list = previous_log_data_line_list[1:]
# Last new log line may not be valid JSON due to MAX_NEW_CHUNK_SIZE:
# In that case, leave it to the next KPI calculation # In that case, leave it to the next KPI calculation
last_new_log_data_line = new_log_data_line_list[-1] last_new_log_data_line = new_log_data_line_list[-1]
try: try:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment