I am trying to log a JSON error message to Google's Cloud Logging platform. Unfortunately, the maximum size message you can log is 8000 bytes and some of the JSON documents I want to log are larger than this. Rather than just chop the JSON string off I would prefer to iteratively remove the largest 'field' (and replace it with a marker such as 'REMOVED') until the document is under the limit.
I've got something working that essentially walks through the deserialized version of the document, field by field, keeping a track of the longest value found so far. This method fails to keep track of where in the object graph this item is located but I am able to perform a simple replace on the JSON string afterwards.
Although this code does the job for what I need, I'm still interested to know if there is an existing algorithm or built-in Python functions that could be applied in this situation.
import json
import sys
class RecursiveApply:
def __init__(self, function_to_apply):
self.function_to_apply = function_to_apply
def process_item(self, current, item):
if isinstance(item, list):
for i in item:
current = self.process_item(current, i)
return current
elif isinstance(item, dict):
for k, v in item.items():
current = self.process_item(current, v)
return current
else:
return self.function_to_apply(current, item)
class GetBigger:
def __init__(self, ignore_value):
self.ignore = ignore_value
def get_bigger(self, biggest, obj):
json_obj = json.dumps(obj)
size = sys.getsizeof(json_obj)
if (biggest is None or size > biggest[0]) and obj != self.ignore:
return size, json_obj
else:
return biggest
class LimitJSON:
def __init__(self, find_biggest, replace_val):
self.find_biggest = find_biggest
self.replace_val = replace_val
def limit_JSON(self, max_len, json_string):
current = None
while len(json_string) > max_len:
biggest_item = self.find_biggest(current, json.loads(json_string))
if biggest_item:
json_string = json_string.replace(biggest_item[1], '"' + REPLACE_VAL + '"')
else:
break
return json_string
REPLACE_VAL = '*REMOVED*'
gb = GetBigger(REPLACE_VAL)
ra = RecursiveApply(gb.get_bigger)
lj = LimitJSON(ra.process_item, REPLACE_VAL)
original_data = ['long array item', {'item1': 'dict item', 'item 2': 'longer dict item'}, 'little item', 123123123123123123123123123]
json_string = json.dumps(original_data)
processed_json = lj.limit_JSON(100, json_string)
print(processed_json)
# RESULT ->
# ["long array item", {"item 2": "*REMOVED*", "item1": "dict item"}, "little item", "*REMOVED*"]