Change in pysim[master]: construct: Recursive normalization of construct parse result

laforge gerrit-no-reply at lists.osmocom.org
Sat Jun 5 09:35:08 UTC 2021


laforge has submitted this change. ( https://gerrit.osmocom.org/c/pysim/+/24452 )

Change subject: construct: Recursive normalization of construct parse result
......................................................................

construct: Recursive normalization of construct parse result

If we want to use construct parse results to generate JSON serializable
dicts, we need to

* apply the filter_dict() operation recursively, and
* simplify the construct Container and ListContainer classes to
  a simple dict and/or list.

We introduce a pySim.construct.parse_construct() helper which is
subsequently used from all pySim.filesystem caller sites.

Change-Id: I319414eb69808ef65895293832bb30519f45949d
---
M pySim/construct.py
M pySim/filesystem.py
2 files changed, 36 insertions(+), 7 deletions(-)

Approvals:
  Jenkins Builder: Verified
  laforge: Looks good to me, approved



diff --git a/pySim/construct.py b/pySim/construct.py
index d0101e3..2a3efd3 100644
--- a/pySim/construct.py
+++ b/pySim/construct.py
@@ -1,3 +1,4 @@
+import typing
 from construct import *
 from pySim.utils import b2h, h2b, swap_nibbles
 import gsm0338
@@ -84,6 +85,34 @@
             res[key] = value
     return res
 
+from construct.lib.containers import Container, ListContainer
+from construct.core import EnumIntegerString
+
+def normalize_construct(c):
+    """Convert a construct specific type to a related base type, mostly useful
+    so we can serialize it."""
+    # we need to include the filter_dict as we otherwise get elements like this
+    # in the dict: '_io': <_io.BytesIO object at 0x7fdb64e05860> which we cannot json-serialize
+    c = filter_dict(c)
+    if isinstance(c, Container) or isinstance(c, dict):
+        r = {k : normalize_construct(v) for (k, v) in c.items()}
+    elif isinstance(c, ListContainer):
+        r = [normalize_construct(x) for x in c]
+    elif isinstance(c, list):
+        r = [normalize_construct(x) for x in c]
+    elif isinstance(c, EnumIntegerString):
+        r = str(c)
+    else:
+        r = c
+    return r
+
+def parse_construct(c, raw_bin_data:bytes, length:typing.Optional[int]=None, exclude_prefix:str='_'):
+    """Helper function to wrap around normalize_construct() and filter_dict()."""
+    if not length:
+        length = len(raw_bin_data)
+    parsed = c.parse(raw_bin_data, total_len=length)
+    return normalize_construct(parsed)
+
 # here we collect some shared / common definitions of data types
 LV = Prefixed(Int8ub, HexAdapter(GreedyBytes))
 
diff --git a/pySim/filesystem.py b/pySim/filesystem.py
index 8c45ba1..9a2f4e1 100644
--- a/pySim/filesystem.py
+++ b/pySim/filesystem.py
@@ -35,7 +35,7 @@
 from typing import cast, Optional, Iterable, List, Any, Dict, Tuple
 
 from pySim.utils import sw_match, h2b, b2h, i2h, is_hex, auto_int, bertlv_parse_one, Hexstr
-from pySim.construct import filter_dict
+from pySim.construct import filter_dict, parse_construct
 from pySim.exceptions import *
 from pySim.jsonpath import js_path_find, js_path_modify
 
@@ -490,7 +490,7 @@
         if callable(method):
             return method(b2h(raw_bin_data))
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_bin_data.hex()}
 
     def decode_hex(self, raw_hex_data:str) -> dict:
@@ -513,7 +513,7 @@
         if callable(method):
             return method(raw_bin_data)
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_bin_data.hex()}
 
     def encode_bin(self, abstract_data:dict) -> bytearray:
@@ -712,7 +712,7 @@
         if callable(method):
             return method(raw_bin_data)
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_bin_data.hex()}
 
     def decode_record_bin(self, raw_bin_data:bytearray) -> dict:
@@ -735,7 +735,7 @@
         if callable(method):
             return method(raw_hex_data)
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_hex_data}
 
     def encode_record_hex(self, abstract_data:dict) -> str:
@@ -834,7 +834,7 @@
         if callable(method):
             return method(raw_bin_data)
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_hex_data}
 
     def decode_record_bin(self, raw_bin_data:bytearray) -> dict:
@@ -857,7 +857,7 @@
         if callable(method):
             return method(raw_hex_data)
         if self._construct:
-            return filter_dict(self._construct.parse(raw_bin_data, total_len=len(raw_bin_data)))
+            return parse_construct(self._construct, raw_bin_data)
         return {'raw': raw_hex_data}
 
     def encode_record_hex(self, abstract_data:dict) -> str:

-- 
To view, visit https://gerrit.osmocom.org/c/pysim/+/24452
To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings

Gerrit-Project: pysim
Gerrit-Branch: master
Gerrit-Change-Id: I319414eb69808ef65895293832bb30519f45949d
Gerrit-Change-Number: 24452
Gerrit-PatchSet: 5
Gerrit-Owner: laforge <laforge at osmocom.org>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: dexter <pmaier at sysmocom.de>
Gerrit-Reviewer: fixeria <vyanitskiy at sysmocom.de>
Gerrit-Reviewer: laforge <laforge at osmocom.org>
Gerrit-Reviewer: pespin <pespin at sysmocom.de>
Gerrit-MessageType: merged
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20210605/cc77e8c6/attachment.htm>


More information about the gerrit-log mailing list