laforge has uploaded this change for review. ( https://gerrit.osmocom.org/c/pysim/+/37965?usp=email )

Change subject: Migrate over to using pyosmocom ......................................................................

Migrate over to using pyosmocom

We're creating a 'pyosmocom' pypi module which contains a number of core Osmocom libraries / interfaces that are not specific to SIM card stuff contained here.

The main modules moved in this initial step are pySim.tlv, pySim.utils and pySim.construct. utils is split, not all of the contents is unrelated to SIM Cards. The other two are moved completely.

Change-Id: I4b63e45bcb0c9ba2424dacf85e0222aee735f411 --- M README.md M contrib/csv-encrypt-columns.py M contrib/es9p_client.py M contrib/saip-tool.py M contrib/unber.py M osmo-smdpp.py M pySim-prog.py M pySim-read.py M pySim-shell.py M pySim-trace.py M pySim/apdu/__init__.py M pySim/apdu/global_platform.py M pySim/apdu/ts_102_221.py M pySim/apdu/ts_102_222.py M pySim/apdu/ts_31_102.py M pySim/ara_m.py M pySim/cat.py M pySim/cdma_ruim.py M pySim/commands.py D pySim/construct.py M pySim/esim/bsp.py M pySim/esim/es8p.py M pySim/esim/rsp.py M pySim/esim/saip/__init__.py M pySim/esim/saip/personalization.py M pySim/euicc.py M pySim/filesystem.py M pySim/global_platform/__init__.py M pySim/global_platform/http.py M pySim/global_platform/scp.py M pySim/global_platform/uicc.py M pySim/gsm_r.py M pySim/gsmtap.py M pySim/iso7816_4.py M pySim/ota.py M pySim/runtime.py M pySim/secure_channel.py M pySim/sms.py M pySim/sysmocom_sja2.py D pySim/tlv.py M pySim/transport/__init__.py M pySim/transport/calypso.py M pySim/transport/modem_atcmd.py M pySim/transport/pcsc.py M pySim/transport/serial.py M pySim/ts_102_221.py M pySim/ts_102_222.py M pySim/ts_102_310.py M pySim/ts_31_102.py M pySim/ts_31_102_telecom.py M pySim/ts_31_103.py M pySim/ts_31_103_shared.py M pySim/ts_31_104.py M pySim/ts_51_011.py M pySim/utils.py M requirements.txt M setup.py M tests/unittests/test_apdu.py D tests/unittests/test_construct.py M tests/unittests/test_esim.py M tests/unittests/test_esim_bsp.py M tests/unittests/test_esim_saip.py M tests/unittests/test_files.py M tests/unittests/test_globalplatform.py M tests/unittests/test_ota.py M tests/unittests/test_sms.py D tests/unittests/test_tlv.py M tests/unittests/test_tlvs.py M tests/unittests/test_utils.py 69 files changed, 151 insertions(+), 2,053 deletions(-)

git pull ssh://gerrit.osmocom.org:29418/pysim refs/changes/65/37965/1

diff --git a/README.md b/README.md index 75f8c44..b169df1 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ - cmd2 >= 1.5.0 - colorlog - construct >= 2.9.51 - - gsm0338 + - osmocom - jsonpath-ng - packaging - pycryptodomex diff --git a/contrib/csv-encrypt-columns.py b/contrib/csv-encrypt-columns.py index 2b2bbf3..49340a2 100755 --- a/contrib/csv-encrypt-columns.py +++ b/contrib/csv-encrypt-columns.py @@ -22,8 +22,8 @@ import csv import argparse from Cryptodome.Cipher import AES +from osmocom.utils import h2b, b2h, Hexstr

-from pySim.utils import h2b, b2h, Hexstr from pySim.card_key_provider import CardKeyProviderCsv

def dict_keys_to_upper(d: dict) -> dict: diff --git a/contrib/es9p_client.py b/contrib/es9p_client.py index 0434eb8..d4c91f3 100755 --- a/contrib/es9p_client.py +++ b/contrib/es9p_client.py @@ -28,10 +28,11 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from cryptography.hazmat.primitives.asymmetric import ec

+from osmocom.utils import h2b, b2h, swap_nibbles, is_hexstr +from osmocom.tlv import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv + import pySim.esim.rsp as rsp from pySim.esim import es9p, PMO -from pySim.utils import h2b, b2h, swap_nibbles, is_hexstr -from pySim.utils import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv from pySim.esim.x509_cert import CertAndPrivkey from pySim.esim.es8p import BoundProfilePackage

diff --git a/contrib/saip-tool.py b/contrib/saip-tool.py index faf6041..763e18d 100755 --- a/contrib/saip-tool.py +++ b/contrib/saip-tool.py @@ -22,11 +22,11 @@ import zipfile from pathlib import Path as PlPath from typing import List +from osmocom.utils import h2b, b2h, swap_nibbles

from pySim.esim.saip import * from pySim.esim.saip.validation import CheckBasicStructure from pySim import javacard -from pySim.utils import h2b, b2h, swap_nibbles from pySim.pprint import HexBytesPrettyPrinter

pp = HexBytesPrettyPrinter(indent=4,width=500) diff --git a/contrib/unber.py b/contrib/unber.py index 6b488b7..50831cb 100755 --- a/contrib/unber.py +++ b/contrib/unber.py @@ -6,7 +6,8 @@ import sys import argparse

-from pySim.utils import bertlv_parse_one, bertlv_encode_tag, b2h, h2b +from osmocom.utils import b2h, h2b +from osmocom.tlv import bertlv_parse_one, bertlv_encode_tag

def process_one_level(content: bytes, indent: int): remainder = content diff --git a/osmo-smdpp.py b/osmo-smdpp.py index 3dbe9b1..351787e 100755 --- a/osmo-smdpp.py +++ b/osmo-smdpp.py @@ -32,7 +32,7 @@ from twisted.web.iweb import IRequest import asn1tools

-from pySim.utils import h2b, b2h, swap_nibbles +from osmocom.utils import h2b, b2h, swap_nibbles

import pySim.esim.rsp as rsp from pySim.esim import saip, PMO diff --git a/pySim-prog.py b/pySim-prog.py index db9c327..6d1deb1 100755 --- a/pySim-prog.py +++ b/pySim-prog.py @@ -33,11 +33,12 @@ import traceback import json import csv +from osmocom.utils import h2b, swap_nibbles, rpad

from pySim.commands import SimCardCommands from pySim.transport import init_reader, argparse_add_reader_args from pySim.legacy.cards import _cards_classes, card_detect -from pySim.utils import h2b, swap_nibbles, rpad, derive_milenage_opc, calculate_luhn, dec_iccid +from pySim.utils import derive_milenage_opc, calculate_luhn, dec_iccid from pySim.ts_51_011 import EF_AD from pySim.legacy.ts_51_011 import EF from pySim.card_handler import * diff --git a/pySim-read.py b/pySim-read.py index d34ddc3..aae5ad4 100755 --- a/pySim-read.py +++ b/pySim-read.py @@ -29,6 +29,8 @@ import re import sys

+from osmocom.utils import h2b, h2s, swap_nibbles, rpad + from pySim.ts_51_011 import EF_SST_map, EF_AD from pySim.legacy.ts_51_011 import EF, DF from pySim.ts_31_102 import EF_UST_map @@ -40,7 +42,7 @@ from pySim.transport import init_reader, argparse_add_reader_args from pySim.exceptions import SwMatchError from pySim.legacy.cards import card_detect, SimCard, UsimCard, IsimCard -from pySim.utils import h2b, h2s, swap_nibbles, rpad, dec_imsi, dec_iccid, dec_msisdn +from pySim.utils import dec_imsi, dec_iccid, dec_msisdn from pySim.legacy.utils import format_xplmn_w_act, dec_st

option_parser = argparse.ArgumentParser(description='Legacy tool for reading some parts of a SIM card', diff --git a/pySim-shell.py b/pySim-shell.py index 43e9dd2..d3f9d91 100755 --- a/pySim-shell.py +++ b/pySim-shell.py @@ -47,11 +47,13 @@

from pprint import pprint as pp

+from osmocom.utils import h2b, b2h, i2h, swap_nibbles, rpad, JsonEncoder, is_hexstr, is_decimal +from osmocom.utils import is_hexstr_or_decimal, Hexstr +from osmocom.tlv import bertlv_parse_one + from pySim.exceptions import * from pySim.transport import init_reader, ApduTracer, argparse_add_reader_args, ProactiveHandler -from pySim.utils import h2b, b2h, i2h, swap_nibbles, rpad, JsonEncoder, bertlv_parse_one, sw_match -from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, Hexstr, dec_iccid -from pySim.utils import is_hexstr_or_decimal, is_hexstr, is_decimal +from pySim.utils import sanitize_pin_adm, tabulate_str_list, boxed_heading_str, dec_iccid, sw_match from pySim.card_handler import CardHandler, CardHandlerAuto

from pySim.filesystem import CardMF, CardEF, CardDF, CardADF diff --git a/pySim-trace.py b/pySim-trace.py index 74dc8b9..3652b3a 100755 --- a/pySim-trace.py +++ b/pySim-trace.py @@ -8,7 +8,8 @@ from pySim.apdu import * from pySim.runtime import RuntimeState

-from pySim.utils import JsonEncoder +from osmocom.utils import JsonEncoder + from pySim.cards import UiccCardBase from pySim.commands import SimCardCommands from pySim.profile import CardProfile diff --git a/pySim/apdu/__init__.py b/pySim/apdu/__init__.py index 469b2fa..4f08113 100644 --- a/pySim/apdu/__init__.py +++ b/pySim/apdu/__init__.py @@ -29,12 +29,11 @@ import typing from typing import List, Dict, Optional from termcolor import colored - from construct import Byte, GreedyBytes from construct import Optional as COptional +from osmocom.construct import * +from osmocom.utils import *

-from pySim.construct import * -from pySim.utils import * from pySim.runtime import RuntimeLchan, RuntimeState, lchan_nr_from_cla from pySim.filesystem import CardADF, CardFile, TransparentEF, LinFixedEF

diff --git a/pySim/apdu/global_platform.py b/pySim/apdu/global_platform.py index 894d864..4bb575a 100644 --- a/pySim/apdu/global_platform.py +++ b/pySim/apdu/global_platform.py @@ -18,9 +18,9 @@ """

from construct import FlagsEnum, Struct -from pySim.tlv import flatten_dict_lists +from osmocom.tlv import flatten_dict_lists +from osmocom.construct import * from pySim.apdu import ApduCommand, ApduCommandSet -from pySim.construct import * from pySim.global_platform import InstallParameters

class GpDelete(ApduCommand, n='DELETE', ins=0xE4, cla=['8X', 'CX', 'EX']): diff --git a/pySim/apdu/ts_102_221.py b/pySim/apdu/ts_102_221.py index 491c0d3..cd14782 100644 --- a/pySim/apdu/ts_102_221.py +++ b/pySim/apdu/ts_102_221.py @@ -22,11 +22,12 @@

from construct import GreedyRange, Struct

-from pySim.construct import * +from osmocom.utils import i2h +from osmocom.construct import * + from pySim.filesystem import * from pySim.runtime import RuntimeLchan from pySim.apdu import ApduCommand, ApduCommandSet -from pySim.utils import i2h from pySim import cat

logger = logging.getLogger(__name__) diff --git a/pySim/apdu/ts_102_222.py b/pySim/apdu/ts_102_222.py index fde16a8..adf4d4d 100644 --- a/pySim/apdu/ts_102_222.py +++ b/pySim/apdu/ts_102_222.py @@ -20,8 +20,8 @@ import logging

from construct import Struct +from osmocom.construct import *

-from pySim.construct import * from pySim.apdu import ApduCommand, ApduCommandSet from pySim.ts_102_221 import FcpTemplate

diff --git a/pySim/apdu/ts_31_102.py b/pySim/apdu/ts_31_102.py index 58f91a2..fea5038 100644 --- a/pySim/apdu/ts_31_102.py +++ b/pySim/apdu/ts_31_102.py @@ -11,9 +11,9 @@

from construct import BitStruct, Enum, BitsInteger, Int8ub, Bytes, this, Struct, If, Switch, Const from construct import Optional as COptional +from osmocom.construct import *

from pySim.filesystem import * -from pySim.construct import * from pySim.ts_31_102 import SUCI_TlvDataObject from pySim.apdu import ApduCommand, ApduCommandSet

diff --git a/pySim/ara_m.py b/pySim/ara_m.py index e6f0520..d25d0e8 100644 --- a/pySim/ara_m.py +++ b/pySim/ara_m.py @@ -28,10 +28,10 @@

from construct import GreedyBytes, GreedyString, Struct, Enum, Int8ub, Int16ub from construct import Optional as COptional -from pySim.construct import * +from osmocom.construct import * +from osmocom.tlv import * +from osmocom.utils import Hexstr from pySim.filesystem import * -from pySim.tlv import * -from pySim.utils import Hexstr import pySim.global_platform

# various BER-TLV encoded Data Objects (DOs) diff --git a/pySim/cat.py b/pySim/cat.py index 73f0a76..fae5fbf 100644 --- a/pySim/cat.py +++ b/pySim/cat.py @@ -23,9 +23,10 @@ from construct import Int8ub, Int16ub, Byte, Bytes, BitsInteger from construct import Struct, Enum, BitStruct, this from construct import GreedyBytes, Switch, GreedyRange, FlagsEnum -from pySim.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection -from pySim.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi, GsmString -from pySim.utils import b2h, dec_xplmn_w_act +from osmocom.tlv import TLV_IE, COMPR_TLV_IE, BER_TLV_IE, TLV_IE_Collection +from osmocom.construct import PlmnAdapter, BcdAdapter, HexAdapter, GsmStringAdapter, TonNpi, GsmString +from osmocom.utils import b2h +from pySim.utils import dec_xplmn_w_act

# Tag values as per TS 101 220 Table 7.23

diff --git a/pySim/cdma_ruim.py b/pySim/cdma_ruim.py index d16aefc..96ff1a4 100644 --- a/pySim/cdma_ruim.py +++ b/pySim/cdma_ruim.py @@ -20,15 +20,15 @@ import enum

from construct import Bytewise, BitStruct, BitsInteger, Struct, FlagsEnum +from osmocom.utils import * +from osmocom.construct import *

-from pySim.utils import * from pySim.filesystem import * from pySim.profile import match_ruim from pySim.profile import CardProfile, CardProfileAddon from pySim.ts_51_011 import CardProfileSIM from pySim.ts_51_011 import DF_TELECOM, DF_GSM from pySim.ts_51_011 import EF_ServiceTable -from pySim.construct import *

# Mapping between CDMA Service Number and its description diff --git a/pySim/commands.py b/pySim/commands.py index 9e231f1..656f71a 100644 --- a/pySim/commands.py +++ b/pySim/commands.py @@ -23,12 +23,13 @@

from typing import List, Tuple import typing # construct also has a Union, so we do typing.Union below - from construct import Construct, Struct, Const, Select from construct import Optional as COptional -from pySim.construct import LV, filter_dict -from pySim.utils import rpad, lpad, b2h, h2b, sw_match, bertlv_encode_len, h2i, i2h, str_sanitize, expand_hex, SwMatchstr -from pySim.utils import Hexstr, SwHexstr, ResTuple +from osmocom.construct import LV, filter_dict +from osmocom.utils import rpad, lpad, b2h, h2b, h2i, i2h, str_sanitize, Hexstr +from osmocom.tlv import bertlv_encode_len + +from pySim.utils import sw_match, expand_hex, SwHexstr, ResTuple, SwMatchstr from pySim.exceptions import SwMatchError from pySim.transport import LinkBase

diff --git a/pySim/construct.py b/pySim/construct.py deleted file mode 100644 index 881f1f2..0000000 --- a/pySim/construct.py +++ /dev/null @@ -1,587 +0,0 @@ -"""Utility code related to the integration of the 'construct' declarative parser.""" - -import typing -import codecs -import ipaddress - -import gsm0338 - -from construct.lib.containers import Container, ListContainer -from construct.core import EnumIntegerString -from construct import Adapter, Prefixed, Int8ub, GreedyBytes, Default, Flag, Byte, Construct, Enum -from construct import BitsInteger, BitStruct, Bytes, StreamError, stream_read_entire, stream_write -from construct import SizeofError, IntegerError, swapbytes -from construct.core import evaluate -from construct.lib import integertypes - -from pySim.utils import b2h, h2b, swap_nibbles - -# (C) 2021-2022 by Harald Welte laforge@osmocom.org -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - - -class HexAdapter(Adapter): - """convert a bytes() type to a string of hex nibbles.""" - - def _decode(self, obj, context, path): - return b2h(obj) - - def _encode(self, obj, context, path): - return h2b(obj) - -class Utf8Adapter(Adapter): - """convert a bytes() type that contains utf8 encoded text to human readable text.""" - - def _decode(self, obj, context, path): - # In case the string contains only 0xff bytes we interpret it as an empty string - if obj == b'\xff' * len(obj): - return "" - return codecs.decode(obj, "utf-8") - - def _encode(self, obj, context, path): - return codecs.encode(obj, "utf-8") - -class GsmOrUcs2Adapter(Adapter): - """Try to encode into a GSM 03.38 string; if that fails, fall back to UCS-2 as described - in TS 102 221 Annex A.""" - def _decode(self, obj, context, path): - # In case the string contains only 0xff bytes we interpret it as an empty string - if obj == b'\xff' * len(obj): - return "" - # one of the magic bytes of TS 102 221 Annex A - if obj[0] in [0x80, 0x81, 0x82]: - ad = Ucs2Adapter(GreedyBytes) - else: - ad = GsmString(GreedyBytes) - return ad._decode(obj, context, path) - - def _encode(self, obj, context, path): - # first try GSM 03.38; then fall back to TS 102 221 Annex A UCS-2 - try: - ad = GsmString(GreedyBytes) - return ad._encode(obj, context, path) - except: - ad = Ucs2Adapter(GreedyBytes) - return ad._encode(obj, context, path) - -class Ucs2Adapter(Adapter): - """convert a bytes() type that contains UCS2 encoded characters encoded as defined in TS 102 221 - Annex A to normal python string representation (and back).""" - def _decode(self, obj, context, path): - # In case the string contains only 0xff bytes we interpret it as an empty string - if obj == b'\xff' * len(obj): - return "" - if obj[0] == 0x80: - # TS 102 221 Annex A Variant 1 - return codecs.decode(obj[1:], 'utf_16_be') - elif obj[0] == 0x81: - # TS 102 221 Annex A Variant 2 - out = "" - # second byte contains a value indicating the number of characters - num_of_chars = obj[1] - # the third byte contains an 8 bit number which defines bits 15 to 8 of a 16 bit base - # pointer, where bit 16 is set to zero, and bits 7 to 1 are also set to zero. These - # sixteen bits constitute a base pointer to a "half-page" in the UCS2 code space - base_ptr = obj[2] << 7 - for ch in obj[3:3+num_of_chars]: - # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a - # GSM Default Alphabet character, whereas if bit 8 of the byte is set to one, then - # the remaining seven bits are an offset value added to the 16 bit base pointer - # defined earlier, and the resultant 16 bit value is a UCS2 code point - if ch & 0x80: - codepoint = (ch & 0x7f) + base_ptr - out += codecs.decode(codepoint.to_bytes(2, byteorder='big'), 'utf_16_be') - else: - out += codecs.decode(bytes([ch]), 'gsm03.38') - return out - elif obj[0] == 0x82: - # TS 102 221 Annex A Variant 3 - out = "" - # second byte contains a value indicating the number of characters - num_of_chars = obj[1] - # third and fourth bytes contain a 16 bit number which defines the complete 16 bit base - # pointer to a half-page in the UCS2 code space, for use with some or all of the - # remaining bytes in the string - base_ptr = obj[2] << 8 | obj[3] - for ch in obj[4:4+num_of_chars]: - # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a - # GSM Default Alphabet character, whereas if bit 8 of the byte is set to one, the - # remaining seven bits are an offset value added to the base pointer defined in - # bytes three and four, and the resultant 16 bit value is a UCS2 code point, else: # - # GSM default alphabet - if ch & 0x80: - codepoint = (ch & 0x7f) + base_ptr - out += codecs.decode(codepoint.to_bytes(2, byteorder='big'), 'utf_16_be') - else: - out += codecs.decode(bytes([ch]), 'gsm03.38') - return out - else: - raise ValueError('First byte of TS 102 221 UCS-2 must be 0x80, 0x81 or 0x82') - - def _encode(self, obj, context, path): - def encodable_in_gsm338(instr: str) -> bool: - """Determine if given input string is encode-ale in gsm03.38.""" - try: - # TODO: figure out if/how we can constrain to default alphabet. The gsm0338 - # library seems to include the spanish lock/shift table - codecs.encode(instr, 'gsm03.38') - except ValueError: - return False - return True - - def codepoints_not_in_gsm338(instr: str) -> typing.List[int]: - """Return an integer list of UCS2 codepoints for all characters of 'inster' - which are not representable in the GSM 03.38 default alphabet.""" - codepoint_list = [] - for c in instr: - if encodable_in_gsm338(c): - continue - c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big') - codepoint_list.append(c_codepoint) - return codepoint_list - - def diff_between_min_and_max_of_list(inlst: typing.List) -> int: - return max(inlst) - min(inlst) - - def encodable_in_variant2(instr: str) -> bool: - codepoint_prefix = None - for c in instr: - if encodable_in_gsm338(c): - continue - c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big') - if c_codepoint >= 0x8000: - return False - c_prefix = c_codepoint >> 7 - if codepoint_prefix is None: - codepoint_prefix = c_prefix - else: - if c_prefix != codepoint_prefix: - return False - return True - - def encodable_in_variant3(instr: str) -> bool: - codepoint_list = codepoints_not_in_gsm338(instr) - # compute delta between max and min; check if it's encodable in 7 bits - if diff_between_min_and_max_of_list(codepoint_list) >= 0x80: - return False - return True - - def _encode_variant1(instr: str) -> bytes: - """Encode according to TS 102 221 Annex A Variant 1""" - return b'\x80' + codecs.encode(instr, 'utf_16_be') - - def _encode_variant2(instr: str) -> bytes: - """Encode according to TS 102 221 Annex A Variant 2""" - codepoint_prefix = None - # second byte contains a value indicating the number of characters - hdr = b'\x81' + len(instr).to_bytes(1, byteorder='big') - chars = b'' - for c in instr: - try: - enc = codecs.encode(c, 'gsm03.38') - except ValueError: - c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big') - c_prefix = c_codepoint >> 7 - if codepoint_prefix is None: - codepoint_prefix = c_prefix - assert codepoint_prefix == c_prefix - enc = (0x80 + (c_codepoint & 0x7f)).to_bytes(1, byteorder='big') - chars += enc - if codepoint_prefix is None: - codepoint_prefix = 0 - return hdr + codepoint_prefix.to_bytes(1, byteorder='big') + chars - - def _encode_variant3(instr: str) -> bytes: - """Encode according to TS 102 221 Annex A Variant 3""" - # second byte contains a value indicating the number of characters - hdr = b'\x82' + len(instr).to_bytes(1, byteorder='big') - chars = b'' - codepoint_list = codepoints_not_in_gsm338(instr) - codepoint_base = min(codepoint_list) - for c in instr: - try: - # if bit 8 of the byte is set to zero, the remaining 7 bits of the byte contain a GSM - # Default # Alphabet character - enc = codecs.encode(c, 'gsm03.38') - except ValueError: - # if bit 8 of the byte is set to one, the remaining seven bits are an offset - # value added to the base pointer defined in bytes three and four, and the - # resultant 16 bit value is a UCS2 code point - c_codepoint = int.from_bytes(codecs.encode(c, 'utf_16_be'), byteorder='big') - c_codepoint_delta = c_codepoint - codepoint_base - assert c_codepoint_delta < 0x80 - enc = (0x80 + c_codepoint_delta).to_bytes(1, byteorder='big') - chars += enc - # third and fourth bytes contain a 16 bit number which defines the complete 16 bit base - # pointer to a half-page in the UCS2 code space - return hdr + codepoint_base.to_bytes(2, byteorder='big') + chars - - if encodable_in_variant2(obj): - return _encode_variant2(obj) - elif encodable_in_variant3(obj): - return _encode_variant3(obj) - else: - return _encode_variant1(obj) - -class BcdAdapter(Adapter): - """convert a bytes() type to a string of BCD nibbles.""" - - def _decode(self, obj, context, path): - return swap_nibbles(b2h(obj)) - - def _encode(self, obj, context, path): - return h2b(swap_nibbles(obj)) - -class PlmnAdapter(BcdAdapter): - """convert a bytes(3) type to BCD string like 262-02 or 262-002.""" - def _decode(self, obj, context, path): - bcd = super()._decode(obj, context, path) - if bcd[3] == 'f': - return '-'.join([bcd[:3], bcd[4:]]) - else: - return '-'.join([bcd[:3], bcd[3:]]) - - def _encode(self, obj, context, path): - l = obj.split('-') - if len(l[1]) == 2: - bcd = l[0] + 'f' + l[1] - else: - bcd = l[0] + l[1] - return super()._encode(bcd, context, path) - -class InvertAdapter(Adapter): - """inverse logic (false->true, true->false).""" - @staticmethod - def _invert_bool_in_obj(obj): - for k,v in obj.items(): - # skip all private entries - if k.startswith('_'): - continue - if v is False: - obj[k] = True - elif v is True: - obj[k] = False - return obj - - def _decode(self, obj, context, path): - return self._invert_bool_in_obj(obj) - - def _encode(self, obj, context, path): - return self._invert_bool_in_obj(obj) - -class Rpad(Adapter): - """ - Encoder appends padding bytes (b'\xff') or characters up to target size. - Decoder removes trailing padding bytes/characters. - - Parameters: - subcon: Subconstruct as defined by construct library - pattern: set padding pattern (default: b'\xff') - num_per_byte: number of 'elements' per byte. E.g. for hex nibbles: 2 - """ - - def __init__(self, subcon, pattern=b'\xff', num_per_byte=1): - super().__init__(subcon) - self.pattern = pattern - self.num_per_byte = num_per_byte - - def _decode(self, obj, context, path): - return obj.rstrip(self.pattern) - - def _encode(self, obj, context, path): - target_size = self.sizeof() * self.num_per_byte - if len(obj) > target_size: - raise SizeofError("Input ({}) exceeds target size ({})".format( - len(obj), target_size)) - return obj + self.pattern * (target_size - len(obj)) - -class MultiplyAdapter(Adapter): - """ - Decoder multiplies by multiplicator - Encoder divides by multiplicator - - Parameters: - subcon: Subconstruct as defined by construct library - multiplier: Multiplier to apply to raw encoded value - """ - - def __init__(self, subcon, multiplicator): - super().__init__(subcon) - self.multiplicator = multiplicator - - def _decode(self, obj, context, path): - return obj * 8 - - def _encode(self, obj, context, path): - return obj // 8 - - -class GsmStringAdapter(Adapter): - """Convert GSM 03.38 encoded bytes to a string.""" - - def __init__(self, subcon, codec='gsm03.38', err='strict'): - super().__init__(subcon) - self.codec = codec - self.err = err - - def _decode(self, obj, context, path): - return obj.decode(self.codec) - - def _encode(self, obj, context, path): - return obj.encode(self.codec, self.err) - -class Ipv4Adapter(Adapter): - """ - Encoder converts from 4 bytes to string representation (A.B.C.D). - Decoder converts from string representation (A.B.C.D) to four bytes. - """ - def _decode(self, obj, context, path): - ia = ipaddress.IPv4Address(obj) - return ia.compressed - - def _encode(self, obj, context, path): - ia = ipaddress.IPv4Address(obj) - return ia.packed - -class Ipv6Adapter(Adapter): - """ - Encoder converts from 16 bytes to string representation. - Decoder converts from string representation to 16 bytes. - """ - def _decode(self, obj, context, path): - ia = ipaddress.IPv6Address(obj) - return ia.compressed - - def _encode(self, obj, context, path): - ia = ipaddress.IPv6Address(obj) - return ia.packed - -class StripTrailerAdapter(Adapter): - """ - Encoder removes all trailing bytes matching the default_value - Decoder pads input data up to total_length with default_value - - This is used in constellations like "FlagsEnum(StripTrailerAdapter(GreedyBytes, 3), ..." - where you have a bit-mask that may have 1, 2 or 3 bytes, depending on whether or not any - of the LSBs are actually set. - """ - def __init__(self, subcon, total_length:int, default_value=b'\x00', min_len=1): - super().__init__(subcon) - assert len(default_value) == 1 - self.total_length = total_length - self.default_value = default_value - self.min_len = min_len - - def _decode(self, obj, context, path): - assert isinstance(obj, bytes) - # pad with suppressed/missing bytes - if len(obj) < self.total_length: - obj += self.default_value * (self.total_length - len(obj)) - return int.from_bytes(obj, 'big') - - def _encode(self, obj, context, path): - assert isinstance(obj, int) - obj = obj.to_bytes(self.total_length, 'big') - # remove trailing bytes if they are zero - while len(obj) > self.min_len and obj[-1] == self.default_value[0]: - obj = obj[:-1] - return obj - - -def filter_dict(d, exclude_prefix='_'): - """filter the input dict to ensure no keys starting with 'exclude_prefix' remain.""" - if not isinstance(d, dict): - return d - res = {} - for (key, value) in d.items(): - if key.startswith(exclude_prefix): - continue - if isinstance(value, dict): - res[key] = filter_dict(value) - else: - res[key] = value - return res - - -def normalize_construct(c, exclude_prefix: str = '_'): - """Convert a construct specific type to a related base type, mostly useful - so we can serialize it.""" - # we need to include the filter_dict as we otherwise get elements like this - # in the dict: '_io': <_io.BytesIO object at 0x7fdb64e05860> which we cannot json-serialize - c = filter_dict(c, exclude_prefix) - if isinstance(c, (Container, dict)): - r = {k: normalize_construct(v) for (k, v) in c.items()} - elif isinstance(c, ListContainer): - r = [normalize_construct(x) for x in c] - elif isinstance(c, list): - r = [normalize_construct(x) for x in c] - elif isinstance(c, EnumIntegerString): - r = str(c) - else: - r = c - return r - - -def parse_construct(c, raw_bin_data: bytes, length: typing.Optional[int] = None, exclude_prefix: str = '_', context: dict = {}): - """Helper function to wrap around normalize_construct() and filter_dict().""" - if not length: - length = len(raw_bin_data) - try: - parsed = c.parse(raw_bin_data, total_len=length, **context) - except StreamError as e: - # if the input is all-ff, this means the content is undefined. Let's avoid passing StreamError - # exceptions in those situations (which might occur if a length field 0xff is 255 but then there's - # actually less bytes in the remainder of the file. - if all(v == 0xff for v in raw_bin_data): - return None - else: - raise e - return normalize_construct(parsed, exclude_prefix) - -def build_construct(c, decoded_data, context: dict = {}): - """Helper function to handle total_len.""" - return c.build(decoded_data, total_len=None, **context) - -# here we collect some shared / common definitions of data types -LV = Prefixed(Int8ub, HexAdapter(GreedyBytes)) - -# Default value for Reserved for Future Use (RFU) bits/bytes -# See TS 31.101 Sec. "3.4 Coding Conventions" -__RFU_VALUE = 0 - -# Field that packs Reserved for Future Use (RFU) bit -FlagRFU = Default(Flag, __RFU_VALUE) - -# Field that packs Reserved for Future Use (RFU) byte -ByteRFU = Default(Byte, __RFU_VALUE) - -# Field that packs all remaining Reserved for Future Use (RFU) bytes -GreedyBytesRFU = Default(GreedyBytes, b'') - - -def BitsRFU(n=1): - ''' - Field that packs Reserved for Future Use (RFU) bit(s) - as defined in TS 31.101 Sec. "3.4 Coding Conventions" - - Use this for (currently) unused/reserved bits whose contents - should be initialized automatically but should not be cleared - in the future or when restoring read data (unlike padding). - - Parameters: - n (Integer): Number of bits (default: 1) - ''' - return Default(BitsInteger(n), __RFU_VALUE) - - -def BytesRFU(n=1): - ''' - Field that packs Reserved for Future Use (RFU) byte(s) - as defined in TS 31.101 Sec. "3.4 Coding Conventions" - - Use this for (currently) unused/reserved bytes whose contents - should be initialized automatically but should not be cleared - in the future or when restoring read data (unlike padding). - - Parameters: - n (Integer): Number of bytes (default: 1) - ''' - return Default(Bytes(n), __RFU_VALUE) - - -def GsmString(n): - ''' - GSM 03.38 encoded byte string of fixed length n. - Encoder appends padding bytes (b'\xff') to maintain - length. Decoder removes those trailing bytes. - - Exceptions are raised for invalid characters - and length excess. - - Parameters: - n (Integer): Fixed length of the encoded byte string - ''' - return GsmStringAdapter(Rpad(Bytes(n), pattern=b'\xff'), codec='gsm03.38') - -def GsmOrUcs2String(n): - ''' - GSM 03.38 or UCS-2 (TS 102 221 Annex A) encoded byte string of fixed length n. - Encoder appends padding bytes (b'\xff') to maintain - length. Decoder removes those trailing bytes. - - Exceptions are raised for invalid characters - and length excess. - - Parameters: - n (Integer): Fixed length of the encoded byte string - ''' - return GsmOrUcs2Adapter(Rpad(Bytes(n), pattern=b'\xff')) - -class GreedyInteger(Construct): - """A variable-length integer implementation, think of combining GrredyBytes with BytesInteger.""" - def __init__(self, signed=False, swapped=False, minlen=0): - super().__init__() - self.signed = signed - self.swapped = swapped - self.minlen = minlen - - def _parse(self, stream, context, path): - data = stream_read_entire(stream, path) - if evaluate(self.swapped, context): - data = swapbytes(data) - try: - return int.from_bytes(data, byteorder='big', signed=self.signed) - except ValueError as e: - raise IntegerError(str(e), path=path) - - def __bytes_required(self, i, minlen=0): - if self.signed: - raise NotImplementedError("FIXME: Implement support for encoding signed integer") - - # compute how many bytes we need - nbytes = 1 - while True: - i = i >> 8 - if i == 0: - break - else: - nbytes = nbytes + 1 - - # round up to the minimum number - # of bytes we anticipate - nbytes = max(nbytes, minlen) - - return nbytes - - def _build(self, obj, stream, context, path): - if not isinstance(obj, integertypes): - raise IntegerError(f"value {obj} is not an integer", path=path) - length = self.__bytes_required(obj, self.minlen) - try: - data = obj.to_bytes(length, byteorder='big', signed=self.signed) - except ValueError as e: - raise IntegerError(str(e), path=path) from e - if evaluate(self.swapped, context): - data = swapbytes(data) - stream_write(stream, data, length, path) - return obj - -# merged definitions of 24.008 + 23.040 -TypeOfNumber = Enum(BitsInteger(3), unknown=0, international=1, national=2, network_specific=3, - short_code=4, alphanumeric=5, abbreviated=6, reserved_for_extension=7) -NumberingPlan = Enum(BitsInteger(4), unknown=0, isdn_e164=1, data_x121=3, telex_f69=4, - sc_specific_5=5, sc_specific_6=6, national=8, private=9, - ermes=10, reserved_cts=11, reserved_for_extension=15) -TonNpi = BitStruct('ext'/Flag, 'type_of_number'/TypeOfNumber, 'numbering_plan_id'/NumberingPlan) diff --git a/pySim/esim/bsp.py b/pySim/esim/bsp.py index 81fe092..24634a7 100644 --- a/pySim/esim/bsp.py +++ b/pySim/esim/bsp.py @@ -35,7 +35,8 @@ from Cryptodome.Cipher import AES from Cryptodome.Hash import CMAC

-from pySim.utils import bertlv_encode_len, bertlv_parse_one, b2h +from osmocom.utils import b2h +from osmocom.tlv import bertlv_encode_len, bertlv_parse_one

# don't log by default logger = logging.getLogger(__name__) diff --git a/pySim/esim/es8p.py b/pySim/esim/es8p.py index ba824d6..dd653f8 100644 --- a/pySim/esim/es8p.py +++ b/pySim/esim/es8p.py @@ -17,11 +17,10 @@ # along with this program. If not, see http://www.gnu.org/licenses/.

from typing import Dict, List, Optional - from cryptography.hazmat.primitives.asymmetric import ec - -from pySim.utils import b2h, h2b, bertlv_encode_tag, bertlv_encode_len, bertlv_parse_one_rawtag -from pySim.utils import bertlv_return_one_rawtlv +from osmocom.utils import b2h, h2b +from osmocom.tlv import bertlv_encode_tag, bertlv_encode_len, bertlv_parse_one_rawtag +from osmocom.tlv import bertlv_return_one_rawtlv

import pySim.esim.rsp as rsp from pySim.esim.bsp import BspInstance diff --git a/pySim/esim/rsp.py b/pySim/esim/rsp.py index c2a163b..4b189e1 100644 --- a/pySim/esim/rsp.py +++ b/pySim/esim/rsp.py @@ -23,8 +23,9 @@ from cryptography.hazmat.primitives.asymmetric import ec from cryptography.hazmat.primitives.serialization import Encoding from cryptography import x509 +from osmocom.utils import b2h +from osmocom.tlv import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv

-from pySim.utils import bertlv_parse_one_rawtag, bertlv_return_one_rawtlv, b2h from pySim.esim import compile_asn1_subdir

asn1 = compile_asn1_subdir('rsp') diff --git a/pySim/esim/saip/__init__.py b/pySim/esim/saip/__init__.py index dc67970..47e67e8 100644 --- a/pySim/esim/saip/__init__.py +++ b/pySim/esim/saip/__init__.py @@ -20,19 +20,19 @@ import io from typing import Tuple, List, Optional, Dict, Union from collections import OrderedDict - import asn1tools +from osmocom.utils import b2h, h2b, Hexstr +from osmocom.tlv import BER_TLV_IE, bertlv_parse_tag, bertlv_parse_len +from osmocom.construct import build_construct, parse_construct, GreedyInteger

-from pySim.utils import bertlv_parse_tag, bertlv_parse_len, b2h, h2b, dec_imsi, Hexstr +from pySim.utils import dec_imsi from pySim.ts_102_221 import FileDescriptor from pySim.filesystem import CardADF, Path from pySim.ts_31_102 import ADF_USIM from pySim.ts_31_103 import ADF_ISIM -from pySim.construct import build_construct, parse_construct, GreedyInteger from pySim.esim import compile_asn1_subdir from pySim.esim.saip import templates from pySim.esim.saip import oid -from pySim.tlv import BER_TLV_IE from pySim.global_platform import KeyType, KeyUsageQualifier from pySim.global_platform.uicc import UiccSdInstallParams

diff --git a/pySim/esim/saip/personalization.py b/pySim/esim/saip/personalization.py index dc3435b..f9451d1 100644 --- a/pySim/esim/saip/personalization.py +++ b/pySim/esim/saip/personalization.py @@ -19,7 +19,7 @@ import io from typing import List, Tuple

-from pySim.tlv import camel_to_snake +from osmocom.tlv import camel_to_snake from pySim.utils import enc_iccid, enc_imsi, h2b, rpad, sanitize_iccid from pySim.esim.saip import ProfileElement, ProfileElementSequence

diff --git a/pySim/euicc.py b/pySim/euicc.py index 8bd1ae4..1ad0f22 100644 --- a/pySim/euicc.py +++ b/pySim/euicc.py @@ -27,11 +27,12 @@

from construct import Array, Struct, FlagsEnum, GreedyRange from cmd2 import cmd2, CommandSet, with_default_category +from osmocom.utils import Hexstr +from osmocom.tlv import * +from osmocom.construct import *

-from pySim.tlv import * -from pySim.construct import * -from pySim.commands import SimCardCommands from pySim.utils import Hexstr, SwHexstr, SwMatchstr +from pySim.commands import SimCardCommands import pySim.global_platform

def compute_eid_checksum(eid) -> str: diff --git a/pySim/filesystem.py b/pySim/filesystem.py index b166bc2..93371a9 100644 --- a/pySim/filesystem.py +++ b/pySim/filesystem.py @@ -35,10 +35,11 @@ from cmd2 import CommandSet, with_default_category from smartcard.util import toBytes

-from pySim.utils import sw_match, h2b, b2h, is_hex, auto_int, auto_uint8, auto_uint16, is_hexstr, JsonEncoder -from pySim.utils import bertlv_parse_one +from osmocom.utils import h2b, b2h, is_hex, auto_int, auto_uint8, auto_uint16, is_hexstr, JsonEncoder +from osmocom.tlv import bertlv_parse_one +from osmocom.construct import filter_dict, parse_construct, build_construct

-from pySim.construct import filter_dict, parse_construct, build_construct +from pySim.utils import sw_match from pySim.jsonpath import js_path_modify from pySim.commands import SimCardCommands from pySim.exceptions import SwMatchError diff --git a/pySim/global_platform/__init__.py b/pySim/global_platform/__init__.py index 308a32a..e499e0f 100644 --- a/pySim/global_platform/__init__.py +++ b/pySim/global_platform/__init__.py @@ -23,12 +23,13 @@ from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const from Cryptodome.Random import get_random_bytes from Cryptodome.Cipher import DES, DES3, AES +from osmocom.utils import * +from osmocom.tlv import * +from osmocom.construct import * +from pySim.utils import ResTuple from pySim.card_key_provider import card_key_provider_get_field from pySim.global_platform.scp import SCP02, SCP03 -from pySim.construct import * -from pySim.utils import * from pySim.filesystem import * -from pySim.tlv import * from pySim.profile import CardProfile from pySim.ota import SimFileAccessAndToolkitAppSpecParams

diff --git a/pySim/global_platform/http.py b/pySim/global_platform/http.py index f0e98aa..dee196d 100644 --- a/pySim/global_platform/http.py +++ b/pySim/global_platform/http.py @@ -20,8 +20,8 @@ from construct import Struct, Int8ub, Int16ub, Bytes, GreedyBytes, GreedyString, BytesInteger from construct import this, len_, Rebuild, Const from construct import Optional as COptional +from osmocom.tlv import BER_TLV_IE

-from pySim.tlv import BER_TLV_IE from pySim import cat

diff --git a/pySim/global_platform/scp.py b/pySim/global_platform/scp.py index 17463ff..d2c72b3 100644 --- a/pySim/global_platform/scp.py +++ b/pySim/global_platform/scp.py @@ -22,7 +22,9 @@ from Cryptodome.Util.strxor import strxor from construct import Struct, Bytes, Int8ub, Int16ub, Const from construct import Optional as COptional -from pySim.utils import b2h, bertlv_parse_len, bertlv_encode_len +from osmocom.utils import b2h +from osmocom.tlv import bertlv_parse_len, bertlv_encode_len + from pySim.secure_channel import SecureChannel

logger = logging.getLogger(__name__) diff --git a/pySim/global_platform/uicc.py b/pySim/global_platform/uicc.py index 2cf5e96..97c0df0 100644 --- a/pySim/global_platform/uicc.py +++ b/pySim/global_platform/uicc.py @@ -19,9 +19,9 @@

from construct import Optional as COptional from construct import Struct, GreedyRange, FlagsEnum, Int16ub, Int24ub, Padding, Bit, Const -from pySim.construct import * -from pySim.utils import * -from pySim.tlv import * +from osmocom.construct import * +from osmocom.utils import * +from osmocom.tlv import *

# Section 11.6.2.3 / Table 11-58 class SecurityDomainAid(BER_TLV_IE, tag=0x4f): diff --git a/pySim/gsm_r.py b/pySim/gsm_r.py index 14dbb2d..b29fbc4 100644 --- a/pySim/gsm_r.py +++ b/pySim/gsm_r.py @@ -28,8 +28,8 @@ from struct import pack, unpack from construct import Struct, Bytes, Int8ub, Int16ub, Int24ub, Int32ub, FlagsEnum from construct import Optional as COptional +from osmocom.construct import *

-from pySim.construct import * from pySim.profile import CardProfileAddon from pySim.filesystem import *

diff --git a/pySim/gsmtap.py b/pySim/gsmtap.py index f113aeb..26ac177 100644 --- a/pySim/gsmtap.py +++ b/pySim/gsmtap.py @@ -26,7 +26,7 @@ from construct import Optional as COptional from construct import Int8ub, Int8sb, Int32ub, BitStruct, Enum, GreedyBytes, Struct, Switch from construct import this, PaddedString -from pySim.construct import * +from osmocom.construct import *

# The root definition of GSMTAP can be found at # https://cgit.osmocom.org/cgit/libosmocore/tree/include/osmocom/core/gsmtap.h diff --git a/pySim/iso7816_4.py b/pySim/iso7816_4.py index 64d814d..ae0c2c8 100644 --- a/pySim/iso7816_4.py +++ b/pySim/iso7816_4.py @@ -18,10 +18,8 @@ """

from construct import GreedyBytes, GreedyString -from pySim.construct import * -from pySim.utils import * -from pySim.filesystem import * -from pySim.tlv import * +from osmocom.tlv import * +from osmocom.construct import *

# Table 91 + Section 8.2.1.2 class ApplicationId(BER_TLV_IE, tag=0x4f): diff --git a/pySim/ota.py b/pySim/ota.py index bb5a1f6..8d7e0fd 100644 --- a/pySim/ota.py +++ b/pySim/ota.py @@ -21,9 +21,9 @@ from typing import Optional, Tuple from construct import Enum, Int8ub, Int16ub, Struct, Bytes, GreedyBytes, BitsInteger, BitStruct from construct import Flag, Padding, Switch, this, PrefixedArray, GreedyRange +from osmocom.construct import * +from osmocom.utils import b2h

-from pySim.construct import * -from pySim.utils import b2h from pySim.sms import UserDataHeader

# ETS TS 102 225 gives the general command structure and the dialects for CAT_TP, TCP/IP and HTTPS diff --git a/pySim/runtime.py b/pySim/runtime.py index a813e6d..b903ccb 100644 --- a/pySim/runtime.py +++ b/pySim/runtime.py @@ -18,8 +18,9 @@ # along with this program. If not, see http://www.gnu.org/licenses/.

from typing import Optional, Tuple +from osmocom.utils import h2b, i2h, is_hex, Hexstr +from osmocom.tlv import bertlv_parse_one

-from pySim.utils import h2b, i2h, is_hex, bertlv_parse_one, Hexstr from pySim.exceptions import * from pySim.filesystem import *

diff --git a/pySim/secure_channel.py b/pySim/secure_channel.py index 974780e..34e84de 100644 --- a/pySim/secure_channel.py +++ b/pySim/secure_channel.py @@ -16,7 +16,9 @@ # along with this program. If not, see http://www.gnu.org/licenses/.

import abc -from pySim.utils import b2h, h2b, ResTuple, Hexstr +from osmocom.utils import b2h, h2b, Hexstr + +from pySim.utils import ResTuple

class SecureChannel(abc.ABC): @abc.abstractmethod diff --git a/pySim/sms.py b/pySim/sms.py index 5ba5ec8..df835e4 100644 --- a/pySim/sms.py +++ b/pySim/sms.py @@ -23,9 +23,8 @@ from construct import Int8ub, Byte, Bytes, Bit, Flag, BitsInteger from construct import Struct, Enum, Tell, BitStruct, this, Padding from construct import Prefixed, GreedyRange, GreedyBytes - -from pySim.construct import HexAdapter, BcdAdapter, TonNpi -from pySim.utils import Hexstr, h2b, b2h +from osmocom.construct import HexAdapter, BcdAdapter, TonNpi +from osmocom.utils import Hexstr, h2b, b2h

from smpp.pdu import pdu_types, operations

diff --git a/pySim/sysmocom_sja2.py b/pySim/sysmocom_sja2.py index 97de5a5..2c6d66a 100644 --- a/pySim/sysmocom_sja2.py +++ b/pySim/sysmocom_sja2.py @@ -21,11 +21,11 @@ from construct import FlagsEnum, Byte, Struct, Int8ub, Bytes, Mapping, Enum, Padding, BitsInteger from construct import Bit, this, Int32ub, Int16ub, Nibble, BytesInteger, GreedyRange, Const from construct import Optional as COptional +from osmocom.utils import * +from osmocom.construct import *

-from pySim.utils import * from pySim.filesystem import * from pySim.runtime import RuntimeState -from pySim.construct import * import pySim

key_type2str = { diff --git a/pySim/tlv.py b/pySim/tlv.py deleted file mode 100644 index bd3c901..0000000 --- a/pySim/tlv.py +++ /dev/null @@ -1,553 +0,0 @@ -"""object-oriented TLV parser/encoder library.""" - -# (C) 2021 by Harald Welte laforge@osmocom.org -# All Rights Reserved -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import inspect -import abc -import re -from typing import List, Tuple, Optional - -from pySim.utils import bertlv_encode_len, bertlv_parse_len, bertlv_encode_tag, bertlv_parse_tag -from pySim.utils import comprehensiontlv_encode_tag, comprehensiontlv_parse_tag -from pySim.utils import bertlv_parse_tag_raw, comprehensiontlv_parse_tag_raw -from pySim.utils import dgi_parse_tag_raw, dgi_parse_len, dgi_encode_tag, dgi_encode_len - -from pySim.construct import build_construct, parse_construct - - -def camel_to_snake(name): - name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower() - -class TlvMeta(abc.ABCMeta): - """Metaclass which we use to set some class variables at the time of defining a subclass. - This allows us to create subclasses for each TLV/IE type, where the class represents fixed - parameters like the tag/type and instances of it represent the actual TLV data.""" - def __new__(mcs, name, bases, namespace, **kwargs): - #print("TlvMeta_new_(mcs=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (mcs, name, bases, namespace, kwargs)) - x = super().__new__(mcs, name, bases, namespace) - # this becomes a _class_ variable, not an instance variable - x.tag = namespace.get('tag', kwargs.get('tag', None)) - x.desc = namespace.get('desc', kwargs.get('desc', None)) - nested = namespace.get('nested', kwargs.get('nested', None)) - if nested is None or inspect.isclass(nested) and issubclass(nested, TLV_IE_Collection): - # caller has specified TLV_IE_Collection sub-class, we can directly reference it - x.nested_collection_cls = nested - else: - # caller passed list of other TLV classes that might possibly appear within us, - # build a dynamically-created TLV_IE_Collection sub-class and reference it - name = 'auto_collection_%s' % (name) - cls = type(name, (TLV_IE_Collection,), {'nested': nested}) - x.nested_collection_cls = cls - return x - - -class TlvCollectionMeta(abc.ABCMeta): - """Metaclass which we use to set some class variables at the time of defining a subclass. - This allows us to create subclasses for each Collection type, where the class represents fixed - parameters like the nested IE classes and instances of it represent the actual TLV data.""" - def __new__(mcs, name, bases, namespace, **kwargs): - #print("TlvCollectionMeta_new_(mcs=%s, name=%s, bases=%s, namespace=%s, kwargs=%s)" % (mcs, name, bases, namespace, kwargs)) - x = super().__new__(mcs, name, bases, namespace) - # this becomes a _class_ variable, not an instance variable - x.possible_nested = namespace.get('nested', kwargs.get('nested', None)) - return x - - -class Transcodable(abc.ABC): - _construct = None - """Base class for something that can be encoded + encoded. Decoding and Encoding happens either - * via a 'construct' object stored in a derived class' _construct variable, or - * via a 'construct' object stored in an instance _construct variable, or - * via a derived class' _{to,from}_bytes() methods.""" - - def __init__(self): - self.encoded = None - self.decoded = None - self._construct = None - - def to_bytes(self, context: dict = {}) -> bytes: - """Convert from internal representation to binary bytes. Store the binary result - in the internal state and return it.""" - if self.decoded is None: - do = b'' - elif self._construct: - do = build_construct(self._construct, self.decoded, context) - elif self.__class__._construct: - do = build_construct(self.__class__._construct, self.decoded, context) - else: - do = self._to_bytes() - self.encoded = do - return do - - # not an abstractmethod, as it is only required if no _construct exists - def _to_bytes(self): - raise NotImplementedError('%s._to_bytes' % type(self).__name__) - - def from_bytes(self, do: bytes, context: dict = {}): - """Convert from binary bytes to internal representation. Store the decoded result - in the internal state and return it.""" - self.encoded = do - if self.encoded == b'': - self.decoded = None - elif self._construct: - self.decoded = parse_construct(self._construct, do, context=context) - elif self.__class__._construct: - self.decoded = parse_construct(self.__class__._construct, do, context=context) - else: - self.decoded = self._from_bytes(do) - return self.decoded - - # not an abstractmethod, as it is only required if no _construct exists - def _from_bytes(self, do: bytes): - raise NotImplementedError('%s._from_bytes' % type(self).__name__) - - -class IE(Transcodable, metaclass=TlvMeta): - # we specify the metaclass so any downstream subclasses will automatically use it - """Base class for various Information Elements. We understand the notion of a hierarchy - of IEs on top of the Transcodable class.""" - # this is overridden by the TlvMeta metaclass, if it is used to create subclasses - nested_collection_cls = None - tag = None - - def __init__(self, **kwargs): - super().__init__() - self.nested_collection = None - if self.nested_collection_cls: - self.nested_collection = self.nested_collection_cls() - # if we are a constructed IE, [ordered] list of actual child-IE instances - self.children = kwargs.get('children', []) - self.decoded = kwargs.get('decoded', None) - - def __repr__(self): - """Return a string representing the [nested] IE data (for print).""" - if len(self.children): - member_strs = [repr(x) for x in self.children] - return '%s(%s)' % (type(self).__name__, ','.join(member_strs)) - else: - return '%s(%s)' % (type(self).__name__, self.decoded) - - def to_val_dict(self): - """Return a JSON-serializable dict representing just the [nested] value portion of the IE - data. This does not include any indication of the type of 'self', so the resulting dict alone - will be insufficient ot recreate an object from it without additional type information.""" - if len(self.children): - return [x.to_dict() for x in self.children] - else: - return self.decoded - - def from_val_dict(self, decoded): - """Set the IE internal decoded representation to data from the argument. - If this is a nested IE, the child IE instance list is re-created. - - This method is symmetrical to to_val_dict() aboe, i.e. there is no outer dict - containig the snake-reformatted type name of 'self'.""" - if self.nested_collection: - self.children = self.nested_collection.from_dict(decoded) - else: - self.children = [] - self.decoded = decoded - - def to_dict(self): - """Return a JSON-serializable dict representing the [nested] IE data. The returned - data contains an outer dict with the snake-reformatted type of 'self' and is hence - sufficient to re-create an object from it.""" - return {camel_to_snake(type(self).__name__): self.to_val_dict()} - - def from_dict(self, decoded: dict): - """Set the IE internal decoded representation to data from the argument. - If this is a nested IE, the child IE instance list is re-created. - - This method is symmetrical to to_dict() above, i.e. the outer dict must contain just a single - key-value pair, where the key is the snake-reformatted type name of 'self'""" - expected_key_name = camel_to_snake(type(self).__name__) - if not expected_key_name in decoded: - raise ValueError("Dict %s doesn't contain expected key %s" % (decoded, expected_key_name)) - self.from_val_dict(decoded[expected_key_name]) - - def is_constructed(self): - """Is this IE constructed by further nested IEs?""" - return bool(len(self.children) > 0) - - @abc.abstractmethod - def to_ie(self, context: dict = {}) -> bytes: - """Convert the internal representation to entire IE including IE header.""" - - def to_bytes(self, context: dict = {}) -> bytes: - """Convert the internal representation *of the value part* to binary bytes.""" - if self.is_constructed(): - # concatenate the encoded IE of all children to form the value part - out = b'' - for c in self.children: - out += c.to_ie(context=context) - return out - else: - return super().to_bytes(context=context) - - def from_bytes(self, do: bytes, context: dict = {}): - """Parse *the value part* from binary bytes to internal representation.""" - if self.nested_collection: - self.children = self.nested_collection.from_bytes(do, context=context) - else: - self.children = [] - return super().from_bytes(do, context=context) - - def child_by_name(self, name: str) -> Optional['IE']: - """Return a child IE instance of given snake-case/json type name. This only works in case - there is no more than one child IE of the given type.""" - children = list(filter(lambda c: camel_to_snake(type(c).__name__) == name, self.children)) - if len(children) > 1: - raise KeyError('There are multiple children of class %s' % name) - elif len(children) == 1: - return children[0] - - def child_by_type(self, cls) -> Optional['IE']: - """Return a child IE instance of given type (class). This only works in case - there is no more than one child IE of the given type.""" - children = list(filter(lambda c: isinstance(c, cls), self.children)) - if len(children) > 1: - raise KeyError('There are multiple children of class %s' % cls) - elif len(children) == 1: - return children[0] - - -class TLV_IE(IE): - """Abstract base class for various TLV type Information Elements.""" - - def _compute_tag(self) -> int: - """Compute the tag (sometimes the tag encodes part of the value).""" - return self.tag - - @classmethod - @abc.abstractmethod - def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]: - """Obtain the raw TAG at the start of the bytes provided by the user.""" - - @classmethod - @abc.abstractmethod - def _parse_len(cls, do: bytes) -> Tuple[int, bytes]: - """Obtain the length encoded at the start of the bytes provided by the user.""" - - @abc.abstractmethod - def _encode_tag(self) -> bytes: - """Encode the tag part. Must be provided by derived (TLV format specific) class.""" - - @abc.abstractmethod - def _encode_len(self, val: bytes) -> bytes: - """Encode the length part assuming a certain binary value. Must be provided by - derived (TLV format specific) class.""" - - def to_ie(self, context: dict = {}): - return self.to_tlv(context=context) - - def to_tlv(self, context: dict = {}): - """Convert the internal representation to binary TLV bytes.""" - val = self.to_bytes(context=context) - return self._encode_tag() + self._encode_len(val) + val - - def is_tag_compatible(self, rawtag) -> bool: - """Is the given rawtag compatible with this class?""" - return rawtag == self._compute_tag() - - def from_tlv(self, do: bytes, context: dict = {}): - if len(do) == 0: - return {}, b'' - (rawtag, remainder) = self.__class__._parse_tag_raw(do) - if rawtag: - if not self.is_tag_compatible(rawtag): - raise ValueError("%s: Encountered tag %s doesn't match our supported tag %s" % - (self, rawtag, self.tag)) - (length, remainder) = self.__class__._parse_len(remainder) - value = remainder[:length] - remainder = remainder[length:] - else: - value = do - remainder = b'' - dec = self.from_bytes(value, context=context) - return dec, remainder - - -class COMPACT_TLV_IE(TLV_IE): - """TLV_IE formatted as COMPACT-TLV described in ISO 7816""" - - @classmethod - def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]: - return do[0] >> 4, do - - @classmethod - def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]: - rawtag, remainder = cls._parse_tag_raw(do) - return {'tag': rawtag}, remainder - - @classmethod - def _parse_len(cls, do: bytes) -> Tuple[int, bytes]: - return do[0] & 0xf, do[1:] - - def _encode_tag(self) -> bytes: - """Not needed as we override the to_tlv() method to encode tag+length into one byte.""" - raise NotImplementedError - - def _encode_len(self): - """Not needed as we override the to_tlv() method to encode tag+length into one byte.""" - raise NotImplementedError - - def to_tlv(self, context: dict = {}): - val = self.to_bytes(context=context) - return bytes([(self.tag << 4) | (len(val) & 0xF)]) + val - - -class BER_TLV_IE(TLV_IE): - """TLV_IE formatted as ASN.1 BER described in ITU-T X.690 8.1.2.""" - - @classmethod - def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]: - return bertlv_parse_tag(do) - - @classmethod - def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]: - return bertlv_parse_tag_raw(do) - - @classmethod - def _parse_len(cls, do: bytes) -> Tuple[int, bytes]: - return bertlv_parse_len(do) - - def _encode_tag(self) -> bytes: - return bertlv_encode_tag(self._compute_tag()) - - def _encode_len(self, val: bytes) -> bytes: - return bertlv_encode_len(len(val)) - - -class ComprTlvMeta(TlvMeta): - def __new__(mcs, name, bases, namespace, **kwargs): - x = super().__new__(mcs, name, bases, namespace, **kwargs) - if x.tag: - # we currently assume that the tag values always have the comprehension bit set; - # let's fix it up if a derived class has forgotten about that - if x.tag > 0xff and x.tag & 0x8000 == 0: - print("Fixing up COMPR_TLV_IE class %s: tag=0x%x has no comprehension bit" % (name, x.tag)) - x.tag = x.tag | 0x8000 - elif x.tag & 0x80 == 0: - print("Fixing up COMPR_TLV_IE class %s: tag=0x%x has no comprehension bit" % (name, x.tag)) - x.tag = x.tag | 0x80 - return x - -class COMPR_TLV_IE(TLV_IE, metaclass=ComprTlvMeta): - """TLV_IE formated as COMPREHENSION-TLV as described in ETSI TS 101 220.""" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.comprehension = False - - @classmethod - def _decode_tag(cls, do: bytes) -> Tuple[dict, bytes]: - return comprehensiontlv_parse_tag(do) - - @classmethod - def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]: - return comprehensiontlv_parse_tag_raw(do) - - @classmethod - def _parse_len(cls, do: bytes) -> Tuple[int, bytes]: - return bertlv_parse_len(do) - - def is_tag_compatible(self, rawtag: int) -> bool: - """Override is_tag_compatible as we need to mask out the - comprehension bit when doing compares.""" - ctag = self._compute_tag() - if ctag > 0xff: - return ctag & 0x7fff == rawtag & 0x7fff - else: - return ctag & 0x7f == rawtag & 0x7f - - def _encode_tag(self) -> bytes: - return comprehensiontlv_encode_tag(self._compute_tag()) - - def _encode_len(self, val: bytes) -> bytes: - return bertlv_encode_len(len(val)) - - -class DGI_TLV_IE(TLV_IE): - """TLV_IE formated as GlobalPlatform Systems Scripting Language Specification v1.1.0 Annex B.""" - - @classmethod - def _parse_tag_raw(cls, do: bytes) -> Tuple[int, bytes]: - return dgi_parse_tag_raw(do) - - @classmethod - def _parse_len(cls, do: bytes) -> Tuple[int, bytes]: - return dgi_parse_len(do) - - def _encode_tag(self) -> bytes: - return dgi_encode_tag(self._compute_tag()) - - def _encode_len(self, val: bytes) -> bytes: - return dgi_encode_len(len(val)) - - -class TLV_IE_Collection(metaclass=TlvCollectionMeta): - # we specify the metaclass so any downstream subclasses will automatically use it - """A TLV_IE_Collection consists of multiple TLV_IE classes identified by their tags. - A given encoded DO may contain any of them in any order, and may contain multiple instances - of each DO.""" - # this is overridden by the TlvCollectionMeta metaclass, if it is used to create subclasses - possible_nested = [] - - def __init__(self, desc=None, **kwargs): - self.desc = desc - #print("possible_nested: ", self.possible_nested) - self.members = kwargs.get('nested', self.possible_nested) - self.members_by_tag = {} - self.members_by_name = {} - self.members_by_tag = {m.tag: m for m in self.members} - self.members_by_name = {camel_to_snake(m.__name__): m for m in self.members} - # if we are a constructed IE, [ordered] list of actual child-IE instances - self.children = kwargs.get('children', []) - self.encoded = None - - def __str__(self): - member_strs = [str(x) for x in self.members] - return '%s(%s)' % (type(self).__name__, ','.join(member_strs)) - - def __repr__(self): - member_strs = [repr(x) for x in self.members] - return '%s(%s)' % (self.__class__, ','.join(member_strs)) - - def __add__(self, other): - """Extending TLV_IE_Collections with other TLV_IE_Collections or TLV_IEs.""" - if isinstance(other, TLV_IE_Collection): - # adding one collection to another - members = self.members + other.members - return TLV_IE_Collection(self.desc, nested=members) - elif inspect.isclass(other) and issubclass(other, TLV_IE): - # adding a member to a collection - return TLV_IE_Collection(self.desc, nested=self.members + [other]) - else: - raise TypeError - - def from_bytes(self, binary: bytes, context: dict = {}) -> List[TLV_IE]: - """Create a list of TLV_IEs from the collection based on binary input data. - Args: - binary : binary bytes of encoded data - Returns: - list of instances of TLV_IE sub-classes containing parsed data - """ - self.encoded = binary - # list of instances of TLV_IE collection member classes appearing in the data - res = [] - remainder = binary - first = next(iter(self.members_by_tag.values())) - # iterate until no binary trailer is left - while len(remainder): - context['siblings'] = res - # obtain the tag at the start of the remainder - tag, _r = first._parse_tag_raw(remainder) - if tag is None: - break - if issubclass(first, COMPR_TLV_IE): - tag = tag | 0x80 # HACK: always assume comprehension - if tag in self.members_by_tag: - cls = self.members_by_tag[tag] - # create an instance and parse accordingly - inst = cls() - _dec, remainder = inst.from_tlv(remainder, context=context) - res.append(inst) - else: - # unknown tag; create the related class on-the-fly using the same base class - name = 'unknown_%s_%X' % (first.__base__.__name__, tag) - cls = type(name, (first.__base__,), {'tag': tag, 'possible_nested': [], - 'nested_collection_cls': None}) - cls._from_bytes = lambda s, a: {'raw': a.hex()} - cls._to_bytes = lambda s: bytes.fromhex(s.decoded['raw']) - # create an instance and parse accordingly - inst = cls() - _dec, remainder = inst.from_tlv(remainder, context=context) - res.append(inst) - self.children = res - return res - - def from_dict(self, decoded: List[dict]) -> List[TLV_IE]: - """Create a list of TLV_IE instances from the collection based on an array - of dicts, where they key indicates the name of the TLV_IE subclass to use.""" - # list of instances of TLV_IE collection member classes appearing in the data - res = [] - # iterate over members of the list passed into "decoded" - for i in decoded: - # iterate over all the keys (typically one!) within the current list item dict - for k in i.keys(): - # check if we have a member identified by the dict key - if k in self.members_by_name: - # resolve the class for that name; create an instance of it - cls = self.members_by_name[k] - inst = cls() - inst.from_dict({k: i[k]}) - res.append(inst) - else: - raise ValueError('%s: Unknown TLV Class %s in %s; expected %s' % - (self, k, decoded, self.members_by_name.keys())) - self.children = res - return res - - def to_dict(self): - # we intentionally return not a dict, but a list of dicts. We could prefix by - # self.__class__.__name__, but that is usually some meaningless auto-generated collection name. - return [x.to_dict() for x in self.children] - - def to_bytes(self, context: dict = {}): - out = b'' - context['siblings'] = self.children - for c in self.children: - out += c.to_tlv(context=context) - return out - - def from_tlv(self, do, context: dict = {}): - return self.from_bytes(do, context=context) - - def to_tlv(self, context: dict = {}): - return self.to_bytes(context=context) - - -def flatten_dict_lists(inp): - """hierarchically flatten each list-of-dicts into a single dict. This is useful to - make the output of hierarchical TLV decoder structures flatter and more easy to read.""" - def are_all_elements_dict(l): - for e in l: - if not isinstance(e, dict): - return False - return True - - def are_elements_unique(lod): - set_of_keys = {list(x.keys())[0] for x in lod} - return len(lod) == len(set_of_keys) - - if isinstance(inp, list): - if are_all_elements_dict(inp) and are_elements_unique(inp): - # flatten into one shared dict - newdict = {} - for e in inp: - key = list(e.keys())[0] - newdict[key] = e[key] - inp = newdict - # process result as any native dict - return {k:flatten_dict_lists(v) for k,v in inp.items()} - else: - return [flatten_dict_lists(x) for x in inp] - elif isinstance(inp, dict): - return {k:flatten_dict_lists(v) for k,v in inp.items()} - else: - return inp diff --git a/pySim/transport/__init__.py b/pySim/transport/__init__.py index 59067b3..47f6547 100644 --- a/pySim/transport/__init__.py +++ b/pySim/transport/__init__.py @@ -8,9 +8,10 @@ import argparse from typing import Optional, Tuple from construct import Construct +from osmocom.utils import b2h, h2b, i2h, Hexstr

from pySim.exceptions import * -from pySim.utils import sw_match, b2h, h2b, i2h, Hexstr, SwHexstr, SwMatchstr, ResTuple +from pySim.utils import SwHexstr, SwMatchstr, ResTuple, sw_match from pySim.cat import ProactiveCommand, CommandDetails, DeviceIdentities, Result

# diff --git a/pySim/transport/calypso.py b/pySim/transport/calypso.py index 6252137..d8e5b89 100644 --- a/pySim/transport/calypso.py +++ b/pySim/transport/calypso.py @@ -22,10 +22,11 @@ import os import argparse from typing import Optional +from osmocom.utils import h2b, b2h, Hexstr

from pySim.transport import LinkBase from pySim.exceptions import ReaderError, ProtocolError -from pySim.utils import h2b, b2h, Hexstr, ResTuple +from pySim.utils import ResTuple

class L1CTLMessage: diff --git a/pySim/transport/modem_atcmd.py b/pySim/transport/modem_atcmd.py index 5943e3a..882cb65 100644 --- a/pySim/transport/modem_atcmd.py +++ b/pySim/transport/modem_atcmd.py @@ -22,8 +22,9 @@ import argparse from typing import Optional import serial +from osmocom.utils import Hexstr

-from pySim.utils import Hexstr, ResTuple +from pySim.utils import ResTuple from pySim.transport import LinkBase from pySim.exceptions import ReaderError, ProtocolError

diff --git a/pySim/transport/pcsc.py b/pySim/transport/pcsc.py index 8a1e03c..7843f7f 100644 --- a/pySim/transport/pcsc.py +++ b/pySim/transport/pcsc.py @@ -27,9 +27,11 @@ from smartcard.System import readers from smartcard.ExclusiveConnectCardConnection import ExclusiveConnectCardConnection

+from osmocom.utils import h2i, i2h, Hexstr + from pySim.exceptions import NoCardError, ProtocolError, ReaderError from pySim.transport import LinkBase -from pySim.utils import h2i, i2h, Hexstr, ResTuple +from pySim.utils import ResTuple

class PcscSimLink(LinkBase): diff --git a/pySim/transport/serial.py b/pySim/transport/serial.py index b4eddcd..e155226 100644 --- a/pySim/transport/serial.py +++ b/pySim/transport/serial.py @@ -21,10 +21,11 @@ import argparse from typing import Optional import serial +from osmocom.utils import h2b, b2h, Hexstr

from pySim.exceptions import NoCardError, ProtocolError from pySim.transport import LinkBase -from pySim.utils import h2b, b2h, Hexstr, ResTuple +from pySim.utils import ResTuple

class SerialSimLink(LinkBase): diff --git a/pySim/ts_102_221.py b/pySim/ts_102_221.py index 9e0db68..d671c3e 100644 --- a/pySim/ts_102_221.py +++ b/pySim/ts_102_221.py @@ -21,10 +21,11 @@ from construct import Select, Const, Bit, Struct, Int16ub, FlagsEnum, GreedyString, ValidationError from construct import Optional as COptional, Computed

-from pySim.construct import * +from osmocom.construct import * +from osmocom.utils import * +from osmocom.tlv import * from pySim.utils import * from pySim.filesystem import * -from pySim.tlv import * from pySim.profile import CardProfile from pySim.profile import match_uicc from pySim import iso7816_4 diff --git a/pySim/ts_102_222.py b/pySim/ts_102_222.py index 30254d1..e54c196 100644 --- a/pySim/ts_102_222.py +++ b/pySim/ts_102_222.py @@ -19,11 +19,9 @@

from typing import List import argparse - import cmd2 from cmd2 import CommandSet, with_default_category - -from pySim.utils import b2h, auto_uint8, auto_uint16, is_hexstr +from osmocom.utils import b2h, auto_uint8, auto_uint16, is_hexstr

from pySim.ts_102_221 import *

diff --git a/pySim/ts_102_310.py b/pySim/ts_102_310.py index ea3a448..e4c1b92 100644 --- a/pySim/ts_102_310.py +++ b/pySim/ts_102_310.py @@ -17,13 +17,12 @@ along with this program. If not, see http://www.gnu.org/licenses/. """

-from pySim.construct import * from construct import * from construct import Optional as COptional +from osmocom.construct import *

-#from pySim.utils import * +from osmocom.tlv import BER_TLV_IE, TLV_IE_Collection from pySim.filesystem import CardDF, TransparentEF -from pySim.tlv import BER_TLV_IE, TLV_IE_Collection

# TS102 310 Section 7.1 class EF_EAPKEYS(TransparentEF): diff --git a/pySim/ts_31_102.py b/pySim/ts_31_102.py index c82cd2f..b70ae54 100644 --- a/pySim/ts_31_102.py +++ b/pySim/ts_31_102.py @@ -32,6 +32,10 @@ from construct import Int32ub, Nibble, GreedyRange, Struct, FlagsEnum, Switch, this, Int16ub, Padding from construct import Bytewise, Int24ub, PaddedString, PrefixedArray, If

+from osmocom.utils import is_hexstr +from osmocom.tlv import * +from osmocom.construct import * + import pySim.ts_102_221 from pySim.ts_51_011 import EF_ACMmax, EF_AAeM, EF_eMLPP, EF_CMI, EF_PNN from pySim.ts_51_011 import EF_MMSN, EF_MMSICP, EF_MMSUP, EF_MMSUCP, EF_VGCS, EF_VGCSS, EF_NIA @@ -40,13 +44,10 @@ from pySim.ts_51_011 import EF_IMSI, EF_xPLMNwAcT, EF_SPN, EF_CBMI, EF_ACC, EF_PLMNsel from pySim.ts_51_011 import EF_Kc, EF_CPBCCH, EF_InvScan from pySim.ts_102_221 import EF_ARR -from pySim.tlv import * from pySim.filesystem import * from pySim.ts_31_102_telecom import DF_PHONEBOOK, EF_UServiceTable from pySim.ts_31_103_shared import EF_IMSConfigData, EF_XCAPConfigData, EF_MuDMiDConfigData from pySim.ts_31_103_shared import EF_AC_GBAUAPI, EF_IMSDCI -from pySim.construct import * -from pySim.utils import is_hexstr from pySim.cat import SMS_TPDU, DeviceIdentities, SMSPPDownload

# Mapping between USIM Service Number and its description diff --git a/pySim/ts_31_102_telecom.py b/pySim/ts_31_102_telecom.py index 962b7dd..93154bb 100644 --- a/pySim/ts_31_102_telecom.py +++ b/pySim/ts_31_102_telecom.py @@ -28,10 +28,10 @@

from construct import Optional as COptional from construct import Struct, Int16ub, Int32ub +from osmocom.tlv import * +from osmocom.construct import *

-from pySim.tlv import * from pySim.filesystem import * -from pySim.construct import *

# TS 31.102 Section 4.2.8 class EF_UServiceTable(TransparentEF): diff --git a/pySim/ts_31_103.py b/pySim/ts_31_103.py index 0de0d62..c0142bc 100644 --- a/pySim/ts_31_103.py +++ b/pySim/ts_31_103.py @@ -23,16 +23,16 @@ #

from construct import Struct, Switch, this, Bytes, GreedyString +from osmocom.utils import * +from osmocom.tlv import * +from osmocom.construct import * from pySim.filesystem import * -from pySim.utils import * -from pySim.tlv import * from pySim.ts_51_011 import EF_AD, EF_SMS, EF_SMSS, EF_SMSR, EF_SMSP from pySim.ts_31_102 import ADF_USIM, EF_FromPreferred from pySim.ts_31_102_telecom import EF_UServiceTable from pySim.ts_31_103_shared import * import pySim.ts_102_221 from pySim.ts_102_221 import EF_ARR -from pySim.construct import *

# Mapping between ISIM Service Number and its description EF_IST_map = { diff --git a/pySim/ts_31_103_shared.py b/pySim/ts_31_103_shared.py index 73416cb..25f3e97 100644 --- a/pySim/ts_31_103_shared.py +++ b/pySim/ts_31_103_shared.py @@ -20,9 +20,9 @@ #

from construct import Struct, Switch, Bytes, GreedyString, GreedyBytes, Int8ub, Prefixed, Enum, Byte -from pySim.construct import HexAdapter, Utf8Adapter +from osmocom.tlv import BER_TLV_IE, TLV_IE_Collection +from osmocom.construct import HexAdapter, Utf8Adapter from pySim.filesystem import * -from pySim.tlv import BER_TLV_IE, TLV_IE_Collection

# TS 31.103 Section 4.2.16 class EF_UICCIARI(LinFixedEF): diff --git a/pySim/ts_31_104.py b/pySim/ts_31_104.py index 239acbf..2c937dc 100644 --- a/pySim/ts_31_104.py +++ b/pySim/ts_31_104.py @@ -20,9 +20,9 @@ # along with this program. If not, see http://www.gnu.org/licenses/. #

+from osmocom.utils import * +from osmocom.tlv import * from pySim.filesystem import * -from pySim.utils import * -from pySim.tlv import * from pySim.ts_31_102 import ADF_USIM from pySim.ts_51_011 import EF_IMSI, EF_AD import pySim.ts_102_221 diff --git a/pySim/ts_51_011.py b/pySim/ts_51_011.py index f6bd3cc..b0d53e0 100644 --- a/pySim/ts_51_011.py +++ b/pySim/ts_51_011.py @@ -29,19 +29,22 @@ # along with this program. If not, see http://www.gnu.org/licenses/. #

+import enum +from struct import pack, unpack +from typing import Tuple + +from construct import Optional as COptional +from construct import * +from osmocom.tlv import * +from osmocom.utils import * +from osmocom.construct import * + +from pySim.utils import dec_iccid, enc_iccid, dec_imsi, enc_imsi, dec_plmn, enc_plmn, dec_xplmn_w_act from pySim.profile import match_sim from pySim.profile import CardProfile, CardProfileAddon from pySim.filesystem import * from pySim.ts_31_102_telecom import DF_PHONEBOOK, DF_MULTIMEDIA, DF_MCS, DF_V2X from pySim.gsm_r import AddonGSMR -import enum -from pySim.construct import * -from construct import Optional as COptional -from construct import * -from struct import pack, unpack -from typing import Tuple -from pySim.tlv import * -from pySim.utils import *

# Mapping between SIM Service Number and its description EF_SST_map = { diff --git a/pySim/utils.py b/pySim/utils.py index 2362b59..05ad962 100644 --- a/pySim/utils.py +++ b/pySim/utils.py @@ -10,6 +10,8 @@ import argparse from io import BytesIO from typing import Optional, List, Dict, Any, Tuple, NewType, Union +from osmocom.utils import * +from osmocom.tlv import bertlv_encode_tag, bertlv_encode_len

# Copyright (C) 2009-2010 Sylvain Munaut tnt@246tNt.com # Copyright (C) 2021 Harald Welte laforge@osmocom.org @@ -28,400 +30,6 @@ # along with this program. If not, see http://www.gnu.org/licenses/. #

-# just to differentiate strings of hex nibbles from everything else -Hexstr = NewType('Hexstr', str) -SwHexstr = NewType('SwHexstr', str) -SwMatchstr = NewType('SwMatchstr', str) -ResTuple = Tuple[Hexstr, SwHexstr] - -def h2b(s: Hexstr) -> bytearray: - """convert from a string of hex nibbles to a sequence of bytes""" - return bytearray.fromhex(s) - - -def b2h(b: bytearray) -> Hexstr: - """convert from a sequence of bytes to a string of hex nibbles""" - return ''.join(['%02x' % (x) for x in b]) - - -def h2i(s: Hexstr) -> List[int]: - """convert from a string of hex nibbles to a list of integers""" - return [(int(x, 16) << 4)+int(y, 16) for x, y in zip(s[0::2], s[1::2])] - - -def i2h(s: List[int]) -> Hexstr: - """convert from a list of integers to a string of hex nibbles""" - return ''.join(['%02x' % (x) for x in s]) - - -def h2s(s: Hexstr) -> str: - """convert from a string of hex nibbles to an ASCII string""" - return ''.join([chr((int(x, 16) << 4)+int(y, 16)) for x, y in zip(s[0::2], s[1::2]) - if int(x + y, 16) != 0xff]) - - -def s2h(s: str) -> Hexstr: - """convert from an ASCII string to a string of hex nibbles""" - b = bytearray() - b.extend(map(ord, s)) - return b2h(b) - - -def i2s(s: List[int]) -> str: - """convert from a list of integers to an ASCII string""" - return ''.join([chr(x) for x in s]) - - -def swap_nibbles(s: Hexstr) -> Hexstr: - """swap the nibbles in a hex string""" - return ''.join([x+y for x, y in zip(s[1::2], s[0::2])]) - - -def rpad(s: str, l: int, c='f') -> str: - """pad string on the right side. - Args: - s : string to pad - l : total length to pad to - c : padding character - Returns: - String 's' padded with as many 'c' as needed to reach total length of 'l' - """ - return s + c * (l - len(s)) - - -def lpad(s: str, l: int, c='f') -> str: - """pad string on the left side. - Args: - s : string to pad - l : total length to pad to - c : padding character - Returns: - String 's' padded with as many 'c' as needed to reach total length of 'l' - """ - return c * (l - len(s)) + s - - -def half_round_up(n: int) -> int: - return (n + 1)//2 - - -def str_sanitize(s: str) -> str: - """replace all non printable chars, line breaks and whitespaces, with ' ', make sure that - there are no whitespaces at the end and at the beginning of the string. - - Args: - s : string to sanitize - Returns: - filtered result of string 's' - """ - - chars_to_keep = string.digits + string.ascii_letters + string.punctuation - res = ''.join([c if c in chars_to_keep else ' ' for c in s]) - return res.strip() - -######################################################################### -# poor man's COMPREHENSION-TLV decoder. -######################################################################### - - -def comprehensiontlv_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]: - """Parse a single Tag according to ETSI TS 101 220 Section 7.1.1""" - if binary[0] in [0x00, 0x80, 0xff]: - raise ValueError("Found illegal value 0x%02x in %s" % - (binary[0], binary)) - if binary[0] == 0x7f: - # three-byte tag - tag = binary[0] << 16 | binary[1] << 8 | binary[2] - return (tag, binary[3:]) - elif binary[0] == 0xff: - return None, binary - else: - # single byte tag - tag = binary[0] - return (tag, binary[1:]) - - -def comprehensiontlv_parse_tag(binary: bytes) -> Tuple[dict, bytes]: - """Parse a single Tag according to ETSI TS 101 220 Section 7.1.1""" - if binary[0] in [0x00, 0x80, 0xff]: - raise ValueError("Found illegal value 0x%02x in %s" % - (binary[0], binary)) - if binary[0] == 0x7f: - # three-byte tag - tag = (binary[1] & 0x7f) << 8 - tag |= binary[2] - compr = bool(binary[1] & 0x80) - return ({'comprehension': compr, 'tag': tag}, binary[3:]) - else: - # single byte tag - tag = binary[0] & 0x7f - compr = bool(binary[0] & 0x80) - return ({'comprehension': compr, 'tag': tag}, binary[1:]) - - -def comprehensiontlv_encode_tag(tag) -> bytes: - """Encode a single Tag according to ETSI TS 101 220 Section 7.1.1""" - # permit caller to specify tag also as integer value - if isinstance(tag, int): - compr = bool(tag < 0xff and tag & 0x80) - tag = {'tag': tag, 'comprehension': compr} - compr = tag.get('comprehension', False) - if tag['tag'] in [0x00, 0x80, 0xff] or tag['tag'] > 0xff: - # 3-byte format - byte3 = tag['tag'] & 0xff - byte2 = (tag['tag'] >> 8) & 0x7f - if compr: - byte2 |= 0x80 - return b'\x7f' + byte2.to_bytes(1, 'big') + byte3.to_bytes(1, 'big') - else: - # 1-byte format - ret = tag['tag'] - if compr: - ret |= 0x80 - return ret.to_bytes(1, 'big') - -# length value coding is equal to BER-TLV - - -def comprehensiontlv_parse_one(binary: bytes) -> Tuple[dict, int, bytes, bytes]: - """Parse a single TLV IE at the start of the given binary data. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (tag:dict, len:int, remainder:bytes) - """ - (tagdict, remainder) = comprehensiontlv_parse_tag(binary) - (length, remainder) = bertlv_parse_len(remainder) - value = remainder[:length] - remainder = remainder[length:] - return (tagdict, length, value, remainder) - - -######################################################################### -# poor man's BER-TLV decoder. To be a more sophisticated OO library later -######################################################################### - -def bertlv_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]: - """Get a single raw Tag from start of input according to ITU-T X.690 8.1.2 - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (tag:int, remainder:bytes) - """ - # check for FF padding at the end, as customary in SIM card files - if binary[0] == 0xff and len(binary) == 1 or binary[0] == 0xff and binary[1] == 0xff: - return None, binary - tag = binary[0] & 0x1f - if tag <= 30: - return binary[0], binary[1:] - else: # multi-byte tag - tag = binary[0] - i = 1 - last = False - while not last: - last = not bool(binary[i] & 0x80) - tag <<= 8 - tag |= binary[i] - i += 1 - return tag, binary[i:] - - -def bertlv_parse_tag(binary: bytes) -> Tuple[dict, bytes]: - """Parse a single Tag value according to ITU-T X.690 8.1.2 - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of ({class:int, constructed:bool, tag:int}, remainder:bytes) - """ - cls = binary[0] >> 6 - constructed = bool(binary[0] & 0x20) - tag = binary[0] & 0x1f - if tag <= 30: - return ({'class': cls, 'constructed': constructed, 'tag': tag}, binary[1:]) - else: # multi-byte tag - tag = 0 - i = 1 - last = False - while not last: - last = not bool(binary[i] & 0x80) - tag <<= 7 - tag |= binary[i] & 0x7f - i += 1 - return ({'class': cls, 'constructed': constructed, 'tag': tag}, binary[i:]) - - -def bertlv_encode_tag(t) -> bytes: - """Encode a single Tag value according to ITU-T X.690 8.1.2 - """ - def get_top7_bits(inp: int) -> Tuple[int, int]: - """Get top 7 bits of integer. Returns those 7 bits as integer and the remaining LSBs.""" - remain_bits = inp.bit_length() - if remain_bits >= 7: - bitcnt = 7 - else: - bitcnt = remain_bits - outp = inp >> (remain_bits - bitcnt) - remainder = inp & ~ (inp << (remain_bits - bitcnt)) - return outp, remainder - - def count_int_bytes(inp: int) -> int: - """count the number of bytes require to represent the given integer.""" - i = 1 - inp = inp >> 8 - while inp: - i += 1 - inp = inp >> 8 - return i - - if isinstance(t, int): - # first convert to a dict representation - tag_size = count_int_bytes(t) - t, _remainder = bertlv_parse_tag(t.to_bytes(tag_size, 'big')) - tag = t['tag'] - constructed = t['constructed'] - cls = t['class'] - if tag <= 30: - t = tag & 0x1f - if constructed: - t |= 0x20 - t |= (cls & 3) << 6 - return bytes([t]) - else: # multi-byte tag - t = 0x1f - if constructed: - t |= 0x20 - t |= (cls & 3) << 6 - tag_bytes = bytes([t]) - remain = tag - while True: - t, remain = get_top7_bits(remain) - if remain: - t |= 0x80 - tag_bytes += bytes([t]) - if not remain: - break - return tag_bytes - - -def bertlv_parse_len(binary: bytes) -> Tuple[int, bytes]: - """Parse a single Length value according to ITU-T X.690 8.1.3; - only the definite form is supported here. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (length, remainder) - """ - if binary[0] < 0x80: - return (binary[0], binary[1:]) - else: - num_len_oct = binary[0] & 0x7f - length = 0 - if len(binary) < num_len_oct + 1: - return (0, b'') - for i in range(1, 1+num_len_oct): - length <<= 8 - length |= binary[i] - return (length, binary[1+num_len_oct:]) - - -def bertlv_encode_len(length: int) -> bytes: - """Encode a single Length value according to ITU-T X.690 8.1.3; - only the definite form is supported here. - Args: - length : length value to be encoded - Returns: - binary output data of BER-TLV length field - """ - if length < 0x80: - return length.to_bytes(1, 'big') - elif length <= 0xff: - return b'\x81' + length.to_bytes(1, 'big') - elif length <= 0xffff: - return b'\x82' + length.to_bytes(2, 'big') - elif length <= 0xffffff: - return b'\x83' + length.to_bytes(3, 'big') - elif length <= 0xffffffff: - return b'\x84' + length.to_bytes(4, 'big') - else: - raise ValueError("Length > 32bits not supported") - - -def bertlv_parse_one(binary: bytes) -> Tuple[dict, int, bytes, bytes]: - """Parse a single TLV IE at the start of the given binary data. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (tag:dict, len:int, remainder:bytes) - """ - (tagdict, remainder) = bertlv_parse_tag(binary) - (length, remainder) = bertlv_parse_len(remainder) - value = remainder[:length] - remainder = remainder[length:] - return (tagdict, length, value, remainder) - -def bertlv_parse_one_rawtag(binary: bytes) -> Tuple[int, int, bytes, bytes]: - """Parse a single TLV IE at the start of the given binary data; return tag as raw integer. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (tag:int, len:int, remainder:bytes) - """ - (tag, remainder) = bertlv_parse_tag_raw(binary) - (length, remainder) = bertlv_parse_len(remainder) - value = remainder[:length] - remainder = remainder[length:] - return (tag, length, value, remainder) - -def bertlv_return_one_rawtlv(binary: bytes) -> Tuple[int, int, bytes, bytes]: - """Return one single [encoded] TLV IE at the start of the given binary data. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (tag:int, len:int, tlv:bytes, remainder:bytes) - """ - (tag, remainder) = bertlv_parse_tag_raw(binary) - (length, remainder) = bertlv_parse_len(remainder) - tl_length = len(binary) - len(remainder) - value = binary[:tl_length] + remainder[:length] - remainder = remainder[length:] - return (tag, length, value, remainder) - -def dgi_parse_tag_raw(binary: bytes) -> Tuple[int, bytes]: - # In absence of any clear spec guidance we assume it's always 16 bit - return int.from_bytes(binary[:2], 'big'), binary[2:] - -def dgi_encode_tag(t: int) -> bytes: - return t.to_bytes(2, 'big') - -def dgi_encode_len(length: int) -> bytes: - """Encode a single Length value according to GlobalPlatform Systems Scripting Language - Specification v1.1.0 Annex B. - Args: - length : length value to be encoded - Returns: - binary output data of encoded length field - """ - if length < 255: - return length.to_bytes(1, 'big') - elif length <= 0xffff: - return b'\xff' + length.to_bytes(2, 'big') - else: - raise ValueError("Length > 32bits not supported") - -def dgi_parse_len(binary: bytes) -> Tuple[int, bytes]: - """Parse a single Length value according to GlobalPlatform Systems Scripting Language - Specification v1.1.0 Annex B. - Args: - binary : binary input data of BER-TLV length field - Returns: - Tuple of (length, remainder) - """ - if binary[0] == 255: - assert len(binary) >= 3 - return ((binary[1] << 8) | binary[2]), binary[3:] - else: - return binary[0], binary[1:] - # IMSI encoded format: # For IMSI 0123456789ABCDE: # @@ -437,6 +45,10 @@ # Because of this, an odd length IMSI fits exactly into len(imsi) + 1 // 2 bytes, whereas an # even length IMSI only uses half of the last byte.

+SwHexstr = NewType('SwHexstr', str) +SwMatchstr = NewType('SwMatchstr', str) +ResTuple = Tuple[Hexstr, SwHexstr] + def enc_imsi(imsi: str): """Converts a string IMSI into the encoded value of the EF""" l = half_round_up( @@ -809,29 +421,6 @@ return ('%02x' % bcd_len) + ('%02x' % npi_ton) + bcd + ("ff" * 2)

-def is_hex(string: str, minlen: int = 2, maxlen: Optional[int] = None) -> bool: - """ - Check if a string is a valid hexstring - """ - - # Filter obviously bad strings - if not string: - return False - if len(string) < minlen or minlen < 2: - return False - if len(string) % 2: - return False - if maxlen and len(string) > maxlen: - return False - - # Try actual encoding to be sure - try: - _try_encode = h2b(string) - return True - except: - return False - - def sanitize_pin_adm(pin_adm, pin_adm_hex=None) -> Hexstr: """ The ADM pin can be supplied either in its hexadecimal form or as @@ -963,26 +552,6 @@ return '\n'.join(table)

-def auto_int(x): - """Helper function for argparse to accept hexadecimal integers.""" - return int(x, 0) - -def _auto_uint(x, max_val: int): - """Helper function for argparse to accept hexadecimal or decimal integers.""" - ret = int(x, 0) - if ret < 0 or ret > max_val: - raise argparse.ArgumentTypeError('Number exceeds permited value range (0, %u)' % max_val) - return ret - -def auto_uint7(x): - return _auto_uint(x, 127) - -def auto_uint8(x): - return _auto_uint(x, 255) - -def auto_uint16(x): - return _auto_uint(x, 65535) - def expand_hex(hexstring, length): """Expand a given hexstring to a specified length by replacing "." or ".." with a filler that is derived from the neighboring nibbles respective @@ -1037,17 +606,6 @@ return hexstring

-class JsonEncoder(json.JSONEncoder): - """Extend the standard library JSONEncoder with support for more types.""" - - def default(self, o): - if isinstance(o, (BytesIO, bytes, bytearray)): - return b2h(o) - elif isinstance(o, datetime.datetime): - return o.isoformat() - return json.JSONEncoder.default(self, o) - - def boxed_heading_str(heading, width=80): """Generate a string that contains a boxed heading.""" # Auto-enlarge box if heading exceeds length @@ -1456,35 +1014,3 @@ if cla and not cmd.match_cla(cla): return None return cmd - - -def all_subclasses(cls) -> set: - """Recursively get all subclasses of a specified class""" - return set(cls.__subclasses__()).union([s for c in cls.__subclasses__() for s in all_subclasses(c)]) - -def is_hexstr_or_decimal(instr: str) -> str: - """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of - [hexa]decimal digits only.""" - if instr.isdecimal(): - return instr - if not all(c in string.hexdigits for c in instr): - raise ValueError('Input must be [hexa]decimal') - if len(instr) & 1: - raise ValueError('Input has un-even number of hex digits') - return instr - -def is_hexstr(instr: str) -> str: - """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of - an even sequence of hexadecimal digits only.""" - if not all(c in string.hexdigits for c in instr): - raise ValueError('Input must be hexadecimal') - if len(instr) & 1: - raise ValueError('Input has un-even number of hex digits') - return instr - -def is_decimal(instr: str) -> str: - """Method that can be used as 'type' in argparse.add_argument() to validate the value consists of - an even sequence of decimal digits only.""" - if not instr.isdecimal(): - raise ValueError('Input must decimal') - return instr diff --git a/requirements.txt b/requirements.txt index c12daf8..c6fdf9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ jsonpath-ng construct>=2.9.51 bidict -gsm0338 +osmocom pyyaml>=5.1 termcolor colorlog diff --git a/setup.py b/setup.py index 1a41d4f..d103461 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "jsonpath-ng", "construct >= 2.9.51", "bidict", - "gsm0338", + "osmocom", "pyyaml >= 5.1", "termcolor", "colorlog", diff --git a/tests/unittests/test_apdu.py b/tests/unittests/test_apdu.py index e09c8eb..b272005 100755 --- a/tests/unittests/test_apdu.py +++ b/tests/unittests/test_apdu.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3

import unittest -from pySim.utils import h2b, b2h -from pySim.construct import filter_dict +from osmocom.utils import h2b, b2h +from osmocom.construct import filter_dict + from pySim.apdu import Apdu from pySim.apdu.ts_31_102 import UsimAuthenticateEven

diff --git a/tests/unittests/test_construct.py b/tests/unittests/test_construct.py deleted file mode 100644 index 6251a14..0000000 --- a/tests/unittests/test_construct.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 - -import unittest -from pySim.utils import b2h, h2b -from pySim.construct import * -from construct import FlagsEnum - -tests = [ - ( b'\x80', 0x80 ), - ( b'\x80\x01', 0x8001 ), - ( b'\x80\x00\x01', 0x800001 ), - ( b'\x80\x23\x42\x01', 0x80234201 ), - ] - -class TestGreedyInt(unittest.TestCase): - def test_GreedyInt_decoder(self): - gi = GreedyInteger() - for t in tests: - self.assertEqual(gi.parse(t[0]), t[1]) - def test_GreedyInt_encoder(self): - gi = GreedyInteger() - for t in tests: - self.assertEqual(t[0], gi.build(t[1])) - pass - -class TestUtils(unittest.TestCase): - def test_filter_dict(self): - inp = {'foo': 0xf00, '_bar' : 0xba5, 'baz': 0xba2 } - out = {'foo': 0xf00, 'baz': 0xba2 } - self.assertEqual(filter_dict(inp), out) - - def test_filter_dict_nested(self): - inp = {'foo': 0xf00, 'nest': {'_bar' : 0xba5}, 'baz': 0xba2 } - out = {'foo': 0xf00, 'nest': {}, 'baz': 0xba2 } - self.assertEqual(filter_dict(inp), out) - - -class TestUcs2Adapter(unittest.TestCase): - # the three examples from TS 102 221 Annex A - EXAMPLE1 = b'\x80\x00\x30\x00\x31\x00\x32\x00\x33' - EXAMPLE2 = b'\x81\x05\x13\x53\x95\xa6\xa6\xff\xff' - EXAMPLE3 = b'\x82\x05\x05\x30\x2d\x82\xd3\x2d\x31' - ad = Ucs2Adapter(GreedyBytes) - - def test_example1_decode(self): - dec = self.ad._decode(self.EXAMPLE1, None, None) - self.assertEqual(dec, "0123") - - def test_example2_decode(self): - dec = self.ad._decode(self.EXAMPLE2, None, None) - self.assertEqual(dec, "S\u0995\u09a6\u09a6\u09ff") - - def test_example3_decode(self): - dec = self.ad._decode(self.EXAMPLE3, None, None) - self.assertEqual(dec, "-\u0532\u0583-1") - - testdata = [ - # variant 2 with only GSM alphabet characters - ( "mahlzeit", '8108006d61686c7a656974' ), - # variant 2 with mixed GSM alphabet + UCS2 - ( "mahlzeit\u099523", '810b136d61686c7a656974953233' ), - # variant 3 due to codepoint exceeding 8 bit - ( "mahl\u8023zeit", '820980236d61686c807a656974' ), - # variant 1 as there is no common codepoint pointer / prefix - ( "\u3000\u2000\u1000", '80300020001000' ), - ] - - def test_data_decode(self): - for string, encoded_hex in self.testdata: - encoded = h2b(encoded_hex) - dec = self.ad._decode(encoded, None, None) - self.assertEqual(dec, string) - - def test_data_encode(self): - for string, encoded_hex in self.testdata: - encoded = h2b(encoded_hex) - re_enc = self.ad._encode(string, None, None) - self.assertEqual(encoded, re_enc) - -class TestTrailerAdapter(unittest.TestCase): - Privileges = FlagsEnum(StripTrailerAdapter(GreedyBytes, 3), security_domain=0x800000, - dap_verification=0x400000, - delegated_management=0x200000, card_lock=0x100000, - card_terminate=0x080000, card_reset=0x040000, - cvm_management=0x020000, mandated_dap_verification=0x010000, - trusted_path=0x8000, authorized_management=0x4000, - token_management=0x2000, global_delete=0x1000, - global_lock=0x0800, global_registry=0x0400, - final_application=0x0200, global_service=0x0100, - receipt_generation=0x80, ciphered_load_file_data_block=0x40, - contactless_activation=0x20, contactless_self_activation=0x10) - examples = ['00', '80', '8040', '400010'] - def test_examples(self): - for e in self.examples: - dec = self.Privileges.parse(h2b(e)) - reenc = self.Privileges.build(dec) - self.assertEqual(e, b2h(reenc)) - - - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unittests/test_esim.py b/tests/unittests/test_esim.py index 81926e5..c88d3e3 100755 --- a/tests/unittests/test_esim.py +++ b/tests/unittests/test_esim.py @@ -18,8 +18,8 @@ import unittest import logging import base64 +from osmocom.utils import b2h, h2b

-from pySim.utils import b2h, h2b from pySim.esim.bsp import * import pySim.esim.rsp as rsp from pySim.esim import ActivationCode diff --git a/tests/unittests/test_esim_bsp.py b/tests/unittests/test_esim_bsp.py index d7b37b0..e8e8131 100755 --- a/tests/unittests/test_esim_bsp.py +++ b/tests/unittests/test_esim_bsp.py @@ -18,8 +18,8 @@ import unittest import logging import base64 +from osmocom.utils import b2h, h2b

-from pySim.utils import b2h, h2b from pySim.esim.bsp import * import pySim.esim.rsp as rsp

diff --git a/tests/unittests/test_esim_saip.py b/tests/unittests/test_esim_saip.py index dcde1b4..e7e324d 100755 --- a/tests/unittests/test_esim_saip.py +++ b/tests/unittests/test_esim_saip.py @@ -18,8 +18,8 @@ import unittest import logging import copy +from osmocom.utils import h2b, b2h

-from pySim.utils import h2b, b2h from pySim.esim.saip import * from pySim.esim.saip.personalization import * from pprint import pprint as pp diff --git a/tests/unittests/test_files.py b/tests/unittests/test_files.py index 9fdb022..2bd3aea 100755 --- a/tests/unittests/test_files.py +++ b/tests/unittests/test_files.py @@ -17,10 +17,9 @@

import unittest import logging +from osmocom.utils import *

-from pySim.utils import * from pySim.filesystem import * - import pySim.iso7816_4 import pySim.ts_102_221 import pySim.ts_102_222 diff --git a/tests/unittests/test_globalplatform.py b/tests/unittests/test_globalplatform.py index ba4ad76..c8fdda5 100644 --- a/tests/unittests/test_globalplatform.py +++ b/tests/unittests/test_globalplatform.py @@ -17,10 +17,10 @@

import unittest import logging +from osmocom.utils import b2h, h2b

from pySim.global_platform import * from pySim.global_platform.scp import * -from pySim.utils import b2h, h2b

KIC = h2b('100102030405060708090a0b0c0d0e0f') # enc KID = h2b('101102030405060708090a0b0c0d0e0f') # MAC diff --git a/tests/unittests/test_ota.py b/tests/unittests/test_ota.py index f75ba8d..888f828 100644 --- a/tests/unittests/test_ota.py +++ b/tests/unittests/test_ota.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3

import unittest -from pySim.utils import h2b, b2h +from osmocom.utils import h2b, b2h from pySim.sms import SMS_SUBMIT, SMS_DELIVER, AddressField from pySim.ota import *

diff --git a/tests/unittests/test_sms.py b/tests/unittests/test_sms.py index 7130399..d190528 100644 --- a/tests/unittests/test_sms.py +++ b/tests/unittests/test_sms.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3

import unittest -from pySim.utils import h2b, b2h +from osmocom.utils import h2b, b2h from pySim.sms import *

class Test_SMS_UDH(unittest.TestCase): diff --git a/tests/unittests/test_tlv.py b/tests/unittests/test_tlv.py deleted file mode 100644 index 3b05f19..0000000 --- a/tests/unittests/test_tlv.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python3 - -# (C) 2022 by Harald Welte laforge@osmocom.org -# All Rights Reserved -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see http://www.gnu.org/licenses/. - -import unittest -from construct import Int8ub, GreedyBytes -from pySim.tlv import * -from pySim.utils import h2b - -class TestUtils(unittest.TestCase): - def test_camel_to_snake(self): - cases = [ - ('CamelCase', 'camel_case'), - ('CamelCaseUPPER', 'camel_case_upper'), - ('Camel_CASE_underSCORE', 'camel_case_under_score'), - ] - for c in cases: - self.assertEqual(camel_to_snake(c[0]), c[1]) - - def test_flatten_dict_lists(self): - inp = [ - { 'first': 1 }, - { 'second': 2 }, - { 'third': 3 }, - ] - out = { 'first': 1, 'second':2, 'third': 3} - self.assertEqual(flatten_dict_lists(inp), out) - - def test_flatten_dict_lists_nodict(self): - inp = [ - { 'first': 1 }, - { 'second': 2 }, - { 'third': 3 }, - 4, - ] - self.assertEqual(flatten_dict_lists(inp), inp) - - def test_flatten_dict_lists_nested(self): - inp = {'top': [ - { 'first': 1 }, - { 'second': 2 }, - { 'third': 3 }, - ] } - out = {'top': { 'first': 1, 'second':2, 'third': 3 } } - self.assertEqual(flatten_dict_lists(inp), out) - -class TestTranscodable(unittest.TestCase): - class XC_constr_class(Transcodable): - _construct = Int8ub - def __init__(self): - super().__init__(); - - def test_XC_constr_class(self): - """Transcodable derived class with _construct class variable""" - xc = TestTranscodable.XC_constr_class() - self.assertEqual(xc.from_bytes(b'\x23'), 35) - self.assertEqual(xc.to_bytes(), b'\x23') - - class XC_constr_instance(Transcodable): - def __init__(self): - super().__init__(); - self._construct = Int8ub - - def test_XC_constr_instance(self): - """Transcodable derived class with _construct instance variable""" - xc = TestTranscodable.XC_constr_instance() - self.assertEqual(xc.from_bytes(b'\x23'), 35) - self.assertEqual(xc.to_bytes(), b'\x23') - - class XC_method_instance(Transcodable): - def __init__(self): - super().__init__(); - def _from_bytes(self, do): - return ('decoded', do) - def _to_bytes(self): - return self.decoded[1] - - def test_XC_method_instance(self): - """Transcodable derived class with _{from,to}_bytes() methods""" - xc = TestTranscodable.XC_method_instance() - self.assertEqual(xc.to_bytes(), b'') - self.assertEqual(xc.from_bytes(b''), None) - self.assertEqual(xc.from_bytes(b'\x23'), ('decoded', b'\x23')) - self.assertEqual(xc.to_bytes(), b'\x23') - -class TestIE(unittest.TestCase): - class MyIE(IE, tag=0x23, desc='My IE description'): - _construct = Int8ub - def to_ie(self): - return self.to_bytes() - - def test_IE_empty(self): - ie = TestIE.MyIE() - self.assertEqual(ie.to_dict(), {'my_ie': None}) - self.assertEqual(repr(ie), 'MyIE(None)') - self.assertEqual(ie.is_constructed(), False) - - def test_IE_from_bytes(self): - ie = TestIE.MyIE() - ie.from_bytes(b'\x42') - self.assertEqual(ie.to_dict(), {'my_ie': 66}) - self.assertEqual(repr(ie), 'MyIE(66)') - self.assertEqual(ie.is_constructed(), False) - self.assertEqual(ie.to_bytes(), b'\x42') - self.assertEqual(ie.to_ie(), b'\x42') - -class TestCompact(unittest.TestCase): - class IE_3(COMPACT_TLV_IE, tag=0x3): - _construct = GreedyBytes - class IE_7(COMPACT_TLV_IE, tag=0x7): - _construct = GreedyBytes - class IE_5(COMPACT_TLV_IE, tag=0x5): - _construct = GreedyBytes - # pylint: disable=undefined-variable - class IE_Coll(TLV_IE_Collection, nested=[IE_3, IE_7, IE_5]): - _construct = GreedyBytes - def test_ATR(self): - atr = h2b("31E073FE211F5745437531301365") - c = self.IE_Coll() - c.from_tlv(atr) - self.assertEqual(c.children[0].tag, 3) - self.assertEqual(c.children[0].to_bytes(), b'\xe0') - self.assertEqual(c.children[1].tag, 7) - self.assertEqual(c.children[1].to_bytes(), b'\xfe\x21\x1f') - self.assertEqual(c.children[2].tag, 5) - self.assertEqual(c.children[2].to_bytes(), b'\x45\x43\x75\x31\x30\x13\x65') - self.assertEqual(c.to_tlv(), atr) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unittests/test_tlvs.py b/tests/unittests/test_tlvs.py index 429b966..7f1b72b 100755 --- a/tests/unittests/test_tlvs.py +++ b/tests/unittests/test_tlvs.py @@ -18,8 +18,8 @@ import unittest import logging

-from pySim.utils import b2h, h2b, all_subclasses -from pySim.tlv import * +from osmocom.utils import b2h, h2b, all_subclasses +from osmocom.tlv import *

import pySim.iso7816_4 import pySim.ts_102_221 diff --git a/tests/unittests/test_utils.py b/tests/unittests/test_utils.py index 2c24e3e..0e61a61 100755 --- a/tests/unittests/test_utils.py +++ b/tests/unittests/test_utils.py @@ -172,70 +172,6 @@ msisdn_decoded = utils.dec_msisdn("ffffffffffffffffffffffffffffffffffffffff0bb121436587092143658709ffff") self.assertEqual(msisdn_decoded, (1, 3, "12345678901234567890"))

-class TestBerTlv(unittest.TestCase): - def test_BerTlvTagDec(self): - res = utils.bertlv_parse_tag(b'\x01') - self.assertEqual(res, ({'tag':1, 'constructed':False, 'class': 0}, b'')) - res = utils.bertlv_parse_tag(b'\x21') - self.assertEqual(res, ({'tag':1, 'constructed':True, 'class': 0}, b'')) - res = utils.bertlv_parse_tag(b'\x81\x23') - self.assertEqual(res, ({'tag':1, 'constructed':False, 'class': 2}, b'\x23')) - res = utils.bertlv_parse_tag(b'\x1f\x8f\x00\x23') - self.assertEqual(res, ({'tag':0xf<<7, 'constructed':False, 'class': 0}, b'\x23')) - - def test_BerTlvLenDec(self): - self.assertEqual(utils.bertlv_encode_len(1), b'\x01') - self.assertEqual(utils.bertlv_encode_len(127), b'\x7f') - self.assertEqual(utils.bertlv_encode_len(128), b'\x81\x80') - self.assertEqual(utils.bertlv_encode_len(0x123456), b'\x83\x12\x34\x56') - - def test_BerTlvLenEnc(self): - self.assertEqual(utils.bertlv_parse_len(b'\x01\x23'), (1, b'\x23')) - self.assertEqual(utils.bertlv_parse_len(b'\x7f'), (127, b'')) - self.assertEqual(utils.bertlv_parse_len(b'\x81\x80'), (128, b'')) - self.assertEqual(utils.bertlv_parse_len(b'\x83\x12\x34\x56\x78'), (0x123456, b'\x78')) - - def test_BerTlvParseOne(self): - res = utils.bertlv_parse_one(b'\x81\x01\x01'); - self.assertEqual(res, ({'tag':1, 'constructed':False, 'class':2}, 1, b'\x01', b'')) - -class TestComprTlv(unittest.TestCase): - def test_ComprTlvTagDec(self): - res = utils.comprehensiontlv_parse_tag(b'\x12\x23') - self.assertEqual(res, ({'tag': 0x12, 'comprehension': False}, b'\x23')) - res = utils.comprehensiontlv_parse_tag(b'\x92') - self.assertEqual(res, ({'tag': 0x12, 'comprehension': True}, b'')) - res = utils.comprehensiontlv_parse_tag(b'\x7f\x12\x34') - self.assertEqual(res, ({'tag': 0x1234, 'comprehension': False}, b'')) - res = utils.comprehensiontlv_parse_tag(b'\x7f\x82\x34\x56') - self.assertEqual(res, ({'tag': 0x234, 'comprehension': True}, b'\x56')) - - def test_ComprTlvTagEnc(self): - res = utils.comprehensiontlv_encode_tag(0x12) - self.assertEqual(res, b'\x12') - res = utils.comprehensiontlv_encode_tag({'tag': 0x12}) - self.assertEqual(res, b'\x12') - res = utils.comprehensiontlv_encode_tag({'tag': 0x12, 'comprehension':True}) - self.assertEqual(res, b'\x92') - res = utils.comprehensiontlv_encode_tag(0x1234) - self.assertEqual(res, b'\x7f\x12\x34') - res = utils.comprehensiontlv_encode_tag({'tag': 0x1234, 'comprehension':True}) - self.assertEqual(res, b'\x7f\x92\x34') - -class TestDgiTlv(unittest.TestCase): - def test_DgiTlvLenEnc(self): - self.assertEqual(utils.dgi_encode_len(10), b'\x0a') - self.assertEqual(utils.dgi_encode_len(254), b'\xfe') - self.assertEqual(utils.dgi_encode_len(255), b'\xff\x00\xff') - self.assertEqual(utils.dgi_encode_len(65535), b'\xff\xff\xff') - with self.assertRaises(ValueError): - utils.dgi_encode_len(65536) - - def testDgiTlvLenDec(self): - self.assertEqual(utils.dgi_parse_len(b'\x0a\x0b'), (10, b'\x0b')) - self.assertEqual(utils.dgi_parse_len(b'\xfe\x0b'), (254, b'\x0b')) - self.assertEqual(utils.dgi_parse_len(b'\xff\x00\xff\x0b'), (255, b'\x0b')) - class TestLuhn(unittest.TestCase): def test_verify(self): utils.verify_luhn('8988211000000530082')