[L] Change in onomondo-eim[master]: rest_handler: add API versions and schema validation
dexter has submitted this change. ( https://gerrit.osmocom.org/c/onomondo-eim/+/43089?usp=email )
Change subject: rest_handler: add API versions and schema validation ......................................................................
rest_handler: add API versions and schema validation
The JSON based REST API currently has no version checking and also does not use any JSON schema based validation yet. This patch adds both. An X-ADMIN-PROTOCOL field in the HTTP header informs about the API version (same style as the JSON interfaces defined in GSMA SGP.32 and SGP.22 would use). For incoming messages, the X-ADMIN-PROTOCOL field is checked to prevent outdated clients from using the REST API.
The incoming and outgoing JSON messages are now validated against JSON schema files. Those are the same JSON schema file which were only provided for documentation purposes only before. Now those files have been fixed up and moved to the priv directory, where they are now actively used.
Change-Id: I76e91e53a009cedf2cb1a0297db86e9e5fe7047a Related: SYS#8100 --- M contrib/restop.py M doc/rest_api.md R priv/rest_api_info_schema.json R priv/rest_api_resource_schema.json R priv/rest_api_response_schema.json M rebar.config M rebar.lock M src/onomondo_eim.app.src M src/onomondo_eim_app.erl M src/rest_handler.erl A src/rest_middleware.erl 11 files changed, 191 insertions(+), 63 deletions(-)
Approvals: dexter: Looks good to me, approved Jenkins Builder: Verified laforge: Looks good to me, but someone else must approve
diff --git a/contrib/restop.py b/contrib/restop.py index 47c1953..a12ccd4 100755 --- a/contrib/restop.py +++ b/contrib/restop.py @@ -14,24 +14,29 @@ DOWNLOAD_DEFAULT='{ "eidValue" : "89882119900000000000000000000005", "order" : {"activationCode" : "1$testsmdpplus1.example.com$OPxLD-UVRuC-jysPI-YkOwT"}}' PSMO_DEFAULT='{ "eidValue" : "89882119900000000000000000000005", "order" : [{"psmo" : "enable", "iccid" : "98001032547698103285", "rollback" : false }]}'
+req_headers = { + 'Content-Type': 'application/json', + 'X-Admin-Protocol': 'onomondo/eim/v1.0.0', +} + def h2b(s) -> bytearray: """convert from a string of hex nibbles to a sequence of bytes""" return bytes.fromhex(s)
def rest_create(host, facility, Json): - r = requests.post("http://" + str(host) + "/" + str(facility) + "/create", json=Json) + r = requests.post("http://" + str(host) + "/" + str(facility) + "/create", json=Json, headers=req_headers) return str(r.url)
def rest_lookup(host, facility, ResourceId): - r = requests.get("http://" + str(host) + "/" + str(facility) + "/lookup/" + str(ResourceId)) + r = requests.get("http://" + str(host) + "/" + str(facility) + "/lookup/" + str(ResourceId), headers=req_headers) return r.json()
def rest_delete(host, facility, ResourceId): - r = requests.get("http://" + str(host) + "/" + str(facility) + "/delete/" + str(ResourceId)) + r = requests.get("http://" + str(host) + "/" + str(facility) + "/delete/" + str(ResourceId), headers=req_headers) return r.json()
def rest_list(host, facility): - r = requests.get("http://" + str(host) + "/" + str(facility) + "/list/") + r = requests.get("http://" + str(host) + "/" + str(facility) + "/list/", headers=req_headers) return r.json()
def main(argv): diff --git a/doc/rest_api.md b/doc/rest_api.md index 59af59c..a3434dc 100644 --- a/doc/rest_api.md +++ b/doc/rest_api.md @@ -171,12 +171,22 @@ This description above describes only the basic concept of the REST API. To give a system integrator a detailed overview on how the requests/responses should look like three JSON schema files are shipped with onomondo-eim:
-* contrib/rest_api_resource_schema.json: This schema describes a REST resource. This is the format that the requests +* priv/rest_api_resource_schema.json: This schema describes a REST resource. This is the format that the requests issued to the REST API should have.
-* contrib/rest_api_response_schema.json: This schema that describes the JSON formatted response that is received from the +* priv/rest_api_response_schema.json: This schema that describes the JSON formatted response that is received from the REST API when the `lookup` operation is performed.
-* contrib/rest_api_info_schema.json: This schema that describes the JSON formatted info page that is received when the +* priv/rest_api_info_schema.json: This schema that describes the JSON formatted info page that is received when the REST API is called without any parameters.
+#### HTTP Header + +When sending a REST request, the following additional HTTP header fields have to be present. The `X-Admin-Protocol` +field must match the NSS of the URN found in the `$id` field of the JSON schema files. + +* Content-Type: application/json +* X-Admin-Protocol: onomondo/eim/v1.0.0 + +Note: Any REST response will also contain an `X-Admin-Protocol` field. Applications may query the `eIM Information` +(see above) to receive and check the current API version used on the REST API. \ No newline at end of file diff --git a/contrib/rest_api_info_schema.json b/priv/rest_api_info_schema.json similarity index 70% rename from contrib/rest_api_info_schema.json rename to priv/rest_api_info_schema.json index d808dd6..b73addc 100644 --- a/contrib/rest_api_info_schema.json +++ b/priv/rest_api_info_schema.json @@ -1,7 +1,7 @@ { - "$id": "https://onomondo.com/schemas/info", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "REST API info", + "$id": "urn:info:onomondo/eim/v1.0.0", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "onomondo-eim REST API info", "type": "object", "properties": { "hostname": { @@ -12,7 +12,7 @@ "description": "erlang node name of the onomondo-eim instance", "type": "string" }, - "version": { + "vsn": { "description": "version number of the onomondo-eim instance", "type": "string" }, @@ -26,7 +26,7 @@ }, "esipaPort": { "description": "IP-Port where the ESipa interface is bound to", - "type": "string" + "type": "integer" }, "esipaSslCert": { "description": "SSL certificate that is used for the ESipa interface (HTTPS)", @@ -46,16 +46,25 @@ "addInitialEimRequest": { "description": "ASN.1 encoded eIM configuration (see also GSMA SGP.32)", "type": "string", - "pattern": "^[0-9,A-F]{2,32}$" + "pattern": "^[0-9A-F].*$" }, "eimConfigurationData": { "description": "ASN.1 encoded eIM configuration (see also GSMA SGP.32)", "type": "string", - "pattern": "^[0-9,A-F]{2,32}$" + "pattern": "^[0-9A-F].*$" }, "counterValue": { "description": "Initial counter value for signature generation (see also GSMA SGP.32)", "type": "integer" + }, + "consumerEuicc": { + "description": "Tells if this eIM treads new eUICCs as consumer eUICCs by default", + "type": "boolean" } - } + }, + "required": [ + "hostname", "node", "vsn", "eimId", "esipaIp", "esipaPort", "esipaSslCert", "eimCert", "rootCiCerts", + "addInitialEimRequest", "eimConfigurationData", "counterValue", "consumerEuicc" + ], + "additionalProperties": false } diff --git a/contrib/rest_api_resource_schema.json b/priv/rest_api_resource_schema.json similarity index 90% rename from contrib/rest_api_resource_schema.json rename to priv/rest_api_resource_schema.json index 8a9e91d..c66bbaa 100644 --- a/contrib/rest_api_resource_schema.json +++ b/priv/rest_api_resource_schema.json @@ -1,7 +1,7 @@ { - "$id": "https://onomondo.com/schemas/resource", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "REST API resource", + "$id": "urn:resource:onomondo/eim/v1.0.0", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "onomondo-eim REST API resource", "type": "object", "properties": { "eidValue": { @@ -25,7 +25,8 @@ }, "required": [ "activationCode" - ] + ], + "additionalProperties": false } }, { @@ -53,7 +54,8 @@ }, "required": [ "iccid" - ] + ], + "additionalProperties": false } }, { @@ -69,7 +71,8 @@ }, "required": [ "iccid" - ] + ], + "additionalProperties": false } }, { @@ -85,7 +88,8 @@ }, "required": [ "iccid" - ] + ], + "additionalProperties": false } }, { @@ -125,14 +129,16 @@ }, "required": [ "searchCriteria" - ] + ], + "additionalProperties": false } }, { "getRAT": { "description": "order the execution of a getRAT PSMO, see also GSMA SGP.32, section 5.9.13", "type": "object", - "properties": {} + "properties": {}, + "additionalProperties": false } }, { @@ -155,7 +161,8 @@ }, "required": [ "immediateEnableFlag" - ] + ], + "additionalProperties": false } }, { @@ -171,7 +178,8 @@ }, "required": [ "iccid" - ] + ], + "additionalProperties": false } }, { @@ -193,12 +201,14 @@ }, "required": [ "defaultDpAddress" - ] + ], + "additionalProperties": false } } ] } - } + }, + "additionalProperties": false } }, { @@ -222,7 +232,8 @@ }, "required": [ "eimConfigurationData" - ] + ], + "additionalProperties": false } }, { @@ -237,7 +248,8 @@ }, "required": [ "eimId" - ] + ], + "additionalProperties": false } }, { @@ -253,7 +265,8 @@ }, "required": [ "eimConfigurationData" - ] + ], + "additionalProperties": false } }, { @@ -264,7 +277,8 @@ } } ] - } + }, + "additionalProperties": false } } }, @@ -281,7 +295,8 @@ }, "required": [ "tagList" - ] + ], + "additionalProperties": false } }, { @@ -328,10 +343,12 @@ "enum": ["counterValue", "consumerEuicc", "associationToken", "signAlgo", "signPubKey"] } } - } + }, + "additionalProperties": false } } ] } - } + }, + "additionalProperties": false } diff --git a/contrib/rest_api_response_schema.json b/priv/rest_api_response_schema.json similarity index 95% rename from contrib/rest_api_response_schema.json rename to priv/rest_api_response_schema.json index e0bc6ed..0f532d3 100644 --- a/contrib/rest_api_response_schema.json +++ b/priv/rest_api_response_schema.json @@ -1,7 +1,7 @@ { - "$id": "https://onomondo.com/schemas/response", - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "REST API response", + "$id": "urn:response:onomondo/eim/v1.0.0", + "$schema": "http://json-schema.org/draft-06/schema#", + "title": "onomondo-eim REST API response", "type": "object", "properties": { "status": { @@ -20,7 +20,7 @@ "type": "string" }, "resource": { - "$ref": "/schemas/resource" + "$ref": "urn:resource:onomondo/eim/v1.0.0" }, "outcome": { "description": "specific result if the executed order", @@ -141,7 +141,8 @@ }, "required": [ "finalResult" - ] + ], + "additionalProperties": false } }, { @@ -185,7 +186,8 @@ }, "required": [ "mccMnc" - ] + ], + "additionalProperties": false }, "consentRequired": { "description": "indicates that the End User consent is required", @@ -198,7 +200,8 @@ "ppr2", "allowedOperators", "consentRequired" - ] + ], + "additionalProperties": false } } }, @@ -277,7 +280,8 @@ }, "required": [ "addEimResultCode" - ] + ], + "additionalProperties": false } }, { @@ -345,7 +349,8 @@ }, "required": [ "finalResult" - ] + ], + "additionalProperties": false } }, { @@ -409,7 +414,8 @@ }, "required": [ "finalResult" - ] + ], + "additionalProperties": false } }, { @@ -511,12 +517,14 @@ "type": "string", "pattern": "^[0-9,A-F]{2,32}$" } - } + }, + "additionalProperties": false } }, "required": [ "edrResult" - ] + ], + "additionalProperties": false } }, { @@ -546,15 +554,16 @@ } ] } - }, - "debuginfo": { - "description": "Erlang ETS encoded debug information string, for debugging/diagnosis only", - "type": "string", - "pattern": "^[0-9,A-F]{2,32}$" } + }, + "debuginfo": { + "description": "Erlang ETS encoded debug information string, for debugging/diagnosis only", + "type": "string", + "pattern": "^[0-9,A-F].*$" } }, "required": [ "status" - ] + ], + "additionalProperties": false } diff --git a/rebar.config b/rebar.config index 8108230..00ed4d9 100644 --- a/rebar.config +++ b/rebar.config @@ -10,7 +10,8 @@ {jiffy, "1.1.2"}, {quickrand, "2.0.7"}, {uuid, "2.0.7", {pkg, uuid_erl}}, - {jsx, "~> 3.1"} + {jsx, "~> 3.1"}, + {jesse, "1.8.2"} ]}.
{project_plugins, [rebar3_asn1_compiler, erlfmt]}. diff --git a/rebar.lock b/rebar.lock index 5216b5c..9b29a63 100644 --- a/rebar.lock +++ b/rebar.lock @@ -4,6 +4,7 @@ {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.12.1">>},1}, {<<"hackney">>,{pkg,<<"hackney">>,<<"1.20.1">>},0}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},1}, + {<<"jesse">>,{pkg,<<"jesse">>,<<"1.8.2">>},0}, {<<"jiffy">>,{pkg,<<"jiffy">>,<<"1.1.2">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},1}, @@ -21,6 +22,7 @@ {<<"cowlib">>, <<"A9FA9A625F1D2025FE6B462CB865881329B5CAFF8F1854D1CBC9F9533F00E1E1">>}, {<<"hackney">>, <<"8D97AEC62DDDDD757D128BFD1DF6C5861093419F8F7A4223823537BAD5D064E2">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, + {<<"jesse">>, <<"6AEFE03B0275CC973D258FEA8EB1D5C27943C73199E6AF257A1BB3C763F552D3">>}, {<<"jiffy">>, <<"A9B6C9A7EC268E7CF493D028F0A4C9144F59CCB878B1AFE42841597800840A1B">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, @@ -37,6 +39,7 @@ {<<"cowlib">>, <<"163B73F6367A7341B33C794C4E88E7DBFE6498AC42DCD69EF44C5BC5507C8DB0">>}, {<<"hackney">>, <<"FE9094E5F1A2A2C0A7D10918FEE36BFEC0EC2A979994CFF8CFE8058CD9AF38E3">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, + {<<"jesse">>, <<"46C9C46F673F07D9DF4158D1ACF62B0D0188DB35E8996833075E28B4FE639BAD">>}, {<<"jiffy">>, <<"BB61BC42A720BBD33CB09A410E48BB79A61012C74CB8B3E75F26D988485CF381">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, diff --git a/src/onomondo_eim.app.src b/src/onomondo_eim.app.src index 049fce0..ffeb39e 100644 --- a/src/onomondo_eim.app.src +++ b/src/onomondo_eim.app.src @@ -15,7 +15,8 @@ hackney, jiffy, quickrand, - uuid + uuid, + jesse ]}, {env, []} ]}. diff --git a/src/onomondo_eim_app.erl b/src/onomondo_eim_app.erl index 20d07a5..89ffd4a 100644 --- a/src/onomondo_eim_app.erl +++ b/src/onomondo_eim_app.erl @@ -119,6 +119,7 @@ ),
% Startup REST server + rest_handler:load_json_schema(), Dispatch_REST = rest_dispatch(), {ok, RestIp} = application:get_env(onomondo_eim, rest_ip), {ok, RestPort} = application:get_env(onomondo_eim, rest_port), @@ -129,7 +130,7 @@ none, none, Dispatch_REST, - [cowboy_router, cowboy_handler], + [cowboy_router, rest_middleware, cowboy_handler], "REST" ),
diff --git a/src/rest_handler.erl b/src/rest_handler.erl index 6718986..1497456 100644 --- a/src/rest_handler.erl +++ b/src/rest_handler.erl @@ -8,6 +8,7 @@
% Functions we provide to the Cowboy REST handler -export([ + load_json_schema/0, init/2, allowed_methods/2, content_types_provided/2, @@ -18,6 +19,34 @@
-record(state, {op}).
+% Load a single JSON schema from file. +load_json_schema_one(JsonSchemaName) -> + {ok, JsonSchemaPath} = utils:get_priv_file_path(onomondo_eim, JsonSchemaName ++ ".json"), + logger:notice("Loading JSON schema from file: ~p~n", [JsonSchemaPath]), + {ok, JsonSchema} = file:read_file(JsonSchemaPath), + JsonSchemaDecoded = jiffy:decode(JsonSchema), + jesse:add_schema(JsonSchemaName, JsonSchemaDecoded). + +% Initial load of the JSON schema files (called once on startup). +load_json_schema() -> + ok = load_json_schema_one("rest_api_resource_schema"), + ok = load_json_schema_one("rest_api_response_schema"), + ok = load_json_schema_one("rest_api_info_schema"). + +% Decode and validate given encoded JSON data against a given JSON schema. +decode_and_validate_json(JsonData, JsonSchemaName) -> + JsonDataDecoded = jiffy:decode(JsonData), + case jesse:validate(JsonSchemaName, JsonDataDecoded) of + {ok, JsonDataDecodedValidated} -> + {ok, JsonDataDecodedValidated}; + {error, ValidationErrorCause} -> + logger:error( + "JSON validation failed:~nJsonSchemaName=~p~nJsonDataDecoded=~p~nValidationErrorCause=~p~n", + [JsonSchemaName, JsonDataDecoded, ValidationErrorCause] + ), + error + end. + % Initiaize state and extract the operation (create, lookup, update, list) init(Req, Options) -> State = #state{op = Options}, @@ -61,14 +90,14 @@
% Create a new resource and return its identifier post_rest_create(Req, State, Facility) -> - {ok, [{Content, true}], Req1} = cowboy_req:read_urlencoded_body(Req), + {ok, [{JsonData, true}], Req1} = cowboy_req:read_urlencoded_body(Req), logger:info( "REST: creating new REST resource,~nPeer=~p, Resource=~p, Facility=~p", - [maps:get(peer, Req), Content, Facility] + [maps:get(peer, Req), JsonData, Facility] ), - ContentDecoded = jiffy:decode(Content), - {[{<<"eidValue">>, EidValue}, _]} = ContentDecoded, - {[_, {<<"order">>, Order}]} = ContentDecoded, + {ok, JsonDataDecodedValidated} = decode_and_validate_json(JsonData, "rest_api_resource_schema"), + {[{<<"eidValue">>, EidValue}, _]} = JsonDataDecodedValidated, + {[_, {<<"order">>, Order}]} = JsonDataDecodedValidated, {ok, ResourceId} = mnesia_db_rest:create(Facility, EidValue, Order), case cowboy_req:method(Req1) of <<"POST">> -> @@ -111,6 +140,7 @@ {[{status, error}]} end, ResponseJson = jiffy:encode(Response), + {ok, _} = decode_and_validate_json(ResponseJson, "rest_api_response_schema"), logger:info( "REST: responding to client,~nPeer=~p, ResourceId=~p, Response:~p~n", [maps:get(peer, Req), ResourceId, ResponseJson] @@ -135,6 +165,7 @@ {[{status, error}]} end, ResponseJson = jiffy:encode(Response), + {ok, _} = decode_and_validate_json(ResponseJson, "rest_api_response_schema"), logger:info( "REST: responding to client,~nPeer=~p, ResourceId=~p, Response:~p~n", [maps:get(peer, Req), ResourceId, ResponseJson] @@ -197,8 +228,10 @@ {counterValue, CounterValue}, {consumerEuicc, ConsumerEuicc} ]}, - InfoListJson = utils:join_binary_list(jiffy:encode(InfoList)), - Response = io_lib:format("~s", [binary_to_list(InfoListJson)]), + ResponseJson = utils:join_binary_list(jiffy:encode(InfoList)), + {ok, _} = decode_and_validate_json(ResponseJson, "rest_api_info_schema"), + + Response = io_lib:format("~s", [binary_to_list(ResponseJson)]), logger:info( "REST: responding to client,~nPeer=~p, Response=~p~n", [maps:get(peer, Req), list_to_binary(Response)] diff --git a/src/rest_middleware.erl b/src/rest_middleware.erl new file mode 100644 index 0000000..f0d73bf --- /dev/null +++ b/src/rest_middleware.erl @@ -0,0 +1,39 @@ +% Copyright (c) 2025 Onomondo ApS & sysmocom - s.f.m.c. GmbH. All rights reserved. +% +% SPDX-License-Identifier: AGPL-3.0-only +% +% Author: Harald Welte hwelte@sysmocom.de / sysmocom - s.f.m.c. GmbH + +-module(rest_middleware). +-behaviour(cowboy_middleware). + +-export([execute/2]). + +% Derive the X-ADMIN-PROTOCOL string from the URN contained in the $id field of the related JSON schema. +protocol_from_json_schema(JsonSchemaName) -> + JsonSchema = jesse_database:load(JsonSchemaName), + {[{<<"$id">>, JsonSchemaId} | _]} = JsonSchema, + JsonSchemaIdUrnPath = maps:get(path, uri_string:parse(JsonSchemaId)), + [_, Protocol] = string:split(JsonSchemaIdUrnPath, ":"), + Protocol. + +% Cowboy middleware callback, make sure that the X-ADMIN-PROTOCOL field contains the expected protocol +% identifier for incoming POST requests. For outgoing requests, set the X-ADMIN-PROTOCOL. Both strings +% are derived from the related JSON schema. +execute(Req0 = #{method := <<"POST">>}, State) -> + ExpectedProtocol = protocol_from_json_schema("rest_api_resource_schema"), + case cowboy_req:header(<<"x-admin-protocol">>, Req0) of + ExpectedProtocol -> + {ok, Req0, State}; + Protocol -> + logger:error( + "Rejecting incompatible REST request: ~s,~nProtocol=~p, Peer=~p, Pid=~p~n", + [maps:get(path, Req0), Protocol, maps:get(peer, Req0), maps:get(pid, Req0)] + ), + Req = cowboy_req:reply(400, #{}, <<"Unsupported x-admin-protocol">>, Req0), + {stop, Req} + end; +execute(Req0, Env) -> + Protocol = protocol_from_json_schema("rest_api_response_schema"), + Req1 = cowboy_req:set_resp_header(<<"x-admin-protocol">>, Protocol, Req0), + {ok, Req1, Env}.
-
dexter