[MERGED] osmo-hlr[master]: add osmo-hlr-db-tool, program to migrate from osmo-nitb db

Harald Welte gerrit-no-reply at lists.osmocom.org
Sat Oct 28 18:36:55 UTC 2017


Harald Welte has submitted this change and it was merged.

Change subject: add osmo-hlr-db-tool, program to migrate from osmo-nitb db
......................................................................


add osmo-hlr-db-tool, program to migrate from osmo-nitb db

Move macro copy_sqlite3_text_to_buf() to db.h, so it can be used in
hlr_db_tool.c.

Add _dbd_decode_binary() from libdbi to avoid depending on the entire libdbi
just for KI BLOB decoding. Add it in a separate file, copying its own license,
the lGPL.

Offer commandline option "import-nitb-db" to read in an old osmo-nitb database
and copy subscriber IMSIs and 2G auth data to OsmoHLR db format.

Anticipate future command line options like "import-csv", so keep the code
generalized.

Change-Id: I0dfa6ec033dd93161c1adc2ce1637195fe5b7a63
---
M src/Makefile.am
M src/db.h
M src/db_hlr.c
A src/dbd_decode_binary.c
A src/hlr_db_tool.c
5 files changed, 495 insertions(+), 11 deletions(-)

Approvals:
  Harald Welte: Looks good to me, approved
  Jenkins Builder: Verified



diff --git a/src/Makefile.am b/src/Makefile.am
index 3b09b7b..9fbb062 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -35,6 +35,7 @@
 
 bin_PROGRAMS = \
 	osmo-hlr \
+	osmo-hlr-db-tool \
 	$(NULL)
 
 noinst_PROGRAMS = \
@@ -66,6 +67,21 @@
 	$(SQLITE3_LIBS) \
 	$(NULL)
 
+osmo_hlr_db_tool_SOURCES = \
+	hlr_db_tool.c \
+	db.c \
+	db_hlr.c \
+	logging.c \
+	rand_urandom.c \
+	dbd_decode_binary.c \
+	$(NULL)
+
+osmo_hlr_db_tool_LDADD = \
+	$(LIBOSMOCORE_LIBS) \
+	$(LIBOSMOGSM_LIBS) \
+	$(SQLITE3_LIBS) \
+	$(NULL)
+
 db_test_SOURCES = \
 	auc.c \
 	db.c \
diff --git a/src/db.h b/src/db.h
index 35e4327..fc8e511 100644
--- a/src/db.h
+++ b/src/db.h
@@ -129,3 +129,14 @@
 		    bool purge_val, bool is_ps);
 
 int hlr_subscr_nam(struct hlr *hlr, struct hlr_subscriber *subscr, bool nam_val, bool is_ps);
+
+/*! Call sqlite3_column_text() and copy result to a char[].
+ * \param[out] buf  A char[] used as sizeof() arg(!) and osmo_strlcpy() target.
+ * \param[in] stmt  An sqlite3_stmt*.
+ * \param[in] idx   Index in stmt's returned columns.
+ */
+#define copy_sqlite3_text_to_buf(buf, stmt, idx) \
+	do { \
+		const char *_txt = (const char *) sqlite3_column_text(stmt, idx); \
+		osmo_strlcpy(buf, _txt, sizeof(buf)); \
+	} while (0)
diff --git a/src/db_hlr.c b/src/db_hlr.c
index bae9a5a..c4d4974 100644
--- a/src/db_hlr.c
+++ b/src/db_hlr.c
@@ -35,17 +35,6 @@
 
 #define LOGHLR(imsi, level, fmt, args ...)	LOGP(DAUC, level, "IMSI='%s': " fmt, imsi, ## args)
 
-/*! Call sqlite3_column_text() and copy result to a char[].
- * \param[out] buf  A char[] used as sizeof() arg(!) and osmo_strlcpy() target.
- * \param[in] stmt  An sqlite3_stmt*.
- * \param[in] idx   Index in stmt's returned columns.
- */
-#define copy_sqlite3_text_to_buf(buf, stmt, idx) \
-	do { \
-		const char *_txt = (const char *) sqlite3_column_text(stmt, idx); \
-		osmo_strlcpy(buf, _txt, sizeof(buf)); \
-	} while (0)
-
 /*! Add new subscriber record to the HLR database.
  * \param[in,out] dbc  database context.
  * \param[in] imsi  ASCII string of IMSI digits, is validated.
diff --git a/src/dbd_decode_binary.c b/src/dbd_decode_binary.c
new file mode 100644
index 0000000..e1a98ad
--- /dev/null
+++ b/src/dbd_decode_binary.c
@@ -0,0 +1,42 @@
+/* This function is blatantly copied from libdbi, from
+ * https://sourceforge.net/p/libdbi/libdbi/ci/master/tree/src/dbd_helper.c
+ * to save having to depend on the entire libdbi just for KI BLOB decoding.
+ */
+
+/*
+ * libdbi - database independent abstraction layer for C.
+ * Copyright (C) 2001-2003, David Parker and Mark Tobenkin.
+ * http://libdbi.sourceforge.net
+ * 
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ * 
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * 
+ * $Id: dbd_helper.c,v 1.44 2011/08/09 11:14:14 mhoenicka Exp $
+ */
+
+#include <sys/types.h>
+
+size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out){
+  int i, e;
+  unsigned char c;
+  e = *(in++);
+  i = 0;
+  while( (c = *(in++))!=0 ){
+    if( c==1 ){
+      c = *(in++) - 1;
+    }
+    out[i++] = c + e;
+  }
+  return (size_t)i;
+}
diff --git a/src/hlr_db_tool.c b/src/hlr_db_tool.c
new file mode 100644
index 0000000..8982739
--- /dev/null
+++ b/src/hlr_db_tool.c
@@ -0,0 +1,426 @@
+/* (C) 2017 by sysmocom - s.f.m.c. GmbH <info at sysmocom.de>
+ *
+ * All Rights Reserved
+ *
+ * Author: Neels Hofmeyr <nhofmeyr at sysmocom.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <stdlib.h>
+#include <signal.h>
+#include <stdio.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <string.h>
+
+#include <osmocom/core/logging.h>
+#include <osmocom/core/application.h>
+
+#include "logging.h"
+#include "db.h"
+#include "rand.h"
+
+struct hlr_db_tool_ctx {
+	/* DB context */
+	struct db_context *dbc;
+};
+
+struct hlr_db_tool_ctx *g_hlr_db_tool_ctx;
+
+static struct {
+	const char *db_file;
+	bool bootstrap;
+	const char *import_nitb_db;
+} cmdline_opts = {
+	.db_file = "hlr.db",
+};
+
+static void print_help()
+{
+	printf("\n");
+	printf("Usage: osmo-hlr-db-tool [-l <hlr.db>] import-nitb-db <nitb.db>]\n");
+	printf("Call without arguments to create a new empty ./hlr.db.\n");
+	printf("  -l --database db-name      The OsmoHLR database to use, default '%s'.\n",
+	       cmdline_opts.db_file);
+	printf("  -h --help                  This text.\n");
+	printf("  -d option --debug=DMAIN:DDB:DAUC  Enable debugging.\n");
+	printf("  -s --disable-color         Do not print ANSI colors in the log\n");
+	printf("  -T --timestamp             Prefix every log line with a timestamp.\n");
+	printf("  -e --log-level number      Set a global loglevel.\n");
+	printf("  -V --version               Print the version of OsmoHLR-db-tool.\n");
+	printf("\n");
+	printf("  import-nitb-db db          Add OsmoNITB db's subscribers to OsmoHLR db.\n");
+	printf("                             Be aware that the import is lossy, only the\n");
+	printf("                             IMSI, MSISDN, nam_cs/ps and 2G auth data are set.\n");
+}
+
+static void print_version(int print_copyright)
+{
+	printf("OsmoHLR-db-tool version %s\n", PACKAGE_VERSION);
+	if (print_copyright)
+		printf("\n"
+       "Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH\n"
+       "License AGPLv3+: GNU AGPL version 3 or later <http://gnu.org/licenses/agpl-3.0.html>\n"
+       "This is free software: you are free to change and redistribute it.\n"
+       "There is NO WARRANTY, to the extent permitted by law.\n"
+       "\n");
+}
+
+static void handle_options(int argc, char **argv)
+{
+	const char *cmd;
+
+	while (1) {
+		int option_index = 0, c;
+		static struct option long_options[] = {
+			{"help", 0, 0, 'h'},
+			{"database", 1, 0, 'l'},
+			{"debug", 1, 0, 'd'},
+			{"disable-color", 0, 0, 's'},
+			{"timestamp", 0, 0, 'T'},
+			{"log-level", 1, 0, 'e'},
+			{"version", 0, 0, 'V' },
+			{0, 0, 0, 0}
+		};
+
+		c = getopt_long(argc, argv, "hl:d:sTe:V",
+				long_options, &option_index);
+		if (c == -1)
+			break;
+
+		switch (c) {
+		case 'h':
+			print_help();
+			exit(EXIT_SUCCESS);
+		case 'l':
+			cmdline_opts.db_file = optarg;
+			break;
+		case 'd':
+			log_parse_category_mask(osmo_stderr_target, optarg);
+			break;
+		case 's':
+			log_set_use_color(osmo_stderr_target, 0);
+			break;
+		case 'T':
+			log_set_print_timestamp(osmo_stderr_target, 1);
+			break;
+		case 'e':
+			log_set_log_level(osmo_stderr_target, atoi(optarg));
+			break;
+		case 'V':
+			print_version(1);
+			exit(EXIT_SUCCESS);
+			break;
+		default:
+			/* catch unknown options *as well as* missing arguments. */
+			fprintf(stderr, "Error in command line options. Exiting.\n");
+			exit(EXIT_FAILURE);
+			break;
+		}
+	}
+
+	if (argc - optind <= 0) {
+		fprintf(stderr, "Error: You must specify a command.\n");
+		print_help();
+		exit(EXIT_FAILURE);
+	}
+
+	cmd = argv[optind++];
+	printf("command '%s', %d extra arguments\n", cmd, argc - optind);
+
+	if (!strcmp(cmd, "import-nitb-db")) {
+		if (argc - optind < 1) {
+			fprintf(stderr, "You must specify an input db file\n");
+			print_help();
+			exit(EXIT_FAILURE);
+		}
+		cmdline_opts.import_nitb_db = argv[optind++];
+	} else {
+		fprintf(stderr, "Error: Unknown command `%s'\n", cmd);
+		print_help();
+		exit(EXIT_FAILURE);
+	}
+}
+
+static void signal_hdlr(int signal)
+{
+	switch (signal) {
+	case SIGINT:
+		LOGP(DMAIN, LOGL_NOTICE, "Terminating due to SIGINT\n");
+		db_close(g_hlr_db_tool_ctx->dbc);
+		log_fini();
+		talloc_report_full(g_hlr_db_tool_ctx, stderr);
+		exit(EXIT_SUCCESS);
+		break;
+	case SIGUSR1:
+		LOGP(DMAIN, LOGL_DEBUG, "Talloc Report due to SIGUSR1\n");
+		talloc_report_full(g_hlr_db_tool_ctx, stderr);
+		break;
+	}
+}
+
+sqlite3 *open_nitb_db(const char *filename)
+{
+	int rc;
+	sqlite3 *nitb_db = NULL;
+
+	rc = sqlite3_open(filename, &nitb_db);
+	if (rc != SQLITE_OK) {
+		LOGP(DDB, LOGL_ERROR, "Unable to open OsmoNITB DB %s; rc = %d\n", filename, rc);
+		return NULL;
+	}
+
+	return nitb_db;
+}
+
+enum nitb_stmt {
+	NITB_SELECT_SUBSCR,
+	NITB_SELECT_AUTH_KEYS,
+};
+
+static const char *nitb_stmt_sql[] = {
+	[NITB_SELECT_SUBSCR] =
+		"SELECT imsi, id, extension, authorized"
+		" FROM Subscriber"
+		" ORDER BY id",
+	[NITB_SELECT_AUTH_KEYS] =
+		"SELECT algorithm_id, a3a8_ki from authkeys"
+		" WHERE subscriber_id = $subscr_id",
+};
+
+sqlite3_stmt *nitb_stmt[ARRAY_SIZE(nitb_stmt_sql)] = {};
+
+size_t _dbd_decode_binary(const unsigned char *in, unsigned char *out);
+
+void import_nitb_subscr_aud(sqlite3 *nitb_db, const char *imsi, int64_t nitb_id, int64_t hlr_id)
+{
+	int rc;
+	struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
+	sqlite3_stmt *stmt;
+
+	int count = 0;
+
+	stmt = nitb_stmt[NITB_SELECT_AUTH_KEYS];
+	if (!db_bind_int(stmt, NULL, nitb_id))
+		return;
+
+	while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
+		const void *blob;
+		unsigned int blob_size;
+		static unsigned char buf[4096];
+		static char ki[128];
+		int decoded_size;
+		struct sub_auth_data_str aud2g = {
+			.type = OSMO_AUTH_TYPE_GSM,
+			.algo = OSMO_AUTH_ALG_NONE,
+			.u.gsm.ki = ki,
+		};
+
+		aud2g.algo = sqlite3_column_int(stmt, 0);
+
+		if (count) {
+			LOGP(DDB, LOGL_ERROR,
+			     "Warning: subscriber has more than one auth key,"
+			     " importing only the first key, for IMSI=%s\n",
+			     imsi);
+			break;
+		}
+
+		blob = sqlite3_column_blob(stmt, 1);
+		blob_size = sqlite3_column_bytes(stmt, 1);
+
+		if (blob_size > sizeof(buf)) {
+			LOGP(DDB, LOGL_ERROR,
+			     "OsmoNITB import to %s: Cannot import auth data for IMSI %s:"
+			     " too large blob: %u\n",
+			     dbc->fname, imsi, blob_size);
+			db_remove_reset(stmt);
+			continue;
+		}
+
+		decoded_size = _dbd_decode_binary(blob, buf);
+		osmo_strlcpy(ki, osmo_hexdump_nospc(buf, decoded_size), sizeof(ki));
+
+		db_subscr_update_aud_by_id(dbc, hlr_id, &aud2g);
+		count ++;
+	}
+
+	if (rc != SQLITE_DONE && rc != SQLITE_ROW) {
+		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
+		     " during stmt '%s'",
+		     rc, sqlite3_errmsg(nitb_db),
+		     nitb_stmt_sql[NITB_SELECT_AUTH_KEYS]);
+	}
+
+	db_remove_reset(stmt);
+}
+
+void import_nitb_subscr(sqlite3 *nitb_db, sqlite3_stmt *stmt)
+{
+	struct db_context *dbc = g_hlr_db_tool_ctx->dbc;
+	int rc;
+	struct hlr_subscriber subscr;
+
+	int64_t nitb_id;
+	int64_t imsi;
+	char imsi_str[32];
+	bool authorized;
+
+	imsi = sqlite3_column_int64(stmt, 0);
+
+	snprintf(imsi_str, sizeof(imsi_str), "%"PRId64, imsi);
+
+	rc = db_subscr_create(dbc, imsi_str);
+	if (rc) {
+		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: failed to create IMSI %s: %d: %s\n",
+		     dbc->fname,
+		     imsi_str,
+		     rc,
+		     strerror(rc));
+		/* on error, still attempt to continue */
+	}
+
+	nitb_id = sqlite3_column_int64(stmt, 1);
+	copy_sqlite3_text_to_buf(subscr.msisdn, stmt, 2);
+	authorized = sqlite3_column_int(stmt, 3) ? true : false;
+
+	db_subscr_update_msisdn_by_imsi(dbc, imsi_str, subscr.msisdn);
+	db_subscr_nam(dbc, imsi_str, authorized, true);
+	db_subscr_nam(dbc, imsi_str, authorized, false);
+
+	/* find the just created id */
+	rc = db_subscr_get_by_imsi(dbc, imsi_str, &subscr);
+	if (rc) {
+		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB import to %s: created IMSI %s,"
+		     " but failed to get new subscriber id: %d: %s\n",
+		     dbc->fname,
+		     imsi_str,
+		     rc,
+		     strerror(rc));
+		return;
+	}
+
+	OSMO_ASSERT(!strcmp(imsi_str, subscr.imsi));
+
+	import_nitb_subscr_aud(nitb_db, imsi_str, nitb_id, subscr.id);
+}
+
+int import_nitb_db(void)
+{
+	int i;
+	int ret;
+	int rc;
+	const char *sql;
+	sqlite3_stmt *stmt;
+
+	sqlite3 *nitb_db = open_nitb_db(cmdline_opts.import_nitb_db);
+
+	if (!nitb_db)
+		return -1;
+	ret = 0;
+
+	for (i = 0; i < ARRAY_SIZE(nitb_stmt_sql); i++) {
+		sql = nitb_stmt_sql[i];
+		rc = sqlite3_prepare_v2(nitb_db, sql, -1, &nitb_stmt[i], NULL);
+		if (rc != SQLITE_OK) {
+			LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: Unable to prepare SQL statement '%s'\n", sql);
+			ret = -1;
+			goto out_free;
+		}
+	}
+
+	stmt = nitb_stmt[NITB_SELECT_SUBSCR];
+
+	while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) {
+		import_nitb_subscr(nitb_db, stmt);
+		/* On failure, carry on with the rest. */
+	}
+	if (rc != SQLITE_DONE) {
+		LOGP(DDB, LOGL_ERROR, "OsmoNITB DB: SQL error: (%d) %s,"
+		     " during stmt '%s'",
+		     rc, sqlite3_errmsg(nitb_db),
+		     nitb_stmt_sql[NITB_SELECT_SUBSCR]);
+		goto out_free;
+	}
+
+	db_remove_reset(stmt);
+	sqlite3_finalize(stmt);
+
+out_free:
+	sqlite3_close(nitb_db);
+	return ret;
+}
+
+int main(int argc, char **argv)
+{
+	int rc;
+	int (*main_action)(void);
+	main_action = NULL;
+
+	g_hlr_db_tool_ctx = talloc_zero(NULL, struct hlr_db_tool_ctx);
+	OSMO_ASSERT(g_hlr_db_tool_ctx);
+	talloc_set_name_const(g_hlr_db_tool_ctx, "OsmoHLR-db-tool");
+
+	rc = osmo_init_logging(&hlr_log_info);
+	if (rc < 0) {
+		fprintf(stderr, "Error initializing logging\n");
+		exit(EXIT_FAILURE);
+	}
+
+	handle_options(argc, argv);
+
+	if (cmdline_opts.import_nitb_db) {
+		if (main_action)
+			goto too_many_actions;
+		main_action = import_nitb_db;
+	}
+	/* Future: add more main_actions, besides --import-nitb-db, here. */
+
+	/* Just in case any db actions need randomness */
+	rc = rand_init();
+	if (rc < 0) {
+		LOGP(DMAIN, LOGL_FATAL, "Error initializing random source\n");
+		exit(EXIT_FAILURE);
+	}
+
+	g_hlr_db_tool_ctx->dbc = db_open(g_hlr_db_tool_ctx, cmdline_opts.db_file);
+	if (!g_hlr_db_tool_ctx->dbc) {
+		LOGP(DMAIN, LOGL_FATAL, "Error opening database\n");
+		exit(EXIT_FAILURE);
+	}
+
+	osmo_init_ignore_signals();
+	signal(SIGINT, &signal_hdlr);
+	signal(SIGUSR1, &signal_hdlr);
+
+	rc = 0;
+	if (main_action)
+		rc = (*main_action)();
+
+	db_close(g_hlr_db_tool_ctx->dbc);
+	log_fini();
+	exit(rc ? EXIT_FAILURE : EXIT_SUCCESS);
+
+too_many_actions:
+	fprintf(stderr, "Too many actions requested.\n");
+	log_fini();
+	exit(EXIT_FAILURE);
+}
+
+/* stubs */
+void lu_op_alloc_conn(void) { OSMO_ASSERT(0); }
+void lu_op_tx_del_subscr_data(void) { OSMO_ASSERT(0); }
+void lu_op_free(void) { OSMO_ASSERT(0); }

-- 
To view, visit https://gerrit.osmocom.org/4441
To unsubscribe, visit https://gerrit.osmocom.org/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I0dfa6ec033dd93161c1adc2ce1637195fe5b7a63
Gerrit-PatchSet: 2
Gerrit-Project: osmo-hlr
Gerrit-Branch: master
Gerrit-Owner: Neels Hofmeyr <nhofmeyr at sysmocom.de>
Gerrit-Reviewer: Harald Welte <laforge at gnumonks.org>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: Neels Hofmeyr <nhofmeyr at sysmocom.de>


More information about the gerrit-log mailing list