This is merely a historical archive of years 2008-2021, before the migration to mailman3.
A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.
Vadim Yanitskiy gerrit-no-reply at lists.osmocom.orgVadim Yanitskiy has submitted this change and it was merged. ( https://gerrit.osmocom.org/c/osmo-hlr/+/14939 )
Change subject: src/db.c: integrate SQLite3 with talloc allocator
......................................................................
src/db.c: integrate SQLite3 with talloc allocator
This change introduces an optional feature that allows to make
SQLite3 use talloc for all internal allocations. This would
facilitate finding memleaks. OsmoHLR needs to be configured
with '--enable-sqlite-talloc'.
full talloc report on 'OsmoHLR' (total 292168 bytes in 449 blocks)
struct osmo_gsup_server contains 162 bytes in 3 blocks (ref 0)
...
struct db_context contains 288407 bytes in 420 blocks (ref 0)
hlr.db contains 7 bytes in 1 blocks (ref 0)
SQLite3 contains 288192 bytes in 418 blocks (ref 0)
db.c:95 contains 48 bytes in 1 blocks (ref 0)
db.c:95 contains 2 bytes in 1 blocks (ref 0)
...
Unfortunately, old SQLite3 versions (such as 3.8.2) run out
of memory when trying to initialize a new database:
DDB ERROR db.c:88 (7) statement aborts at 3: []
DDB ERROR db.c:420 Unable to set Write-Ahead Logging: out of memory
DDB ERROR db.c:88 (7) statement aborts at 3: []
DDB ERROR db.c:238 Unable to prepare SQL statement
'SELECT name FROM sqlite_master WHERE type='table' AND name=?'
...
I've noticed a huge difference in heap usage footprint compared to
generic malloc. At the same time, the recent versions (at least
3.24.0), work just fine.
Change-Id: Icfe67ed0f063b63e6794f9516da3003d01cf20a7
---
M configure.ac
M src/Makefile.am
M src/db.c
M src/db.h
A src/db_debug.c
M tests/db/Makefile.am
6 files changed, 126 insertions(+), 0 deletions(-)
Approvals:
laforge: Looks good to me, approved
neels: Looks good to me, but someone else must approve
Jenkins Builder: Verified
diff --git a/configure.ac b/configure.ac
index 6694f80..ef703f2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -59,6 +59,21 @@
CPPFLAGS="$CPPFLAGS -fsanitize=address -fsanitize=undefined"
fi
+AC_ARG_ENABLE([sqlite_talloc],
+ AC_HELP_STRING([--enable-sqlite-talloc],
+ [Configure SQLite3 to use talloc memory allocator [default=no]]),
+ [sqlite_talloc="$enableval"],[sqlite_talloc="no"])
+if test "x$sqlite_talloc" = "xyes" ; then
+ # Older versions of SQLite3 (at least 3.8.2) become unstable with talloc.
+ # Feel free to relax to 3.24.0 > VER > 3.8.2 if it works for you.
+ # FIXME: PKG_CHECK_MODULES() may return cached result here!
+ PKG_CHECK_MODULES(SQLITE3, sqlite3 >= 3.24.0)
+ AC_DEFINE([SQLITE_USE_TALLOC], 1, [Use talloc for SQLite3])
+fi
+AC_MSG_CHECKING([whether to use talloc for SQLite3])
+AC_MSG_RESULT([$sqlite_talloc])
+AM_CONDITIONAL([DB_SQLITE_DEBUG], [test "x$sqlite_talloc" = "xyes"])
+
AC_ARG_ENABLE(werror,
[AS_HELP_STRING(
[--enable-werror],
diff --git a/src/Makefile.am b/src/Makefile.am
index 131b44f..a042e4e 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -97,6 +97,11 @@
$(LIBOSMOGSM_LIBS) \
$(NULL)
+if DB_SQLITE_DEBUG
+osmo_hlr_SOURCES += db_debug.c
+osmo_hlr_db_tool_SOURCES += db_debug.c
+endif
+
BOOTSTRAP_SQL = $(top_srcdir)/sql/hlr.sql
db_bootstrap.h: $(BOOTSTRAP_SQL) $(srcdir)/db_sql2c.sed
diff --git a/src/db.c b/src/db.c
index 7de61a2..5e6b5eb 100644
--- a/src/db.c
+++ b/src/db.c
@@ -365,6 +365,17 @@
LOGP(DDB, LOGL_INFO, "Compiled against SQLite3 lib version %s\n", SQLITE_VERSION);
LOGP(DDB, LOGL_INFO, "Running with SQLite3 lib version %s\n", sqlite3_libversion());
+#ifdef SQLITE_USE_TALLOC
+ /* Configure SQLite3 to use talloc memory allocator */
+ rc = db_sqlite3_use_talloc(ctx);
+ if (rc == SQLITE_OK) {
+ LOGP(DDB, LOGL_NOTICE, "SQLite3 is configured to use talloc\n");
+ } else {
+ LOGP(DDB, LOGL_ERROR, "Failed to configure SQLite3 "
+ "to use talloc, using default memory allocator\n");
+ }
+#endif
+
dbc->fname = talloc_strdup(dbc, fname);
for (i = 0; i < 0xfffff; i++) {
diff --git a/src/db.h b/src/db.h
index 6e4bf49..15d83de 100644
--- a/src/db.h
+++ b/src/db.h
@@ -39,6 +39,11 @@
sqlite3_stmt *stmt[_NUM_DB_STMT];
};
+/* Optional feature to make SQLite3 using talloc */
+#ifdef SQLITE_USE_TALLOC
+int db_sqlite3_use_talloc(void *ctx);
+#endif
+
void db_remove_reset(sqlite3_stmt *stmt);
bool db_bind_text(sqlite3_stmt *stmt, const char *param_name, const char *text);
bool db_bind_int(sqlite3_stmt *stmt, const char *param_name, int nr);
diff --git a/src/db_debug.c b/src/db_debug.c
new file mode 100644
index 0000000..13ccdd6
--- /dev/null
+++ b/src/db_debug.c
@@ -0,0 +1,86 @@
+/*
+ * libtalloc based memory allocator for SQLite3.
+ *
+ * (C) 2019 by Vadim Yanitskiy <axilirator at gmail.com>
+ *
+ * All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <sqlite3.h>
+#include <talloc.h>
+#include <errno.h>
+
+/* Dedicated talloc context for SQLite */
+static void *db_sqlite_ctx = NULL;
+
+static void *tall_xMalloc(int size)
+{
+ return talloc_size(db_sqlite_ctx, size);
+}
+
+static void tall_xFree(void *ptr)
+{
+ talloc_free(ptr);
+}
+
+static void *tall_xRealloc(void *ptr, int size)
+{
+ return talloc_realloc_fn(db_sqlite_ctx, ptr, size);
+}
+
+static int tall_xSize(void *ptr)
+{
+ return talloc_total_size(ptr);
+}
+
+/* DUMMY: talloc doesn't round up the allocation size */
+static int tall_xRoundup(int size) { return size; }
+
+/* DUMMY: nothing to initialize */
+static int tall_xInit(void *data) { return 0; }
+
+/* DUMMY: nothing to deinitialize */
+static void tall_xShutdown(void *data) { }
+
+/* Interface between SQLite and talloc memory allocator */
+static const struct sqlite3_mem_methods tall_sqlite_if = {
+ /* Memory allocation function */
+ .xMalloc = &tall_xMalloc,
+ /* Free a prior allocation */
+ .xFree = &tall_xFree,
+ /* Resize an allocation */
+ .xRealloc = &tall_xRealloc,
+ /* Return the size of an allocation */
+ .xSize = &tall_xSize,
+ /* Round up request size to allocation size */
+ .xRoundup = &tall_xRoundup,
+ /* Initialize the memory allocator */
+ .xInit = &tall_xInit,
+ /* Deinitialize the memory allocator */
+ .xShutdown = &tall_xShutdown,
+ /* Argument to xInit() and xShutdown() */
+ .pAppData = NULL,
+};
+
+int db_sqlite3_use_talloc(void *ctx)
+{
+ if (db_sqlite_ctx != NULL)
+ return -EEXIST;
+
+ db_sqlite_ctx = talloc_named_const(ctx, 0, "SQLite3");
+ return sqlite3_config(SQLITE_CONFIG_MALLOC, &tall_sqlite_if);
+}
diff --git a/tests/db/Makefile.am b/tests/db/Makefile.am
index fa925f8..5730937 100644
--- a/tests/db/Makefile.am
+++ b/tests/db/Makefile.am
@@ -36,6 +36,10 @@
$(SQLITE3_LIBS) \
$(NULL)
+if DB_SQLITE_DEBUG
+db_test_LDADD += $(top_builddir)/src/db_debug.o
+endif
+
.PHONY: db_test.db update_exp manual manual-nonverbose manual-gdb
db_test.db:
rm -f db_test.db
--
To view, visit https://gerrit.osmocom.org/c/osmo-hlr/+/14939
To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings
Gerrit-Project: osmo-hlr
Gerrit-Branch: master
Gerrit-Change-Id: Icfe67ed0f063b63e6794f9516da3003d01cf20a7
Gerrit-Change-Number: 14939
Gerrit-PatchSet: 4
Gerrit-Owner: Vadim Yanitskiy <axilirator at gmail.com>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: Vadim Yanitskiy <axilirator at gmail.com>
Gerrit-Reviewer: laforge <laforge at gnumonks.org>
Gerrit-Reviewer: neels <nhofmeyr at sysmocom.de>
Gerrit-Reviewer: pespin <pespin at sysmocom.de>
Gerrit-MessageType: merged
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20190730/74a1c423/attachment.htm>