Dynamic loading of database driver - libzdb-general

7 Nov 2010

Hi all,
 I implemented the feature noted in $subject on top of libzdb 2.7
(trunk@380). My initial goal was to produce a nicer package which would
not require to install client libraries for database system I don't use.
 I made the feature optional (disabled by default in configure, new
option '--enable-libdl'), and it is lightly tested (test/select still
works after the 2 below patches are applied, both with the libdl enabled
or not).
 The only extra-requirement is that the patch uses GCC non-standard
"__attribute__ ((constructor))" to not have to modify any user. I didn't
even try to implement driver unloading, although it is certainly
possible too.
 Let me know what you think of it.
 Thanks,
Matthieu
...
From 234b8f8145872ee6dc02d66e4aa6cd39c5a603b4 Mon Sep 17 00:00:00 2001
From: Matthieu Verbert matthieu.verbert@aitb.org
Date: Sat, 6 Nov 2010 13:02:04 +0100
Subject: [PATCH 1/2] Update autotools to support multiple libs and libdl
Create separate variables for each database 'driver', and
add them only where they need to be used (Oracle untested).
configure's behaviour is unchanged if libdl is not enabled,
else multiple libraries will be created. Note that the
test program breaks at link time if this option is enabled.
---
 Makefile.am  |   76 +++++++++++++++++++++++++++++++++++++++---------
 configure.ac |   90 ++++++++++++++++++++++++++++++++++++----------------------
 2 files changed, 117 insertions(+), 49 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 31d92cd..2061c3d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,7 +14,7 @@ RE2C		= @RE2C@
 RE2CFLAGS       = -b
 FILTERH         = ./tools/bin/filterh
-AM_CPPFLAGS     = $(CPPFLAGS) $(DBCPPFLAGS)
+AM_CPPFLAGS     = 
 INCLUDES        = -Isrc -Isrc/util -Isrc/net -Isrc/db -Isrc/exceptions
lib_LTLIBRARIES = libzdb.la
@@ -23,29 +23,75 @@ libzdb_la_SOURCES = src/util/Util.c src/util/Str.c src/util/Mem.c \
                     src/db/Connection.c src/db/ResultSet.c \
                     src/db/PreparedStatement.c src/util/StringBuffer.c \
                     src/exceptions/assert.c src/exceptions/Exception.c
-
 if ! WITH_ZILD
 libzdb_la_SOURCES += src/net/URL.c 
 endif
+
+libzdb_la_LDFLAGS =
+noinst_LTLIBRARIES =
+libzdb_la_LIBADD = 
+
+if WITH_LIBDL
+libzdb_la_LDFLAGS += -ldl
+endif
+
 if WITH_MYSQL
-libzdb_la_SOURCES += src/db/mysql/MysqlConnection.c \
-                     src/db/mysql/MysqlResultSet.c \
-                     src/db/mysql/MysqlPreparedStatement.c
+if WITH_LIBDL
+lib_LTLIBRARIES += libzdb-mysql.la
+else
+noinst_LTLIBRARIES += libzdb-mysql.la
+libzdb_la_LIBADD += libzdb-mysql.la
 endif
+
+libzdb_mysql_la_SOURCES = src/db/mysql/MysqlConnection.c \
+		          src/db/mysql/MysqlResultSet.c \
+		          src/db/mysql/MysqlPreparedStatement.c
+libzdb_mysql_la_CPPFLAGS = $(DB_MYSQL_CPPFLAGS)
+libzdb_mysql_la_LDFLAGS = $(DB_MYSQL_LDFLAGS)
+endif # WITH_MYSQL
+
 if WITH_POSTGRESQL
-libzdb_la_SOURCES += src/db/postgresql/PostgresqlConnection.c \
-                     src/db/postgresql/PostgresqlResultSet.c \
-                     src/db/postgresql/PostgresqlPreparedStatement.c
+if WITH_LIBDL
+lib_LTLIBRARIES += libzdb-postgresql.la
+else
+noinst_LTLIBRARIES += libzdb-postgresql.la
+libzdb_la_LIBADD += libzdb-postgresql.la
 endif
+libzdb_postgresql_la_SOURCES = src/db/postgresql/PostgresqlConnection.c \
+		               src/db/postgresql/PostgresqlResultSet.c \
+		               src/db/postgresql/PostgresqlPreparedStatement.c
+libzdb_postgresql_la_CPPFLAGS = $(DB_POSTGRESQL_CPPFLAGS)
+libzdb_postgresql_la_LDFLAGS = $(DB_POSTGRESQL_LDFLAGS)
+endif # WITH_POSTGRESQL
+
 if WITH_SQLITE
-libzdb_la_SOURCES += src/db/sqlite/SQLiteConnection.c \
-                     src/db/sqlite/SQLiteResultSet.c \
-                     src/db/sqlite/SQLitePreparedStatement.c
+if WITH_LIBDL
+lib_LTLIBRARIES += libzdb-sqlite.la
+else
+noinst_LTLIBRARIES += libzdb-sqlite.la
+libzdb_la_LIBADD += libzdb-sqlite.la
 endif
+
+libzdb_sqlite_la_SOURCES = src/db/sqlite/SQLiteConnection.c \
+		           src/db/sqlite/SQLiteResultSet.c \
+		           src/db/sqlite/SQLitePreparedStatement.c
+libzdb_sqlite_la_CPPFLAGS = $(DB_SQLITE_CPPFLAGS)
+libzdb_sqlite_la_LDFLAGS = $(DB_SQLITE_LDFLAGS)
+endif # WITH_SQLITE
+
 if WITH_ORACLE
-libzdb_la_SOURCES += src/db/oracle/OracleConnection.c \
-                     src/db/oracle/OracleResultSet.c \
-                     src/db/oracle/OraclePreparedStatement.c
+if WITH_LIBDL
+lib_LTLIBRARIES += libzdb-oracle.la
+else
+noinst_LTLIBRARIES += libzdb-oracle.la
+libzdb_la_LIBADD += libzdb-oracle.la
+endif
+
+libzdb_oracle_la_SOURCES = src/db/oracle/OracleConnection.c \
+		           src/db/oracle/OracleResultSet.c \
+		           src/db/oracle/OraclePreparedStatement.c
+libzdb_oracle_la_CPPFLAGS = $(DB_ORACLE_CPPFLAGS)
+libzdb_oracle_la_LDFLAGS = $(DB_ORACLE_LDFLAGS)
 endif
@@ -55,7 +101,7 @@ API_INTERFACES  = src/zdb.h src/db/ConnectionPool.h src/db/Connection.h \
nobase_nodist_include_HEADERS = $(patsubst %, $(LIBRARY_NAME)/%, $(notdir $(API_INTERFACES)))
-libzdb_la_LDFLAGS = $(DBLDFLAGS) -version-info 7:1:0
+AM_LDFLAGS = -version-info 7:1:0
BUILT_SOURCES   = $(nobase_nodist_include_HEADERS)
diff --git a/configure.ac b/configure.ac
index f771cda..ac51435 100644
--- a/configure.ac
+++ b/configure.ac
@@ -109,6 +109,18 @@ AC_ARG_ENABLE(profiling,
     ]
 )
+AC_ARG_ENABLE(libdl, 
+              AS_HELP_STRING([--enable-libdl],
+              [Use libdl to dynamic load database drivers]),
+              [AC_CHECK_LIB([dl], [dlopen], [libdl="yes"], [libdl="no"])],
+              [libdl="no"]
+)
+if test "xyes" = "x$libdl"; then
+        AC_DEFINE([HAVE_LIBDL], 1, [Define to 1 to enable libdl])
+fi
+
+AM_CONDITIONAL([WITH_LIBDL], test "xyes" = "x$libdl")
+
 AC_ARG_ENABLE([openssl],
         AS_HELP_STRING([--enable-openssl(=<path>)], 
                 [Link libzdb with openssl. If database libraries were linked static,
@@ -205,6 +217,7 @@ AC_ARG_WITH([mysql],
 if test "xyes" = "x$mysql"; then
         svd_CPPFLAGS=$CPPFLAGS
         svd_LDFLAGS=$LDFLAGS
+        svd_LIBS=$LIBS
         CPPFLAGS="`$MYSQLCONFIG --include` $CPPFLAGS"
         LDFLAGS="`$MYSQLCONFIG --libs` $LDFLAGS"
         AC_CHECK_HEADERS([mysql.h])
@@ -226,13 +239,15 @@ if test "xyes" = "x$mysql"; then
                 fi
         ], [mysql="no"], [-lz])
         if test "xyes" = "x$mysql"; then
-                DBCPPFLAGS="$DBCPPFLAGS `$MYSQLCONFIG --include`"
-                DBLDFLAGS="$DBLDFLAGS `$MYSQLCONFIG --libs`"
+                DB_MYSQL_CPPFLAGS="`$MYSQLCONFIG --include`"
+                DB_MYSQL_LDFLAGS="`$MYSQLCONFIG --libs` -lz"
                 AC_DEFINE([HAVE_LIBMYSQLCLIENT], 1, [Define to 1 to enable mysql])
-        else
-                CPPFLAGS=$svd_CPPFLAGS
-                LDFLAGS=$svd_LDFLAGS
+                AC_SUBST(DB_MYSQL_LDFLAGS)
+                AC_SUBST(DB_MYSQL_CPPFLAGS)
         fi
+        CPPFLAGS=$svd_CPPFLAGS
+        LDFLAGS=$svd_LDFLAGS
+        LIBS=$svd_LIBS
 fi
 AM_CONDITIONAL([WITH_MYSQL], test "xyes" = "x$mysql")
@@ -273,13 +288,14 @@ if test "xyes" = "x$postgresql"; then
         LDFLAGS="-L`$PGCONFIG --libdir` $LDFLAGS"
         AC_CHECK_HEADERS([libpq-fe.h], [], [postgresql="no"])
         if test "xyes" = "x$postgresql"; then
-                DBCPPFLAGS="$DBCPPFLAGS -I`$PGCONFIG --includedir`"
-                DBLDFLAGS="$DBLDFLAGS -L`$PGCONFIG --libdir` -lpq"
+                DB_POSTGRESQL_CPPFLAGS="-I`$PGCONFIG --includedir`"
+                DB_POSTGRESQL_LDFLAGS="-L`$PGCONFIG --libdir` -lpq"
                 AC_DEFINE([HAVE_LIBPQ], 1, [Define to 1 to enable postgresql])
-        else
-                CPPFLAGS=$svd_CPPFLAGS
-                LDFLAGS=$svd_LDFLAGS
+                AC_SUBST(DB_POSTGRESQL_LDFLAGS)
+                AC_SUBST(DB_POSTGRESQL_CPPFLAGS)
         fi
+        CPPFLAGS=$svd_CPPFLAGS
+        LDFLAGS=$svd_LDFLAGS
 fi
 AM_CONDITIONAL([WITH_POSTGRESQL], test "xyes" = "x$postgresql")
@@ -294,51 +310,52 @@ AC_ARG_WITH([sqlite],
                 sqlite="no"
         else
                 AC_MSG_RESULT([yes])
-                svd_LDFLAGS=$LDFLAGS
-                svd_CPPFLAGS=$CPPFLAGS
-                LDFLAGS="-L$with_sqlite/lib $LDFLAGS"
-                CPPFLAGS="-I$with_sqlite/include $CPPFLAGS"
-                AC_CHECK_HEADERS([sqlite3.h], [
-                        sqlite="yes"
-                        if test -r "$with_sqlite/lib/libsqlite3.a"; then
-                                DBCPPFLAGS="$DBCPPFLAGS -I$with_sqlite/include"
-                                DBLDFLAGS="$DBLDFLAGS -L$with_sqlite/lib/ -lsqlite3"
-                        else
-                                sqlite="no"
-                        fi
-                ], [sqlite="no"])
-                LDFLAGS=$svd_LDFLAGS
-                CPPFLAGS=$svd_CPPFLAGS
         fi
         ], 
         [
-                AC_MSG_RESULT([yes])
-                AC_CHECK_LIB([sqlite3], [sqlite3_open], [], [sqlite="no"])
+               AC_MSG_RESULT([yes])
         ])
+
 if test "xyes" = "x$sqlite"; then
-        AC_DEFINE([HAVE_LIBSQLITE3])
+       svd_LDFLAGS=$LDFLAGS
+       svd_CPPFLAGS=$CPPFLAGS
+       svd_LIBS=$LIBS
+       CPPFLAGS="-I$with_sqlite/include $CPPFLAGS"
+       LDFLAGS="-L$with_sqlite/lib $LDFLAGS"
+       AC_CHECK_HEADERS([sqlite3.h], [sqlite="yes"], [sqlite="no"])
+       AC_CHECK_LIB([sqlite3], [sqlite3_open], [], [sqlite="no"])
+       if test "xyes" = "x$sqlite"; then
+            DB_SQLITE_CPPFLAGS="-I$with_sqlite/include"
+            DB_SQLITE_LDFLAGS="-L$with_sqlite/lib/ -lsqlite3"
+            AC_SUBST(DB_SQLITE_CPPFLAGS)
+            AC_SUBST(DB_SQLITE_LDFLAGS)                        
+       fi
+       LDFLAGS=$svd_LDFLAGS
+       CPPFLAGS=$svd_CPPFLAGS
+       LIBS=$svd_LIBS
 fi
+
 AM_CONDITIONAL([WITH_SQLITE], test "xyes" = "x$sqlite")
oracle="yes"
 AC_MSG_CHECKING(for oracle)
 AX_LIB_ORACLE_OCI
 if test -n "$ORACLE_OCI_CFLAGS" -a -n "$ORACLE_OCI_LDFLAGS"; then
-    DBCPPFLAGS="$DBCPPFLAGS $ORACLE_OCI_CFLAGS"
-    DBLDFLAGS="$DBLDFLAGS $ORACLE_OCI_LDFLAGS"
+    DB_ORACLE_CPPFLAGS="$ORACLE_OCI_CFLAGS"
+    DB_ORACLE_LDFLAGS="$ORACLE_OCI_LDFLAGS"
     AC_DEFINE([HAVE_ORACLE], 1, [Define to 1 to enable oracle])
+    AC_SUBST(DB_ORACLE_CPPFLAGS)
+    AC_SUBST(DB_ORACLE_LDFLAGS)
 else
         oracle="no"
 fi
 AM_CONDITIONAL([WITH_ORACLE], test "xyes" = "x$oracle")
# Test if any database system was found
-if test "xno" = "x$postgresql" -a "xno" = "x$mysql" -a "xno" = "x$sqlite" -a "xno" = "x$oracle"; then
-        AC_MSG_ERROR([No available database found or selected. Try configure --help])
+if test "xno" = "x$postgresql" -a "xno" = "x$mysql" -a "xno" = "x$sqlite" -a "xno" = "x$oracle" -a "xno" = "x$libdl" ; then
+        AC_MSG_ERROR([No available database found or selected, and dynamic loading disabled. Try configure --help])
 fi
-AC_SUBST(DBLDFLAGS)
-AC_SUBST(DBCPPFLAGS)
# ---------------------------------------------------------------------------
 # Header files 
@@ -413,6 +430,11 @@ echo "|   Profiling:                                    DISABLED   |"
 else
 echo "|   Profiling:                                    ENABLED    |"
 fi
+if test "xno" = "x$libdl"; then
+echo "|   Dynamically load drivers:                     DISABLED   |"
+else
+echo "|   Dynamically load drivers:                     ENABLED    |"
+fi
 if test "xfalse" = "x$zild_protect"; then
 echo "|   Zild:                                         DISABLED   |"
 else
-- 
1.7.2.3

>From 462d869e0a7f563eab4129241fb5884480e6da2d Mon Sep 17 00:00:00 2001
From: Matthieu Verbert matthieu.verbert@aitb.org
Date: Sat, 6 Nov 2010 21:11:33 +0100
Subject: [PATCH 2/2] Register 'driver' on library load

Add new function ConnectionDelegate_register to add a new
'driver' dynamically. It is auto-loaded, so no need to modify
any user.
Oracle is not compile-tested, but the conversion is trivial.
---
 src/db/Connection.c                      |   65 +++++++++++++++--------------
 src/db/ConnectionDelegate.h              |    2 +
 src/db/mysql/MysqlConnection.c           |    6 ++-
 src/db/oracle/OracleConnection.c         |    6 ++-
 src/db/postgresql/PostgresqlConnection.c |    6 ++-
 src/db/sqlite/SQLiteConnection.c         |    6 ++-
 6 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/src/db/Connection.c b/src/db/Connection.c
index cc27cf4..dcfd1e7 100644
--- a/src/db/Connection.c
+++ b/src/db/Connection.c
@@ -18,7 +18,9 @@

 #include <stdio.h>
 #include <stdarg.h>
+#include <dlfcn.h>

+#include "StringBuffer.h"
 #include "URL.h"
 #include "Vector.h"
 #include "ResultSet.h"
@@ -38,34 +40,11 @@
 /* ----------------------------------------------------------- Definitions */


-#ifdef HAVE_LIBMYSQLCLIENT
-extern const struct Cop_T mysqlcops;
-#endif
-#ifdef HAVE_LIBPQ
-extern const struct Cop_T postgresqlcops;
-#endif
-#ifdef HAVE_LIBSQLITE3
-extern const struct Cop_T sqlite3cops;
-#endif
-#ifdef HAVE_ORACLE
-extern const struct Cop_T oraclesqlcops;
-#endif
+static Vector_T cops;

-static const struct Cop_T *cops[] = {
-#ifdef HAVE_LIBMYSQLCLIENT
-        &mysqlcops,
-#endif
-#ifdef HAVE_LIBPQ
-        &postgresqlcops,
-#endif
-#ifdef HAVE_LIBSQLITE3
-        &sqlite3cops,
-#endif
-#ifdef HAVE_ORACLE
-        &oraclesqlcops,
-#endif
-        NULL
-};
+static void __attribute__ ((constructor (200))) init_cops() {
+  cops = Vector_new(1);
+}

 #define T Connection_T
 struct T {
@@ -87,10 +66,28 @@ struct T {


 static Cop_T getOp(const char *protocol) {
-        for (int i = 0; cops[i]; i++) 
-                if (Str_startsWith(protocol, cops[i]->name)) 
-                        return (Cop_T)cops[i];
-        return NULL;
+  int e = 0;
+  do {
+    for (int i = 0; i < Vector_size(cops); i++) {
+      Cop_T op = (Cop_T) Vector_get(cops, i);
+      if (Str_startsWith(protocol, op->name))
+        return op;
+    }
+    if (e > 0)
+      break;
+    {
+      StringBuffer_T fn = StringBuffer_new("libzdb-");
+      StringBuffer_append(fn, protocol);
+      StringBuffer_append(fn, ".so");
+      void * lib = dlopen(StringBuffer_toString(fn), RTLD_LAZY);
+      StringBuffer_free(&fn);
+      if (lib)
+        e++;
+      else
+        break;
+    }
+  } while (true);
+  return NULL;
 }


@@ -343,3 +340,9 @@ int Connection_isSupported(const char *url) {
         return (url ? (getOp(url) != NULL) : false);
 }

+
+
+void ConnectionDelegate_register(Cop_T op) {
+  Vector_push(cops, op);
+}
+
diff --git a/src/db/ConnectionDelegate.h b/src/db/ConnectionDelegate.h
index 88c959c..8ff485e 100644
--- a/src/db/ConnectionDelegate.h
+++ b/src/db/ConnectionDelegate.h
@@ -46,5 +46,7 @@ typedef struct Cop_T {
         const char *(*getLastError)(T C);
 } *Cop_T;

+void ConnectionDelegate_register(Cop_T op);
+
 #undef T
 #endif
diff --git a/src/db/mysql/MysqlConnection.c b/src/db/mysql/MysqlConnection.c
index 0f0e848..3159132 100644
--- a/src/db/mysql/MysqlConnection.c
+++ b/src/db/mysql/MysqlConnection.c
@@ -45,7 +45,7 @@

 #define MYSQL_OK 0

-const struct Cop_T mysqlcops = {
+static const struct Cop_T mysqlcops = {
         "mysql",
         MysqlConnection_new,
         MysqlConnection_free,
@@ -63,6 +63,10 @@ const struct Cop_T mysqlcops = {
         MysqlConnection_getLastError
 };

+static void __attribute__ ((constructor (300))) register_mysql() {
+  ConnectionDelegate_register(&mysqlcops);
+}
+
 #define T ConnectionDelegate_T
 struct T {
         URL_T url;
diff --git a/src/db/oracle/OracleConnection.c b/src/db/oracle/OracleConnection.c
index 4d210ff..2267499 100644
--- a/src/db/oracle/OracleConnection.c
+++ b/src/db/oracle/OracleConnection.c
@@ -44,7 +44,7 @@
 /* ----------------------------------------------------------- Definitions */


-const struct Cop_T oraclesqlcops = {
+static const struct Cop_T oraclesqlcops = {
         "oracle",
         OracleConnection_new,
         OracleConnection_free,
@@ -62,6 +62,10 @@ const struct Cop_T oraclesqlcops = {
         OracleConnection_getLastError
 };

+static void __attribute__ ((constructor (300))) register_oracle() {
+  ConnectionDelegate_register(&oraclesqlcops);
+}
+
 #define ERB_SIZE 152
 #define ORACLE_TRANSACTION_PERIOD 10

diff --git a/src/db/postgresql/PostgresqlConnection.c b/src/db/postgresql/PostgresqlConnection.c
index e477c73..0bdd854 100644
--- a/src/db/postgresql/PostgresqlConnection.c
+++ b/src/db/postgresql/PostgresqlConnection.c
@@ -41,7 +41,7 @@
 /* ----------------------------------------------------------- Definitions */


-const struct Cop_T postgresqlcops = {
+static const struct Cop_T postgresqlcops = {
         "postgresql",
         PostgresqlConnection_new,
         PostgresqlConnection_free,
@@ -59,6 +59,10 @@ const struct Cop_T postgresqlcops = {
         PostgresqlConnection_getLastError
 };

+static void __attribute__ ((constructor (300))) register_postgresql() {
+  ConnectionDelegate_register(&postgresqlcops);
+}
+
 #define T ConnectionDelegate_T
 struct T {
         URL_T url;
diff --git a/src/db/sqlite/SQLiteConnection.c b/src/db/sqlite/SQLiteConnection.c
index ab102ae..175bb9b 100644
--- a/src/db/sqlite/SQLiteConnection.c
+++ b/src/db/sqlite/SQLiteConnection.c
@@ -39,7 +39,7 @@
 /* ----------------------------------------------------------- Definitions */


-const struct Cop_T sqlite3cops = {
+static const struct Cop_T sqlite3cops = {
         "sqlite",
         SQLiteConnection_new,
         SQLiteConnection_free,
@@ -57,6 +57,10 @@ const struct Cop_T sqlite3cops = {
         SQLiteConnection_getLastError
 };

+static void __attribute__ ((constructor (300))) register_sqlite() {
+  ConnectionDelegate_register(&sqlite3cops);
+}
+
 #define T ConnectionDelegate_T
 struct T {
         URL_T url;
-- 
1.7.2.3




    