summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRex Dieter <rdieter@math.unl.edu>2014-04-16 14:23:26 (GMT)
committerRex Dieter <rdieter@math.unl.edu>2014-04-16 14:23:26 (GMT)
commitfcd69dcca69f9d9419a5831dedebba8324213ef1 (patch)
treec4886226719d3a8df9a7c55c7468240b3d9ed587
parentfdc262e626173bca787f2298c95dacd25fe1e74c (diff)
downloadakonadi-master.zip
akonadi-master.tar.gz
akonadi-master.tar.xz
backport master/ branch commits to test sqlite backend concurrency supportHEADmaster
-rw-r--r--0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch190
-rw-r--r--0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch167
-rw-r--r--akonadi.spec11
3 files changed, 367 insertions, 1 deletions
diff --git a/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch b/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch
new file mode 100644
index 0000000..5717639
--- /dev/null
+++ b/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch
@@ -0,0 +1,190 @@
+diff --git a/qsqlite/src/qsql_sqlite.cpp b/qsqlite/src/qsql_sqlite.cpp
+index c1e9508..5da232f 100644
+--- a/qsqlite/src/qsql_sqlite.cpp
++++ b/qsqlite/src/qsql_sqlite.cpp
+@@ -528,7 +528,7 @@ static int qGetSqliteOpenMode(QString opts)
+ return SQLITE_OPEN_READONLY;
+ }
+ // The SQLITE_OPEN_NOMUTEX flag causes the database connection to be in the multi-thread mode
+- return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX;
++ return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE;
+ }
+
+ /*
+@@ -543,8 +543,10 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c
+ if (db.isEmpty())
+ return false;
+
++ sqlite3_enable_shared_cache(1);
+ if (sqlite3_open_v2(db.toUtf8().constData(), &d->access, qGetSqliteOpenMode(conOpts), NULL) == SQLITE_OK) {
+ sqlite3_busy_timeout(d->access, qGetSqliteTimeout(conOpts));
++ sqlite3_extended_result_codes(d->access, 1);
+ setOpen(true);
+ setOpenError(false);
+ return true;
+diff --git a/qsqlite/src/sqlite_blocking.cpp b/qsqlite/src/sqlite_blocking.cpp
+index c0fe3f2..180685c 100644
+--- a/qsqlite/src/sqlite_blocking.cpp
++++ b/qsqlite/src/sqlite_blocking.cpp
+@@ -1,63 +1,94 @@
++/*
++ Copyright (c) 2009 Bertjan Broeksema <broeksema@kde.org>
++ Copyright (c) 2014 Daniel Vrátil <dvratil@redhat.com>
++
++ This library is free software; you can redistribute it and/or modify it
++ under the terms of the GNU Library General Public License as published by
++ the Free Software Foundation; either version 2 of the License, or (at your
++ option) any later version.
++
++ This library is distributed in the hope that it will be useful, but WITHOUT
++ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
++ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
++ License for more details.
++
++ You should have received a copy of the GNU Library General Public License
++ along with this library; see the file COPYING.LIB. If not, write to the
++ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
++ 02110-1301, USA.
++*/
++
+ #include "sqlite_blocking.h"
+
+ #include <sqlite3.h>
+-#ifndef _WIN32
+-#include <unistd.h>
+-#else
+-#include <Windows.h>
+-#define usleep(x) Sleep(x/1000)
+-#endif
+-
+-#include "qdebug.h"
++
++#include <QMutex>
++#include <QWaitCondition>
+ #include "qstringbuilder.h"
+ #include "qthread.h"
++#include <QDateTime>
++
++/* Based on example in http://www.sqlite.org/unlock_notify.html */
+
+-QString debugString()
++struct UnlockNotification {
++ bool fired;
++ QWaitCondition cond;
++ QMutex mutex;
++};
++
++static void qSqlite3UnlockNotifyCb(void **apArg, int nArg)
+ {
+- return QString( QLatin1Literal("[QSQLITE3: ") + QString::number( quint64( QThread::currentThreadId() ) ) + QLatin1Literal("] ") );
++ for (int i = 0; i < nArg; ++i) {
++ UnlockNotification *ntf = static_cast<UnlockNotification*>(apArg[i]);
++ ntf->mutex.lock();
++ ntf->fired = true;
++ ntf->cond.wakeOne();
++ ntf->mutex.unlock();
++ }
+ }
+
+-int sqlite3_blocking_step( sqlite3_stmt *pStmt )
++static int qSqlite3WaitForUnlockNotify(sqlite3 *db)
+ {
+- // NOTE: The example at http://www.sqlite.org/unlock_notify.html says to wait
+- // for SQLITE_LOCK but for some reason I don't understand I get
+- // SQLITE_BUSY.
+- int rc = sqlite3_step( pStmt );
+-
+- QThread::currentThreadId();
+- if ( rc == SQLITE_BUSY )
+- qDebug() << debugString() << "sqlite3_blocking_step: Entering while loop";
+-
+- while( rc == SQLITE_BUSY ) {
+- usleep(5000);
+- sqlite3_reset( pStmt );
+- rc = sqlite3_step( pStmt );
+-
+- if ( rc != SQLITE_BUSY ) {
+- qDebug() << debugString() << "sqlite3_blocking_step: Leaving while loop";
++ int rc;
++ UnlockNotification un;
++ un.fired = false;
++
++ rc = sqlite3_unlock_notify(db, qSqlite3UnlockNotifyCb, (void *)&un);
++ Q_ASSERT(rc == SQLITE_LOCKED || rc == SQLITE_OK);
++
++ if (rc == SQLITE_OK) {
++ un.mutex.lock();
++ if (!un.fired) {
++ un.cond.wait(&un.mutex);
+ }
++ un.mutex.unlock();
+ }
+
+ return rc;
+ }
+
+-int sqlite3_blocking_prepare16_v2( sqlite3 *db, /* Database handle. */
+- const void *zSql, /* SQL statement, UTF-16 encoded */
+- int nSql, /* Length of zSql in bytes. */
+- sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */
+- const void **pzTail /* OUT: Pointer to unused portion of zSql */ )
++int sqlite3_blocking_step(sqlite3_stmt *pStmt)
+ {
+- int rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail );
+-
+- if ( rc == SQLITE_BUSY )
+- qDebug() << debugString() << "sqlite3_blocking_prepare16_v2: Entering while loop";
++ int rc;
++ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_step(pStmt))) {
++ rc = qSqlite3WaitForUnlockNotify(sqlite3_db_handle(pStmt));
++ if (rc != SQLITE_OK) {
++ break;
++ }
++ sqlite3_reset(pStmt);
++ }
+
+- while( rc == SQLITE_BUSY ) {
+- usleep(500000);
+- rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail );
++ return rc;
++}
+
+- if ( rc != SQLITE_BUSY ) {
+- qDebug() << debugString() << "sqlite3_prepare16_v2: Leaving while loop";
++int sqlite3_blocking_prepare16_v2(sqlite3 *db, const void *zSql, int nSql,
++ sqlite3_stmt **ppStmt, const void **pzTail)
++{
++ int rc;
++ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_prepare16_v2(db, zSql, nSql, ppStmt, pzTail))) {
++ rc = qSqlite3WaitForUnlockNotify(db);
++ if (rc != SQLITE_OK) {
++ break;
+ }
+ }
+
+diff --git a/qsqlite/src/sqlite_blocking.h b/qsqlite/src/sqlite_blocking.h
+index 0d6f6a0..9f13946 100644
+--- a/qsqlite/src/sqlite_blocking.h
++++ b/qsqlite/src/sqlite_blocking.h
+@@ -1,3 +1,22 @@
++/*
++ Copyright (c) 2009 Bertjan Broeksema <broeksema@kde.org>
++
++ This library is free software; you can redistribute it and/or modify it
++ under the terms of the GNU Library General Public License as published by
++ the Free Software Foundation; either version 2 of the License, or (at your
++ option) any later version.
++
++ This library is distributed in the hope that it will be useful, but WITHOUT
++ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
++ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
++ License for more details.
++
++ You should have received a copy of the GNU Library General Public License
++ along with this library; see the file COPYING.LIB. If not, write to the
++ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
++ 02110-1301, USA.
++*/
++
+ #ifndef SQLITE_BLOCKING_H
+ #define SQLITE_BLOCKING_H
diff --git a/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch b/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch
new file mode 100644
index 0000000..34de4e1
--- /dev/null
+++ b/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch
@@ -0,0 +1,167 @@
+From 24413dc44b0637d6c64e6b2105c2bcf1b99849a5 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Dan=20Vr=C3=A1til?= <dvratil@redhat.com>
+Date: Sun, 6 Apr 2014 19:50:38 +0200
+Subject: [PATCH 13/16] Disable global transaction mutex for QSQLITE3 and
+ enable transaction recording
+
+Our QSQLITE3 driver now supports concurrency, so we don't need to serialize
+transactions in DataStore anymore. It is however still needed for the
+QSQLITE driver shipped with Qt.
+
+Secondary, concurrency support also means possible transactions deadlocks and
+timeouts, so we also need to enable transaction recording and replaying for
+the QSQLITE3 backend.
+---
+ server/src/storage/datastore.cpp | 18 +++++++++++-------
+ server/src/storage/datastore.h | 2 +-
+ server/src/storage/dbtype.cpp | 5 +++++
+ server/src/storage/dbtype.h | 3 +++
+ server/src/storage/querybuilder.cpp | 20 ++++++++++++++++----
+ server/src/storage/querybuilder.h | 2 +-
+ 6 files changed, 37 insertions(+), 13 deletions(-)
+
+diff --git a/server/src/storage/datastore.cpp b/server/src/storage/datastore.cpp
+index 57d1e4e..0f04fa5 100644
+--- a/server/src/storage/datastore.cpp
++++ b/server/src/storage/datastore.cpp
+@@ -61,8 +61,8 @@ using namespace Akonadi::Server;
+ static QMutex sTransactionMutex;
+ bool DataStore::s_hasForeignKeyConstraints = false;
+
+-#define TRANSACTION_MUTEX_LOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.lock()
+-#define TRANSACTION_MUTEX_UNLOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.unlock()
++#define TRANSACTION_MUTEX_LOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.lock()
++#define TRANSACTION_MUTEX_UNLOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.unlock()
+
+ /***************************************************************************
+ * DataStore *
+@@ -1083,23 +1083,27 @@ QDateTime DataStore::dateTimeToQDateTime( const QByteArray &dateTime )
+
+ void DataStore::addQueryToTransaction( const QSqlQuery &query, bool isBatch )
+ {
+- DbType::Type dbType = DbType::type( m_database );
+ // This is used for replaying deadlocked transactions, so only record queries
+ // for backends that support concurrent transactions.
+- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) {
++ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) {
+ return;
+ }
+
+ m_transactionQueries.append( qMakePair( query, isBatch ) );
+ }
+
+-QSqlQuery DataStore::retryLastTransaction()
++QSqlQuery DataStore::retryLastTransaction( bool rollbackFirst )
+ {
+- DbType::Type dbType = DbType::type( m_database );
+- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) {
++ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) {
+ return QSqlQuery();
+ }
+
++ if ( rollbackFirst ) {
++ // In some cases the SQL database won't rollback the failed transaction, so
++ // we need to do it manually
++ m_database.driver()->rollbackTransaction();
++ }
++
+ // The database has rolled back the actual transaction, so reset the counter
+ // to 0 and start a new one in beginTransaction(). Then restore the level
+ // because this has to be completely transparent to the original caller
+diff --git a/server/src/storage/datastore.h b/server/src/storage/datastore.h
+index 8b4a2b7..8a0fe01 100644
+--- a/server/src/storage/datastore.h
++++ b/server/src/storage/datastore.h
+@@ -317,7 +317,7 @@ protected:
+ * @return Returns an invalid query when error occurs, or the last replayed
+ * query on success.
+ */
+- QSqlQuery retryLastTransaction();
++ QSqlQuery retryLastTransaction( bool rollbackFirst );
+
+ private Q_SLOTS:
+ void sendKeepAliveQuery();
+diff --git a/server/src/storage/dbtype.cpp b/server/src/storage/dbtype.cpp
+index 495f532..7df2fb1 100644
+--- a/server/src/storage/dbtype.cpp
++++ b/server/src/storage/dbtype.cpp
+@@ -39,3 +39,8 @@ DbType::Type DbType::typeForDriverName( const QString &driverName )
+ }
+ return Unknown;
+ }
++
++bool DbType::isSystemSQLite( const QSqlDatabase &db )
++{
++ return db.driverName() == QLatin1String( "QSQLITE" );
++}
+diff --git a/server/src/storage/dbtype.h b/server/src/storage/dbtype.h
+index a95a833..3595604 100644
+--- a/server/src/storage/dbtype.h
++++ b/server/src/storage/dbtype.h
+@@ -42,6 +42,9 @@ namespace DbType
+ /** Returns the type for the given driver name. */
+ Type typeForDriverName( const QString &driverName );
+
++ /** Returns true when using QSQLITE driver shipped with Qt, FALSE otherwise */
++ bool isSystemSQLite( const QSqlDatabase &db );
++
+ } // namespace DbType
+ } // namespace Server
+ } // namespace Akonadi
+diff --git a/server/src/storage/querybuilder.cpp b/server/src/storage/querybuilder.cpp
+index 0abad4a..0530b11 100644
+--- a/server/src/storage/querybuilder.cpp
++++ b/server/src/storage/querybuilder.cpp
+@@ -320,10 +320,10 @@ QString QueryBuilder::buildQuery()
+ return statement;
+ }
+
+-bool QueryBuilder::retryLastTransaction()
++bool QueryBuilder::retryLastTransaction( bool rollback )
+ {
+ #ifndef QUERYBUILDER_UNITTEST
+- mQuery = DataStore::self()->retryLastTransaction();
++ mQuery = DataStore::self()->retryLastTransaction( rollback );
+ return !mQuery.lastError().isValid();
+ #else
+ return true;
+@@ -400,9 +400,21 @@ bool QueryBuilder::exec()
+ akDebug() << mQuery.lastError().text();
+ return retryLastTransaction();
+ }
++ } else if ( mDatabaseType == DbType::Sqlite && !DbType::isSystemSQLite( DataStore::self()->database() ) ) {
++ const int error = mQuery.lastError().number();
++ if ( error == 6 /* SQLITE_LOCKED */ ) {
++ akDebug() << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction";
++ akDebug() << mQuery.lastError().text();
++ return retryLastTransaction();
++ } else if ( error == 5 /* SQLITE_BUSY */ ) {
++ akDebug() << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction";
++ akDebug() << mQuery.lastError().text();
++ return retryLastTransaction( true );
++ }
+ } else if ( mDatabaseType == DbType::Sqlite ) {
+- // We can't have a transaction deadlock in SQLite, because it does not support
+- // concurrent transactions and DataStore serializes them through a global lock.
++ // We can't have a transaction deadlock in SQLite when using driver shipped
++ // with Qt, because it does not support concurrent transactions and DataStore
++ // serializes them through a global lock.
+ }
+
+ akError() << "DATABASE ERROR:";
+diff --git a/server/src/storage/querybuilder.h b/server/src/storage/querybuilder.h
+index 235a099..b380f93 100644
+--- a/server/src/storage/querybuilder.h
++++ b/server/src/storage/querybuilder.h
+@@ -244,7 +244,7 @@ class QueryBuilder
+ */
+ void sqliteAdaptUpdateJoin( Query::Condition &cond );
+
+- bool retryLastTransaction();
++ bool retryLastTransaction( bool rollback = false);
+
+ private:
+ QString mTable;
+--
+1.9.0
+
diff --git a/akonadi.spec b/akonadi.spec
index 3f2aa3f..936333e 100644
--- a/akonadi.spec
+++ b/akonadi.spec
@@ -19,7 +19,7 @@
Summary: PIM Storage Service
Name: akonadi
Version: 1.12.1
-Release: 3%{?dist}
+Release: 4%{?dist}
License: LGPLv2+
URL: http://community.kde.org/KDE_PIM/Akonadi
@@ -38,6 +38,9 @@ Source10: akonadiserverrc.mysql
## upstreamable patches
## upstream patches
+# master branch
+Patch212: 0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch
+Patch213: 0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch
%define mysql_conf_timestamp 20140415
@@ -94,6 +97,9 @@ See also: %{_sysconfdir}/akonadi/mysql-global.conf
%prep
%setup -q -n akonadi-%{version}
+%patch212 -p1 -b .0012
+%patch213 -p1 -b .0013
+
%build
mkdir -p %{_target_platform}
@@ -198,6 +204,9 @@ fi
%changelog
+* Wed Apr 16 2014 Rex Dieter <rdieter@fedoraproject.org> 1.12.1-4
+- backport master/ branch commits to test sqlite backend concurrency support
+
* Wed Apr 16 2014 Rex Dieter <rdieter@fedoraproject.org> 1.12.1-3
- WITH_SOPRANO=OFF (kde-4.13,fc21+)