From fcd69dcca69f9d9419a5831dedebba8324213ef1 Mon Sep 17 00:00:00 2001 From: Rex Dieter Date: Apr 16 2014 14:23:26 +0000 Subject: backport master/ branch commits to test sqlite backend concurrency support --- diff --git a/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch b/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch new file mode 100644 index 0000000..5717639 --- /dev/null +++ b/0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch @@ -0,0 +1,190 @@ +diff --git a/qsqlite/src/qsql_sqlite.cpp b/qsqlite/src/qsql_sqlite.cpp +index c1e9508..5da232f 100644 +--- a/qsqlite/src/qsql_sqlite.cpp ++++ b/qsqlite/src/qsql_sqlite.cpp +@@ -528,7 +528,7 @@ static int qGetSqliteOpenMode(QString opts) + return SQLITE_OPEN_READONLY; + } + // The SQLITE_OPEN_NOMUTEX flag causes the database connection to be in the multi-thread mode +- return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX; ++ return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; + } + + /* +@@ -543,8 +543,10 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c + if (db.isEmpty()) + return false; + ++ sqlite3_enable_shared_cache(1); + if (sqlite3_open_v2(db.toUtf8().constData(), &d->access, qGetSqliteOpenMode(conOpts), NULL) == SQLITE_OK) { + sqlite3_busy_timeout(d->access, qGetSqliteTimeout(conOpts)); ++ sqlite3_extended_result_codes(d->access, 1); + setOpen(true); + setOpenError(false); + return true; +diff --git a/qsqlite/src/sqlite_blocking.cpp b/qsqlite/src/sqlite_blocking.cpp +index c0fe3f2..180685c 100644 +--- a/qsqlite/src/sqlite_blocking.cpp ++++ b/qsqlite/src/sqlite_blocking.cpp +@@ -1,63 +1,94 @@ ++/* ++ Copyright (c) 2009 Bertjan Broeksema ++ Copyright (c) 2014 Daniel Vrátil ++ ++ This library is free software; you can redistribute it and/or modify it ++ under the terms of the GNU Library General Public License as published by ++ the Free Software Foundation; either version 2 of the License, or (at your ++ option) any later version. ++ ++ This library is distributed in the hope that it will be useful, but WITHOUT ++ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public ++ License for more details. ++ ++ You should have received a copy of the GNU Library General Public License ++ along with this library; see the file COPYING.LIB. If not, write to the ++ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA ++ 02110-1301, USA. ++*/ ++ + #include "sqlite_blocking.h" + + #include +-#ifndef _WIN32 +-#include +-#else +-#include +-#define usleep(x) Sleep(x/1000) +-#endif +- +-#include "qdebug.h" ++ ++#include ++#include + #include "qstringbuilder.h" + #include "qthread.h" ++#include ++ ++/* Based on example in http://www.sqlite.org/unlock_notify.html */ + +-QString debugString() ++struct UnlockNotification { ++ bool fired; ++ QWaitCondition cond; ++ QMutex mutex; ++}; ++ ++static void qSqlite3UnlockNotifyCb(void **apArg, int nArg) + { +- return QString( QLatin1Literal("[QSQLITE3: ") + QString::number( quint64( QThread::currentThreadId() ) ) + QLatin1Literal("] ") ); ++ for (int i = 0; i < nArg; ++i) { ++ UnlockNotification *ntf = static_cast(apArg[i]); ++ ntf->mutex.lock(); ++ ntf->fired = true; ++ ntf->cond.wakeOne(); ++ ntf->mutex.unlock(); ++ } + } + +-int sqlite3_blocking_step( sqlite3_stmt *pStmt ) ++static int qSqlite3WaitForUnlockNotify(sqlite3 *db) + { +- // NOTE: The example at http://www.sqlite.org/unlock_notify.html says to wait +- // for SQLITE_LOCK but for some reason I don't understand I get +- // SQLITE_BUSY. +- int rc = sqlite3_step( pStmt ); +- +- QThread::currentThreadId(); +- if ( rc == SQLITE_BUSY ) +- qDebug() << debugString() << "sqlite3_blocking_step: Entering while loop"; +- +- while( rc == SQLITE_BUSY ) { +- usleep(5000); +- sqlite3_reset( pStmt ); +- rc = sqlite3_step( pStmt ); +- +- if ( rc != SQLITE_BUSY ) { +- qDebug() << debugString() << "sqlite3_blocking_step: Leaving while loop"; ++ int rc; ++ UnlockNotification un; ++ un.fired = false; ++ ++ rc = sqlite3_unlock_notify(db, qSqlite3UnlockNotifyCb, (void *)&un); ++ Q_ASSERT(rc == SQLITE_LOCKED || rc == SQLITE_OK); ++ ++ if (rc == SQLITE_OK) { ++ un.mutex.lock(); ++ if (!un.fired) { ++ un.cond.wait(&un.mutex); + } ++ un.mutex.unlock(); + } + + return rc; + } + +-int sqlite3_blocking_prepare16_v2( sqlite3 *db, /* Database handle. */ +- const void *zSql, /* SQL statement, UTF-16 encoded */ +- int nSql, /* Length of zSql in bytes. */ +- sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ +- const void **pzTail /* OUT: Pointer to unused portion of zSql */ ) ++int sqlite3_blocking_step(sqlite3_stmt *pStmt) + { +- int rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail ); +- +- if ( rc == SQLITE_BUSY ) +- qDebug() << debugString() << "sqlite3_blocking_prepare16_v2: Entering while loop"; ++ int rc; ++ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_step(pStmt))) { ++ rc = qSqlite3WaitForUnlockNotify(sqlite3_db_handle(pStmt)); ++ if (rc != SQLITE_OK) { ++ break; ++ } ++ sqlite3_reset(pStmt); ++ } + +- while( rc == SQLITE_BUSY ) { +- usleep(500000); +- rc = sqlite3_prepare16_v2( db, zSql, nSql, ppStmt, pzTail ); ++ return rc; ++} + +- if ( rc != SQLITE_BUSY ) { +- qDebug() << debugString() << "sqlite3_prepare16_v2: Leaving while loop"; ++int sqlite3_blocking_prepare16_v2(sqlite3 *db, const void *zSql, int nSql, ++ sqlite3_stmt **ppStmt, const void **pzTail) ++{ ++ int rc; ++ while (SQLITE_LOCKED_SHAREDCACHE == (rc = sqlite3_prepare16_v2(db, zSql, nSql, ppStmt, pzTail))) { ++ rc = qSqlite3WaitForUnlockNotify(db); ++ if (rc != SQLITE_OK) { ++ break; + } + } + +diff --git a/qsqlite/src/sqlite_blocking.h b/qsqlite/src/sqlite_blocking.h +index 0d6f6a0..9f13946 100644 +--- a/qsqlite/src/sqlite_blocking.h ++++ b/qsqlite/src/sqlite_blocking.h +@@ -1,3 +1,22 @@ ++/* ++ Copyright (c) 2009 Bertjan Broeksema ++ ++ This library is free software; you can redistribute it and/or modify it ++ under the terms of the GNU Library General Public License as published by ++ the Free Software Foundation; either version 2 of the License, or (at your ++ option) any later version. ++ ++ This library is distributed in the hope that it will be useful, but WITHOUT ++ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or ++ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public ++ License for more details. ++ ++ You should have received a copy of the GNU Library General Public License ++ along with this library; see the file COPYING.LIB. If not, write to the ++ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA ++ 02110-1301, USA. ++*/ ++ + #ifndef SQLITE_BLOCKING_H + #define SQLITE_BLOCKING_H diff --git a/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch b/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch new file mode 100644 index 0000000..34de4e1 --- /dev/null +++ b/0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch @@ -0,0 +1,167 @@ +From 24413dc44b0637d6c64e6b2105c2bcf1b99849a5 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Dan=20Vr=C3=A1til?= +Date: Sun, 6 Apr 2014 19:50:38 +0200 +Subject: [PATCH 13/16] Disable global transaction mutex for QSQLITE3 and + enable transaction recording + +Our QSQLITE3 driver now supports concurrency, so we don't need to serialize +transactions in DataStore anymore. It is however still needed for the +QSQLITE driver shipped with Qt. + +Secondary, concurrency support also means possible transactions deadlocks and +timeouts, so we also need to enable transaction recording and replaying for +the QSQLITE3 backend. +--- + server/src/storage/datastore.cpp | 18 +++++++++++------- + server/src/storage/datastore.h | 2 +- + server/src/storage/dbtype.cpp | 5 +++++ + server/src/storage/dbtype.h | 3 +++ + server/src/storage/querybuilder.cpp | 20 ++++++++++++++++---- + server/src/storage/querybuilder.h | 2 +- + 6 files changed, 37 insertions(+), 13 deletions(-) + +diff --git a/server/src/storage/datastore.cpp b/server/src/storage/datastore.cpp +index 57d1e4e..0f04fa5 100644 +--- a/server/src/storage/datastore.cpp ++++ b/server/src/storage/datastore.cpp +@@ -61,8 +61,8 @@ using namespace Akonadi::Server; + static QMutex sTransactionMutex; + bool DataStore::s_hasForeignKeyConstraints = false; + +-#define TRANSACTION_MUTEX_LOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.lock() +-#define TRANSACTION_MUTEX_UNLOCK if ( DbType::type( m_database ) == DbType::Sqlite ) sTransactionMutex.unlock() ++#define TRANSACTION_MUTEX_LOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.lock() ++#define TRANSACTION_MUTEX_UNLOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.unlock() + + /*************************************************************************** + * DataStore * +@@ -1083,23 +1083,27 @@ QDateTime DataStore::dateTimeToQDateTime( const QByteArray &dateTime ) + + void DataStore::addQueryToTransaction( const QSqlQuery &query, bool isBatch ) + { +- DbType::Type dbType = DbType::type( m_database ); + // This is used for replaying deadlocked transactions, so only record queries + // for backends that support concurrent transactions. +- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) { ++ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) { + return; + } + + m_transactionQueries.append( qMakePair( query, isBatch ) ); + } + +-QSqlQuery DataStore::retryLastTransaction() ++QSqlQuery DataStore::retryLastTransaction( bool rollbackFirst ) + { +- DbType::Type dbType = DbType::type( m_database ); +- if ( !inTransaction() || ( dbType != DbType::MySQL && dbType != DbType::PostgreSQL ) ) { ++ if ( !inTransaction() || DbType::isSystemSQLite( m_database ) ) { + return QSqlQuery(); + } + ++ if ( rollbackFirst ) { ++ // In some cases the SQL database won't rollback the failed transaction, so ++ // we need to do it manually ++ m_database.driver()->rollbackTransaction(); ++ } ++ + // The database has rolled back the actual transaction, so reset the counter + // to 0 and start a new one in beginTransaction(). Then restore the level + // because this has to be completely transparent to the original caller +diff --git a/server/src/storage/datastore.h b/server/src/storage/datastore.h +index 8b4a2b7..8a0fe01 100644 +--- a/server/src/storage/datastore.h ++++ b/server/src/storage/datastore.h +@@ -317,7 +317,7 @@ protected: + * @return Returns an invalid query when error occurs, or the last replayed + * query on success. + */ +- QSqlQuery retryLastTransaction(); ++ QSqlQuery retryLastTransaction( bool rollbackFirst ); + + private Q_SLOTS: + void sendKeepAliveQuery(); +diff --git a/server/src/storage/dbtype.cpp b/server/src/storage/dbtype.cpp +index 495f532..7df2fb1 100644 +--- a/server/src/storage/dbtype.cpp ++++ b/server/src/storage/dbtype.cpp +@@ -39,3 +39,8 @@ DbType::Type DbType::typeForDriverName( const QString &driverName ) + } + return Unknown; + } ++ ++bool DbType::isSystemSQLite( const QSqlDatabase &db ) ++{ ++ return db.driverName() == QLatin1String( "QSQLITE" ); ++} +diff --git a/server/src/storage/dbtype.h b/server/src/storage/dbtype.h +index a95a833..3595604 100644 +--- a/server/src/storage/dbtype.h ++++ b/server/src/storage/dbtype.h +@@ -42,6 +42,9 @@ namespace DbType + /** Returns the type for the given driver name. */ + Type typeForDriverName( const QString &driverName ); + ++ /** Returns true when using QSQLITE driver shipped with Qt, FALSE otherwise */ ++ bool isSystemSQLite( const QSqlDatabase &db ); ++ + } // namespace DbType + } // namespace Server + } // namespace Akonadi +diff --git a/server/src/storage/querybuilder.cpp b/server/src/storage/querybuilder.cpp +index 0abad4a..0530b11 100644 +--- a/server/src/storage/querybuilder.cpp ++++ b/server/src/storage/querybuilder.cpp +@@ -320,10 +320,10 @@ QString QueryBuilder::buildQuery() + return statement; + } + +-bool QueryBuilder::retryLastTransaction() ++bool QueryBuilder::retryLastTransaction( bool rollback ) + { + #ifndef QUERYBUILDER_UNITTEST +- mQuery = DataStore::self()->retryLastTransaction(); ++ mQuery = DataStore::self()->retryLastTransaction( rollback ); + return !mQuery.lastError().isValid(); + #else + return true; +@@ -400,9 +400,21 @@ bool QueryBuilder::exec() + akDebug() << mQuery.lastError().text(); + return retryLastTransaction(); + } ++ } else if ( mDatabaseType == DbType::Sqlite && !DbType::isSystemSQLite( DataStore::self()->database() ) ) { ++ const int error = mQuery.lastError().number(); ++ if ( error == 6 /* SQLITE_LOCKED */ ) { ++ akDebug() << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; ++ akDebug() << mQuery.lastError().text(); ++ return retryLastTransaction(); ++ } else if ( error == 5 /* SQLITE_BUSY */ ) { ++ akDebug() << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; ++ akDebug() << mQuery.lastError().text(); ++ return retryLastTransaction( true ); ++ } + } else if ( mDatabaseType == DbType::Sqlite ) { +- // We can't have a transaction deadlock in SQLite, because it does not support +- // concurrent transactions and DataStore serializes them through a global lock. ++ // We can't have a transaction deadlock in SQLite when using driver shipped ++ // with Qt, because it does not support concurrent transactions and DataStore ++ // serializes them through a global lock. + } + + akError() << "DATABASE ERROR:"; +diff --git a/server/src/storage/querybuilder.h b/server/src/storage/querybuilder.h +index 235a099..b380f93 100644 +--- a/server/src/storage/querybuilder.h ++++ b/server/src/storage/querybuilder.h +@@ -244,7 +244,7 @@ class QueryBuilder + */ + void sqliteAdaptUpdateJoin( Query::Condition &cond ); + +- bool retryLastTransaction(); ++ bool retryLastTransaction( bool rollback = false); + + private: + QString mTable; +-- +1.9.0 + diff --git a/akonadi.spec b/akonadi.spec index 3f2aa3f..936333e 100644 --- a/akonadi.spec +++ b/akonadi.spec @@ -19,7 +19,7 @@ Summary: PIM Storage Service Name: akonadi Version: 1.12.1 -Release: 3%{?dist} +Release: 4%{?dist} License: LGPLv2+ URL: http://community.kde.org/KDE_PIM/Akonadi @@ -38,6 +38,9 @@ Source10: akonadiserverrc.mysql ## upstreamable patches ## upstream patches +# master branch +Patch212: 0012-Enable-concurrency-in-our-copy-of-QSQLITE-driver.patch +Patch213: 0013-Disable-global-transaction-mutex-for-QSQLITE3-and-en.patch %define mysql_conf_timestamp 20140415 @@ -94,6 +97,9 @@ See also: %{_sysconfdir}/akonadi/mysql-global.conf %prep %setup -q -n akonadi-%{version} +%patch212 -p1 -b .0012 +%patch213 -p1 -b .0013 + %build mkdir -p %{_target_platform} @@ -198,6 +204,9 @@ fi %changelog +* Wed Apr 16 2014 Rex Dieter 1.12.1-4 +- backport master/ branch commits to test sqlite backend concurrency support + * Wed Apr 16 2014 Rex Dieter 1.12.1-3 - WITH_SOPRANO=OFF (kde-4.13,fc21+)