]>
Commit | Line | Data |
---|---|---|
8a8f9fb3 AM |
1 | From c24329bb570ee16c033228588e6d22b0f6000f95 Mon Sep 17 00:00:00 2001 |
2 | From: =?UTF-8?q?Dan=20Vr=C3=A1til?= <dvratil@redhat.com> | |
3 | Date: Fri, 5 Dec 2014 18:23:33 +0100 | |
4 | Subject: [PATCH 22/30] Implement cache for CollectionStatistics to | |
5 | significantly reduce amount of SQL queries | |
6 | ||
7 | Collection statistics are being requested extremely often (basically whenever | |
8 | a PimItem is changed, or when a Collection itself is changed), and it's always | |
9 | requested by at least 5 or so clients (including agents that listen to | |
10 | everything). | |
11 | ||
12 | To decrease the load on database we now cache the Collection statistics and | |
13 | we only invalidate a cache entry when respective collection (or it's content) | |
14 | is changed. The invalidation is invoked from NotificationCollector, which is | |
15 | basically a hack, but performance-wise it's the best place to avoid additional | |
16 | expensive queries. | |
17 | ||
18 | This patch also optimizes the SQL query needed to get up-to-date statistics. | |
19 | We now have only one query to get both full count and read items count, which | |
20 | a bit is faster as the database only has to deal with one large JOIN. | |
21 | ||
22 | Thanks to the cache the number of SQL queries for Collection statistics have | |
23 | reduced by 70%-80%, and average query duration is now between 20 and 80ms | |
24 | depending on average collection size and database used. | |
25 | --- | |
26 | server/CMakeLists.txt | 1 + | |
27 | server/src/handler/link.cpp | 2 +- | |
28 | server/src/handler/merge.cpp | 4 +- | |
29 | server/src/handler/select.cpp | 14 ++-- | |
30 | server/src/handler/status.cpp | 20 ++--- | |
31 | server/src/handlerhelper.cpp | 81 ++------------------ | |
32 | server/src/handlerhelper.h | 22 ------ | |
33 | server/src/storage/collectionstatistics.cpp | 108 +++++++++++++++++++++++++++ | |
34 | server/src/storage/collectionstatistics.h | 70 +++++++++++++++++ | |
35 | server/src/storage/datastore.cpp | 8 +- | |
36 | server/src/storage/datastore.h | 6 +- | |
37 | server/src/storage/notificationcollector.cpp | 8 ++ | |
38 | server/tests/unittest/fakedatastore.cpp | 8 +- | |
39 | server/tests/unittest/fakedatastore.h | 2 + | |
40 | 14 files changed, 224 insertions(+), 130 deletions(-) | |
41 | create mode 100644 server/src/storage/collectionstatistics.cpp | |
42 | create mode 100644 server/src/storage/collectionstatistics.h | |
43 | ||
44 | diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt | |
45 | index 275938d..f0e0093 100644 | |
46 | --- a/server/CMakeLists.txt | |
47 | +++ b/server/CMakeLists.txt | |
48 | @@ -161,6 +161,7 @@ set(libakonadiprivate_SRCS | |
49 | src/search/searchmanager.cpp | |
50 | ||
51 | src/storage/collectionqueryhelper.cpp | |
52 | + src/storage/collectionstatistics.cpp | |
53 | src/storage/entity.cpp | |
54 | ${CMAKE_CURRENT_BINARY_DIR}/entities.cpp | |
55 | ${CMAKE_CURRENT_BINARY_DIR}/akonadischema.cpp | |
56 | diff --git a/server/src/handler/link.cpp b/server/src/handler/link.cpp | |
57 | index ce18e47..227de11 100644 | |
58 | --- a/server/src/handler/link.cpp | |
59 | +++ b/server/src/handler/link.cpp | |
60 | @@ -25,10 +25,10 @@ | |
61 | #include "storage/itemqueryhelper.h" | |
62 | #include "storage/transaction.h" | |
63 | #include "storage/selectquerybuilder.h" | |
64 | +#include "storage/collectionqueryhelper.h" | |
65 | #include "entities.h" | |
66 | ||
67 | #include "imapstreamparser.h" | |
68 | -#include <storage/collectionqueryhelper.h> | |
69 | ||
70 | using namespace Akonadi::Server; | |
71 | ||
72 | diff --git a/server/src/handler/merge.cpp b/server/src/handler/merge.cpp | |
73 | index c26917d..5149916 100644 | |
74 | --- a/server/src/handler/merge.cpp | |
75 | +++ b/server/src/handler/merge.cpp | |
76 | @@ -88,7 +88,7 @@ bool Merge::mergeItem( PimItem &newItem, PimItem ¤tItem, | |
77 | if ( !itemFlags.removed.isEmpty() ) { | |
78 | const Flag::List removedFlags = HandlerHelper::resolveFlags( itemFlags.removed ); | |
79 | DataStore::self()->removeItemsFlags( PimItem::List() << currentItem, removedFlags, | |
80 | - &flagsRemoved, true ); | |
81 | + &flagsRemoved, col, true ); | |
82 | } | |
83 | ||
84 | if ( flagsAdded || flagsRemoved ) { | |
85 | @@ -98,7 +98,7 @@ bool Merge::mergeItem( PimItem &newItem, PimItem ¤tItem, | |
86 | bool flagsChanged = false; | |
87 | const Flag::List flags = HandlerHelper::resolveFlags( itemFlags.added ); | |
88 | DataStore::self()->setItemsFlags( PimItem::List() << currentItem, flags, | |
89 | - &flagsChanged, true ); | |
90 | + &flagsChanged, col, true ); | |
91 | if ( flagsChanged ) { | |
92 | mChangedParts << AKONADI_PARAM_FLAGS; | |
93 | } | |
94 | diff --git a/server/src/handler/select.cpp b/server/src/handler/select.cpp | |
95 | index 1c5dd8a..f1ecc44 100644 | |
96 | --- a/server/src/handler/select.cpp | |
97 | +++ b/server/src/handler/select.cpp | |
98 | @@ -27,6 +27,7 @@ | |
99 | #include "handlerhelper.h" | |
100 | #include "imapstreamparser.h" | |
101 | #include "storage/selectquerybuilder.h" | |
102 | +#include "storage/collectionstatistics.h" | |
103 | #include "commandcontext.h" | |
104 | ||
105 | #include "response.h" | |
106 | @@ -96,19 +97,14 @@ bool Select::parseStream() | |
107 | response.setString( "FLAGS (" + Flag::joinByName( Flag::retrieveAll(), QLatin1String( " " ) ).toLatin1() + ")" ); | |
108 | Q_EMIT responseAvailable( response ); | |
109 | ||
110 | - const int itemCount = HandlerHelper::itemCount( col ); | |
111 | - if ( itemCount < 0 ) { | |
112 | + const CollectionStatistics::Statistics stats = CollectionStatistics::instance()->statistics(col); | |
113 | + if ( stats.count == -1 ) { | |
114 | return failureResponse( "Unable to determine item count" ); | |
115 | } | |
116 | - response.setString( QByteArray::number( itemCount ) + " EXISTS" ); | |
117 | + response.setString( QByteArray::number( stats.count ) + " EXISTS" ); | |
118 | Q_EMIT responseAvailable( response ); | |
119 | ||
120 | - int readCount = HandlerHelper::itemWithFlagsCount( col, QStringList() << QLatin1String( AKONADI_FLAG_SEEN ) | |
121 | - << QLatin1String( AKONADI_FLAG_IGNORED ) ); | |
122 | - if ( readCount < 0 || itemCount < readCount ) { | |
123 | - return failureResponse( "Unable to retrieve unseen count" ); | |
124 | - } | |
125 | - response.setString( "OK [UNSEEN " + QByteArray::number( itemCount - readCount ) + "] Message 0 is first unseen" ); | |
126 | + response.setString( "OK [UNSEEN " + QByteArray::number( stats.count - stats.read ) + "] Message 0 is first unseen" ); | |
127 | Q_EMIT responseAvailable( response ); | |
128 | } | |
129 | ||
130 | diff --git a/server/src/handler/status.cpp b/server/src/handler/status.cpp | |
131 | index 8c6823d..283532c 100644 | |
132 | --- a/server/src/handler/status.cpp | |
133 | +++ b/server/src/handler/status.cpp | |
134 | @@ -25,6 +25,7 @@ | |
135 | #include "storage/datastore.h" | |
136 | #include "storage/entity.h" | |
137 | #include "storage/countquerybuilder.h" | |
138 | +#include "storage/collectionstatistics.h" | |
139 | ||
140 | #include "response.h" | |
141 | #include "handlerhelper.h" | |
142 | @@ -62,9 +63,9 @@ bool Status::parseStream() | |
143 | // Responses: | |
144 | // REQUIRED untagged responses: STATUS | |
145 | ||
146 | - qint64 itemCount, itemSize; | |
147 | - if ( !HandlerHelper::itemStatistics( col, itemCount, itemSize ) ) { | |
148 | - return failureResponse( "Failed to query statistics." ); | |
149 | + const CollectionStatistics::Statistics &stats = CollectionStatistics::instance()->statistics(col); | |
150 | + if (stats.count == -1) { | |
151 | + return failureResponse( "Failed to query statistics." ); | |
152 | } | |
153 | ||
154 | // build STATUS response | |
155 | @@ -72,7 +73,7 @@ bool Status::parseStream() | |
156 | // MESSAGES - The number of messages in the mailbox | |
157 | if ( attributeList.contains( AKONADI_ATTRIBUTE_MESSAGES ) ) { | |
158 | statusResponse += AKONADI_ATTRIBUTE_MESSAGES " "; | |
159 | - statusResponse += QByteArray::number( itemCount ); | |
160 | + statusResponse += QByteArray::number( stats.count ); | |
161 | } | |
162 | ||
163 | if ( attributeList.contains( AKONADI_ATTRIBUTE_UNSEEN ) ) { | |
164 | @@ -80,21 +81,14 @@ bool Status::parseStream() | |
165 | statusResponse += " "; | |
166 | } | |
167 | statusResponse += AKONADI_ATTRIBUTE_UNSEEN " "; | |
168 | - | |
169 | - // itemWithFlagCount is twice as fast as itemWithoutFlagCount... | |
170 | - const int count = HandlerHelper::itemWithFlagsCount( col, QStringList() << QLatin1String( AKONADI_FLAG_SEEN ) | |
171 | - << QLatin1String( AKONADI_FLAG_IGNORED ) ); | |
172 | - if ( count < 0 ) { | |
173 | - return failureResponse( "Unable to retrieve unread count" ); | |
174 | - } | |
175 | - statusResponse += QByteArray::number( itemCount - count ); | |
176 | + statusResponse += QByteArray::number( stats.count - stats.read ); | |
177 | } | |
178 | if ( attributeList.contains( AKONADI_PARAM_SIZE ) ) { | |
179 | if ( !statusResponse.isEmpty() ) { | |
180 | statusResponse += " "; | |
181 | } | |
182 | statusResponse += AKONADI_PARAM_SIZE " "; | |
183 | - statusResponse += QByteArray::number( itemSize ); | |
184 | + statusResponse += QByteArray::number( stats.size ); | |
185 | } | |
186 | ||
187 | Response response; | |
188 | diff --git a/server/src/handlerhelper.cpp b/server/src/handlerhelper.cpp | |
189 | index 82347b4..39583ce 100644 | |
190 | --- a/server/src/handlerhelper.cpp | |
191 | +++ b/server/src/handlerhelper.cpp | |
192 | @@ -22,6 +22,7 @@ | |
193 | #include "storage/countquerybuilder.h" | |
194 | #include "storage/datastore.h" | |
195 | #include "storage/selectquerybuilder.h" | |
196 | +#include "storage/collectionstatistics.h" | |
197 | #include "storage/queryhelper.h" | |
198 | #include "libs/imapparser_p.h" | |
199 | #include "libs/protocol_p.h" | |
200 | @@ -78,74 +79,6 @@ QString HandlerHelper::pathForCollection( const Collection &col ) | |
201 | return parts.join( QLatin1String( "/" ) ); | |
202 | } | |
203 | ||
204 | -bool HandlerHelper::itemStatistics( const Collection &col, qint64 &count, qint64 &size ) | |
205 | -{ | |
206 | - QueryBuilder qb( PimItem::tableName() ); | |
207 | - qb.addAggregation( PimItem::idColumn(), QLatin1String( "count" ) ); | |
208 | - qb.addAggregation( PimItem::sizeColumn(), QLatin1String( "sum" ) ); | |
209 | - | |
210 | - if ( col.isVirtual() ) { | |
211 | - qb.addJoin( QueryBuilder::InnerJoin, CollectionPimItemRelation::tableName(), | |
212 | - CollectionPimItemRelation::rightFullColumnName(), PimItem::idFullColumnName() ); | |
213 | - qb.addValueCondition( CollectionPimItemRelation::leftFullColumnName(), Query::Equals, col.id() ); | |
214 | - } else { | |
215 | - qb.addValueCondition( PimItem::collectionIdColumn(), Query::Equals, col.id() ); | |
216 | - } | |
217 | - | |
218 | - if ( !qb.exec() ) { | |
219 | - return false; | |
220 | - } | |
221 | - if ( !qb.query().next() ) { | |
222 | - akError() << "Error during retrieving result of statistics query:" << qb.query().lastError().text(); | |
223 | - return false; | |
224 | - } | |
225 | - count = qb.query().value( 0 ).toLongLong(); | |
226 | - size = qb.query().value( 1 ).toLongLong(); | |
227 | - return true; | |
228 | -} | |
229 | - | |
230 | -int HandlerHelper::itemWithFlagsCount( const Collection &col, const QStringList &flags ) | |
231 | -{ | |
232 | - CountQueryBuilder qb( PimItem::tableName(), PimItem::idFullColumnName(), CountQueryBuilder::Distinct ); | |
233 | - qb.addJoin( QueryBuilder::InnerJoin, PimItemFlagRelation::tableName(), | |
234 | - PimItem::idFullColumnName(), PimItemFlagRelation::leftFullColumnName() ); | |
235 | - if ( col.isVirtual() ) { | |
236 | - qb.addJoin( QueryBuilder::InnerJoin, CollectionPimItemRelation::tableName(), | |
237 | - CollectionPimItemRelation::rightFullColumnName(), PimItem::idFullColumnName() ); | |
238 | - qb.addValueCondition( CollectionPimItemRelation::leftFullColumnName(), Query::Equals, col.id() ); | |
239 | - } else { | |
240 | - qb.addValueCondition( PimItem::collectionIdFullColumnName(), Query::Equals, col.id() ); | |
241 | - } | |
242 | - Query::Condition cond( Query::Or ); | |
243 | - // We use the below instead of an inner join in the query above because postgres seems | |
244 | - // to struggle to optimize the two inner joins, despite having indices that should | |
245 | - // facilitate that. This exploits the fact that the Flag::retrieveByName is fast because | |
246 | - // it hits an in-memory cache. | |
247 | - Q_FOREACH ( const QString &flag, flags ) { | |
248 | - const Flag f = Flag::retrieveByName( flag ); | |
249 | - if (!f.isValid()) { | |
250 | - // since we OR this condition, we can skip invalid flags to speed up the query | |
251 | - continue; | |
252 | - } | |
253 | - cond.addValueCondition( PimItemFlagRelation::rightFullColumnName(), Query::Equals, f.id() ); | |
254 | - } | |
255 | - qb.addCondition( cond ); | |
256 | - if ( !qb.exec() ) { | |
257 | - return -1; | |
258 | - } | |
259 | - return qb.result(); | |
260 | -} | |
261 | - | |
262 | -int HandlerHelper::itemCount( const Collection &col ) | |
263 | -{ | |
264 | - CountQueryBuilder qb( PimItem::tableName() ); | |
265 | - qb.addValueCondition( PimItem::collectionIdColumn(), Query::Equals, col.id() ); | |
266 | - if ( !qb.exec() ) { | |
267 | - return -1; | |
268 | - } | |
269 | - return qb.result(); | |
270 | -} | |
271 | - | |
272 | int HandlerHelper::parseCachePolicy( const QByteArray &data, Collection &col, int start, bool *changed ) | |
273 | { | |
274 | bool inheritChanged = false; | |
275 | @@ -233,14 +166,12 @@ QByteArray HandlerHelper::collectionToByteArray( const Collection &col, bool hid | |
276 | b += " " AKONADI_PARAM_VIRTUAL " " + QByteArray::number( col.isVirtual() ) + ' '; | |
277 | ||
278 | if ( includeStatistics ) { | |
279 | - qint64 itemCount, itemSize; | |
280 | - if ( itemStatistics( col, itemCount, itemSize ) ) { | |
281 | - b += AKONADI_ATTRIBUTE_MESSAGES " " + QByteArray::number( itemCount ) + ' '; | |
282 | - // itemWithFlagCount is twice as fast as itemWithoutFlagCount, so emulated that... | |
283 | + const CollectionStatistics::Statistics &stats = CollectionStatistics::instance()->statistics(col); | |
284 | + if (stats.count > -1) { | |
285 | + b += AKONADI_ATTRIBUTE_MESSAGES " " + QByteArray::number( stats.count ) + ' '; | |
286 | b += AKONADI_ATTRIBUTE_UNSEEN " "; | |
287 | - b += QByteArray::number( itemCount - itemWithFlagsCount( col, QStringList() << QLatin1String( AKONADI_FLAG_SEEN ) | |
288 | - << QLatin1String( AKONADI_FLAG_IGNORED ) ) ); | |
289 | - b += " " AKONADI_PARAM_SIZE " " + QByteArray::number( itemSize ) + ' '; | |
290 | + b += QByteArray::number( stats.count - stats.read) ; | |
291 | + b += " " AKONADI_PARAM_SIZE " " + QByteArray::number( stats.size ) + ' '; | |
292 | } | |
293 | } | |
294 | ||
295 | diff --git a/server/src/handlerhelper.h b/server/src/handlerhelper.h | |
296 | index 22e6e1c..cf9ac22 100644 | |
297 | --- a/server/src/handlerhelper.h | |
298 | +++ b/server/src/handlerhelper.h | |
299 | @@ -52,28 +52,6 @@ class HandlerHelper | |
300 | static QString pathForCollection( const Collection &col ); | |
301 | ||
302 | /** | |
303 | - Returns the amount of existing items in the given collection. | |
304 | - @return -1 on error | |
305 | - */ | |
306 | - static int itemCount( const Collection &col ); | |
307 | - | |
308 | - /** | |
309 | - * Queries for collection statistics. | |
310 | - * @param col The collection to query. | |
311 | - * @param count The total amount of items in this collection. | |
312 | - * @param size The size of all items in this collection. | |
313 | - * @return @c false on a query error, @c true otherwise | |
314 | - */ | |
315 | - static bool itemStatistics( const Collection &col, qint64 &count, qint64 &size ); | |
316 | - | |
317 | - /** | |
318 | - Returns the amount of existing items in the given collection | |
319 | - which have a given flag set. | |
320 | - @return -1 on error. | |
321 | - */ | |
322 | - static int itemWithFlagsCount( const Collection &col, const QStringList &flags ); | |
323 | - | |
324 | - /** | |
325 | Parse cache policy and update the given Collection object accoordingly. | |
326 | @param changed Indicates whether or not the cache policy already available in @p col | |
327 | has actually changed | |
328 | diff --git a/server/src/storage/collectionstatistics.cpp b/server/src/storage/collectionstatistics.cpp | |
329 | new file mode 100644 | |
330 | index 0000000..85ee449 | |
331 | --- /dev/null | |
332 | +++ b/server/src/storage/collectionstatistics.cpp | |
333 | @@ -0,0 +1,108 @@ | |
334 | +/* | |
335 | + * Copyright (C) 2014 Daniel VrĂĄtil <dvratil@redhat.com> | |
336 | + * | |
337 | + * This library is free software; you can redistribute it and/or | |
338 | + * modify it under the terms of the GNU Lesser General Public | |
339 | + * License as published by the Free Software Foundation; either | |
340 | + * version 2.1 of the License, or (at your option) any later version. | |
341 | + * | |
342 | + * This library is distributed in the hope that it will be useful, | |
343 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
344 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
345 | + * Lesser General Public License for more details. | |
346 | + * | |
347 | + * You should have received a copy of the GNU Lesser General Public | |
348 | + * License along with this library; if not, write to the Free Software | |
349 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
350 | + * | |
351 | + */ | |
352 | + | |
353 | +#include "collectionstatistics.h" | |
354 | +#include "querybuilder.h" | |
355 | +#include "countquerybuilder.h" | |
356 | +#include "akdebug.h" | |
357 | +#include "entities.h" | |
358 | + | |
359 | +#include <libs/protocol_p.h> | |
360 | + | |
361 | +#include <QDateTime> | |
362 | + | |
363 | +using namespace Akonadi::Server; | |
364 | + | |
365 | +CollectionStatistics *CollectionStatistics::sInstance = 0; | |
366 | + | |
367 | +CollectionStatistics* CollectionStatistics::instance() | |
368 | +{ | |
369 | + static QMutex lock; | |
370 | + lock.lock(); | |
371 | + if (sInstance == 0) { | |
372 | + sInstance = new CollectionStatistics(); | |
373 | + } | |
374 | + lock.unlock(); | |
375 | + return sInstance; | |
376 | +} | |
377 | + | |
378 | +void CollectionStatistics::invalidateCollection(const Collection &col) | |
379 | +{ | |
380 | + QMutexLocker lock(&mCacheLock); | |
381 | + mCache.remove(col.id()); | |
382 | +} | |
383 | + | |
384 | +const CollectionStatistics::Statistics& CollectionStatistics::statistics(const Collection &col) | |
385 | +{ | |
386 | + QMutexLocker lock(&mCacheLock); | |
387 | + auto it = mCache.find(col.id()); | |
388 | + if (it == mCache.constEnd()) { | |
389 | + it = mCache.insert(col.id(), getCollectionStatistics(col)); | |
390 | + } | |
391 | + return it.value(); | |
392 | +} | |
393 | + | |
394 | +CollectionStatistics::Statistics CollectionStatistics::getCollectionStatistics(const Collection &col) | |
395 | +{ | |
396 | + QueryBuilder qb(PimItem::tableName()); | |
397 | + // COUNT(DISTINCT PimItemTable.id) | |
398 | + qb.addAggregation(QString::fromLatin1("DISTINCT %1") | |
399 | + .arg(PimItem::idFullColumnName()), | |
400 | + QLatin1String("count")); | |
401 | + // SUM(PimItemTable.size) | |
402 | + qb.addAggregation(PimItem::sizeFullColumnName(), QLatin1String("sum")); | |
403 | + // SUM(CASE WHEN FlagTable.name IN ('\SEEN', '$IGNORED') THEN 1 ELSE 0 END) | |
404 | + // This allows us to get read messages count in a single query with the other | |
405 | + // statistics. It is much than doing two queries, because the database | |
406 | + // only has to calculate the JOINs once. | |
407 | + // | |
408 | + // Flag::retrieveByName() will hit the Entity cache, which allows us to avoid | |
409 | + // a second JOIN with FlagTable, which PostgreSQL seems to struggle to optimize. | |
410 | + Query::Condition cond(Query::Or); | |
411 | + cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), | |
412 | + Query::Equals, | |
413 | + Flag::retrieveByName(QLatin1String(AKONADI_FLAG_SEEN)).id()); | |
414 | + cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), | |
415 | + Query::Equals, | |
416 | + Flag::retrieveByName(QLatin1String(AKONADI_FLAG_IGNORED)).id()); | |
417 | + Query::Case caseStmt(cond, QLatin1String("1"), QLatin1String("0")); | |
418 | + qb.addAggregation(caseStmt, QLatin1String("sum")); | |
419 | + | |
420 | + qb.addJoin(QueryBuilder::LeftJoin, PimItemFlagRelation::tableName(), | |
421 | + PimItem::idFullColumnName(), PimItemFlagRelation::leftFullColumnName()); | |
422 | + if (col.isVirtual()) { | |
423 | + qb.addJoin(QueryBuilder::InnerJoin, CollectionPimItemRelation::tableName(), | |
424 | + CollectionPimItemRelation::rightFullColumnName(), PimItem::idFullColumnName()); | |
425 | + qb.addValueCondition(CollectionPimItemRelation::leftFullColumnName(), Query::Equals, col.id()); | |
426 | + } else { | |
427 | + qb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, col.id()); | |
428 | + } | |
429 | + | |
430 | + if (!qb.exec()) { | |
431 | + return { -1, -1, -1 }; | |
432 | + } | |
433 | + if (!qb.query().next()) { | |
434 | + akError() << "Error during retrieving result of statistics query:" << qb.query().lastError().text(); | |
435 | + return { -1, -1, -1 }; | |
436 | + } | |
437 | + | |
438 | + return { qb.query().value(0).toLongLong(), | |
439 | + qb.query().value(1).toLongLong(), | |
440 | + qb.query().value(2).toLongLong() }; | |
441 | +} | |
442 | diff --git a/server/src/storage/collectionstatistics.h b/server/src/storage/collectionstatistics.h | |
443 | new file mode 100644 | |
444 | index 0000000..2c0af6a | |
445 | --- /dev/null | |
446 | +++ b/server/src/storage/collectionstatistics.h | |
447 | @@ -0,0 +1,70 @@ | |
448 | +/* | |
449 | + * Copyright (C) 2014 Daniel VrĂĄtil <dvratil@redhat.com> | |
450 | + * | |
451 | + * This library is free software; you can redistribute it and/or | |
452 | + * modify it under the terms of the GNU Lesser General Public | |
453 | + * License as published by the Free Software Foundation; either | |
454 | + * version 2.1 of the License, or (at your option) any later version. | |
455 | + * | |
456 | + * This library is distributed in the hope that it will be useful, | |
457 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
458 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
459 | + * Lesser General Public License for more details. | |
460 | + * | |
461 | + * You should have received a copy of the GNU Lesser General Public | |
462 | + * License along with this library; if not, write to the Free Software | |
463 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
464 | + * | |
465 | + */ | |
466 | + | |
467 | +#ifndef AKONADI_SERVER_COLLECTIONSTATISTICS_H | |
468 | +#define AKONADI_SERVER_COLLECTIONSTATISTICS_H | |
469 | + | |
470 | +class QMutex; | |
471 | + | |
472 | +#include <QHash> | |
473 | +#include <QMutex> | |
474 | + | |
475 | +namespace Akonadi { | |
476 | +namespace Server { | |
477 | + | |
478 | +class Collection; | |
479 | + | |
480 | +/** | |
481 | + * Provides cache for collection statistics | |
482 | + * | |
483 | + * Collection statistics are requested very often, so to take some load from the | |
484 | + * database we cache the results until the statistics are invalidated (see | |
485 | + * NotificationCollector, which takes care for invalidating the statistics). | |
486 | + * | |
487 | + * The cache (together with optimization of the actual SQL query) seems to | |
488 | + * massively improve initial folder listing on system start (when IO and CPU loads | |
489 | + * are very high). | |
490 | + */ | |
491 | +class CollectionStatistics | |
492 | +{ | |
493 | +public: | |
494 | + struct Statistics { | |
495 | + qint64 count; | |
496 | + qint64 size; | |
497 | + qint64 read; | |
498 | + }; | |
499 | + | |
500 | + static CollectionStatistics* instance(); | |
501 | + | |
502 | + const Statistics& statistics(const Collection &col); | |
503 | + void invalidateCollection(const Collection &col); | |
504 | + | |
505 | +private: | |
506 | + Statistics getCollectionStatistics(const Collection &col); | |
507 | + | |
508 | + QMutex mCacheLock; | |
509 | + QHash<qint64, Statistics> mCache; | |
510 | + | |
511 | + static CollectionStatistics *sInstance; | |
512 | +}; | |
513 | + | |
514 | +} // namespace Server | |
515 | +} // namespace Akonadi | |
516 | + | |
517 | +#endif // AKONADI_SERVER_COLLECTIONSTATISTICS_H | |
518 | diff --git a/server/src/storage/datastore.cpp b/server/src/storage/datastore.cpp | |
519 | index 304f0e8..0983d84 100644 | |
520 | --- a/server/src/storage/datastore.cpp | |
521 | +++ b/server/src/storage/datastore.cpp | |
522 | @@ -209,7 +209,7 @@ DataStore *DataStore::self() | |
523 | /* --- ItemFlags ----------------------------------------------------- */ | |
524 | ||
525 | bool DataStore::setItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, | |
526 | - bool *flagsChanged, bool silent ) | |
527 | + bool *flagsChanged, const Collection &col, bool silent ) | |
528 | { | |
529 | QSet<QByteArray> removedFlags; | |
530 | QSet<QByteArray> addedFlags; | |
531 | @@ -258,7 +258,7 @@ bool DataStore::setItemsFlags( const PimItem::List &items, const QVector<Flag> & | |
532 | } | |
533 | ||
534 | if ( !silent && ( !addedFlags.isEmpty() || !removedFlags.isEmpty() ) ) { | |
535 | - mNotificationCollector->itemsFlagsChanged( items, addedFlags, removedFlags ); | |
536 | + mNotificationCollector->itemsFlagsChanged( items, addedFlags, removedFlags, col ); | |
537 | } | |
538 | ||
539 | setBoolPtr( flagsChanged, ( addedFlags != removedFlags ) ); | |
540 | @@ -361,7 +361,7 @@ bool DataStore::appendItemsFlags( const PimItem::List &items, const QVector<Flag | |
541 | } | |
542 | ||
543 | bool DataStore::removeItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, | |
544 | - bool *flagsChanged, bool silent ) | |
545 | + bool *flagsChanged, const Collection &col, bool silent ) | |
546 | { | |
547 | QSet<QByteArray> removedFlags; | |
548 | QVariantList itemsIds; | |
549 | @@ -393,7 +393,7 @@ bool DataStore::removeItemsFlags( const PimItem::List &items, const QVector<Flag | |
550 | if ( qb.query().numRowsAffected() != 0 ) { | |
551 | setBoolPtr( flagsChanged, true ); | |
552 | if ( !silent ) { | |
553 | - mNotificationCollector->itemsFlagsChanged( items, QSet<QByteArray>(), removedFlags ); | |
554 | + mNotificationCollector->itemsFlagsChanged( items, QSet<QByteArray>(), removedFlags, col ); | |
555 | } | |
556 | } | |
557 | ||
558 | diff --git a/server/src/storage/datastore.h b/server/src/storage/datastore.h | |
559 | index 395b227..a2d8a42 100644 | |
560 | --- a/server/src/storage/datastore.h | |
561 | +++ b/server/src/storage/datastore.h | |
562 | @@ -119,10 +119,12 @@ class DataStore : public QObject | |
563 | static DataStore *self(); | |
564 | ||
565 | /* --- ItemFlags ----------------------------------------------------- */ | |
566 | - virtual bool setItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, bool *flagsChanged = 0, bool silent = false ); | |
567 | + virtual bool setItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, | |
568 | + bool *flagsChanged = 0, const Collection &col = Collection(), bool silent = false ); | |
569 | virtual bool appendItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, bool *flagsChanged = 0, | |
570 | bool checkIfExists = true, const Collection &col = Collection(), bool silent = false ); | |
571 | - virtual bool removeItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, bool *tagsChanged = 0, bool silent = false ); | |
572 | + virtual bool removeItemsFlags( const PimItem::List &items, const QVector<Flag> &flags, bool *tagsChanged = 0, | |
573 | + const Collection &collection = Collection(), bool silent = false ); | |
574 | ||
575 | /* --- ItemTags ----------------------------------------------------- */ | |
576 | virtual bool setItemsTags( const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = 0, bool silent = false ); | |
577 | diff --git a/server/src/storage/notificationcollector.cpp b/server/src/storage/notificationcollector.cpp | |
578 | index 67f57d1..dbc7883 100644 | |
579 | --- a/server/src/storage/notificationcollector.cpp | |
580 | +++ b/server/src/storage/notificationcollector.cpp | |
581 | @@ -20,6 +20,7 @@ | |
582 | #include "notificationcollector.h" | |
583 | #include "storage/datastore.h" | |
584 | #include "storage/entity.h" | |
585 | +#include "storage/collectionstatistics.h" | |
586 | #include "handlerhelper.h" | |
587 | #include "cachecleaner.h" | |
588 | #include "intervalcheck.h" | |
589 | @@ -133,6 +134,7 @@ void NotificationCollector::collectionChanged( const Collection &collection, | |
590 | if ( AkonadiServer::instance()->intervalChecker() ) { | |
591 | AkonadiServer::instance()->intervalChecker()->collectionAdded( collection.id() ); | |
592 | } | |
593 | + CollectionStatistics::instance()->invalidateCollection(collection); | |
594 | collectionNotification( NotificationMessageV2::Modify, collection, collection.parentId(), -1, resource, changes.toSet() ); | |
595 | } | |
596 | ||
597 | @@ -159,6 +161,8 @@ void NotificationCollector::collectionRemoved( const Collection &collection, | |
598 | if ( AkonadiServer::instance()->intervalChecker() ) { | |
599 | AkonadiServer::instance()->intervalChecker()->collectionRemoved( collection.id() ); | |
600 | } | |
601 | + CollectionStatistics::instance()->invalidateCollection(collection); | |
602 | + | |
603 | collectionNotification( NotificationMessageV2::Remove, collection, collection.parentId(), -1, resource ); | |
604 | } | |
605 | ||
606 | @@ -183,6 +187,8 @@ void NotificationCollector::collectionUnsubscribed( const Collection &collection | |
607 | if ( AkonadiServer::instance()->intervalChecker() ) { | |
608 | AkonadiServer::instance()->intervalChecker()->collectionRemoved( collection.id() ); | |
609 | } | |
610 | + CollectionStatistics::instance()->invalidateCollection(collection); | |
611 | + | |
612 | collectionNotification( NotificationMessageV2::Unsubscribe, collection, collection.parentId(), -1, resource, QSet<QByteArray>() ); | |
613 | } | |
614 | ||
615 | @@ -282,6 +288,7 @@ void NotificationCollector::itemNotification( NotificationMessageV2::Operation o | |
616 | copy.setParentCollection( iter.key() ); | |
617 | copy.setResource( resource ); | |
618 | ||
619 | + CollectionStatistics::instance()->invalidateCollection(Collection::retrieveById(iter.key())); | |
620 | dispatchNotification( copy ); | |
621 | } | |
622 | ||
623 | @@ -304,6 +311,7 @@ void NotificationCollector::itemNotification( NotificationMessageV2::Operation o | |
624 | } | |
625 | msg.setResource( res ); | |
626 | ||
627 | + CollectionStatistics::instance()->invalidateCollection(col); | |
628 | dispatchNotification( msg ); | |
629 | } | |
630 | ||
631 | diff --git a/server/tests/unittest/fakedatastore.cpp b/server/tests/unittest/fakedatastore.cpp | |
632 | index 12214fa..43ef7e6 100644 | |
633 | --- a/server/tests/unittest/fakedatastore.cpp | |
634 | +++ b/server/tests/unittest/fakedatastore.cpp | |
635 | @@ -91,13 +91,15 @@ bool FakeDataStore::init() | |
636 | bool FakeDataStore::setItemsFlags( const PimItem::List &items, | |
637 | const QVector<Flag> &flags, | |
638 | bool *flagsChanged, | |
639 | + const Collection &col, | |
640 | bool silent ) | |
641 | { | |
642 | mChanges.insert( QLatin1String( "setItemsFlags" ), | |
643 | QVariantList() << QVariant::fromValue( items ) | |
644 | << QVariant::fromValue( flags ) | |
645 | + << QVariant::fromValue( col ) | |
646 | << silent ); | |
647 | - return DataStore::setItemsFlags( items, flags, flagsChanged, silent ); | |
648 | + return DataStore::setItemsFlags( items, flags, flagsChanged, col, silent ); | |
649 | } | |
650 | ||
651 | bool FakeDataStore::appendItemsFlags( const PimItem::List &items, | |
652 | @@ -119,13 +121,15 @@ bool FakeDataStore::appendItemsFlags( const PimItem::List &items, | |
653 | bool FakeDataStore::removeItemsFlags( const PimItem::List &items, | |
654 | const QVector<Flag> &flags, | |
655 | bool *flagsChanged, | |
656 | + const Collection &col, | |
657 | bool silent ) | |
658 | { | |
659 | mChanges.insert( QLatin1String( "removeItemsFlags" ), | |
660 | QVariantList() << QVariant::fromValue( items ) | |
661 | << QVariant::fromValue( flags ) | |
662 | + << QVariant::fromValue( col ) | |
663 | << silent ); | |
664 | - return DataStore::removeItemsFlags( items, flags, flagsChanged, silent ); | |
665 | + return DataStore::removeItemsFlags( items, flags, flagsChanged, col, silent ); | |
666 | } | |
667 | ||
668 | ||
669 | diff --git a/server/tests/unittest/fakedatastore.h b/server/tests/unittest/fakedatastore.h | |
670 | index 62c5b75..cd9ab13 100644 | |
671 | --- a/server/tests/unittest/fakedatastore.h | |
672 | +++ b/server/tests/unittest/fakedatastore.h | |
673 | @@ -41,6 +41,7 @@ class FakeDataStore: public DataStore | |
674 | virtual bool setItemsFlags( const PimItem::List &items, | |
675 | const QVector<Flag> &flags, | |
676 | bool *flagsChanged = 0, | |
677 | + const Collection &col = Collection(), | |
678 | bool silent = false ); | |
679 | virtual bool appendItemsFlags( const PimItem::List &items, | |
680 | const QVector<Flag> &flags, | |
681 | @@ -51,6 +52,7 @@ class FakeDataStore: public DataStore | |
682 | virtual bool removeItemsFlags( const PimItem::List &items, | |
683 | const QVector<Flag> &flags, | |
684 | bool *flagsChanged = 0, | |
685 | + const Collection &col = Collection(), | |
686 | bool silent = false ); | |
687 | ||
688 | virtual bool setItemsTags( const PimItem::List &items, | |
689 | -- | |
690 | 2.1.0 | |
691 |