1/*
2 This file is part of the KDE Baloo project.
3 SPDX-FileCopyrightText: 2015 Vishesh Handa <vhanda@kde.org>
4 SPDX-FileCopyrightText: 2016 Christoph Cullmann <cullmann@kde.org>
5
6 SPDX-License-Identifier: LGPL-2.1-or-later
7*/
8
9#include "database.h"
10#include "transaction.h"
11#include "postingdb.h"
12#include "documentdb.h"
13#include "documenturldb.h"
14#include "documentiddb.h"
15#include "positiondb.h"
16#include "documenttimedb.h"
17#include "documentdatadb.h"
18#include "mtimedb.h"
19
20#include "enginequery.h"
21
22#include "andpostingiterator.h"
23#include "orpostingiterator.h"
24#include "phraseanditerator.h"
25
26#include "writetransaction.h"
27#include "idutils.h"
28#include "fsutils.h"
29
30#include "enginedebug.h"
31
32// MSVC does not understand the inline assembly in valgrind.h
33// Defining NVALGRIND stubs out all definitions, so we can use
34// the macros without ifdef'ing these in place
35#if defined _MSC_VER && !defined NVALGRIND
36#define NVALGRIND 1
37#endif
38#include "valgrind.h"
39
40#include <QFile>
41#include <QFileInfo>
42#include <QDir>
43#include <QMutexLocker>
44
45using namespace Baloo;
46
47Database::Database(const QString& path)
48 : m_path(path)
49 , m_env(nullptr)
50{
51}
52
53Database::~Database()
54{
55 // try only to close if we did open the DB successfully
56 if (m_env) {
57 mdb_env_close(env: m_env);
58 m_env = nullptr;
59 }
60}
61
62bool Database::open(OpenMode mode)
63{
64 QMutexLocker locker(&m_mutex);
65
66 // nop if already open!
67 if (m_env) {
68 return true;
69 }
70
71 QDir dir(m_path);
72 if (!dir.exists()) {
73 dir.mkpath(QStringLiteral("."));
74 dir.refresh();
75 }
76 QFileInfo indexInfo(dir, QStringLiteral("index"));
77
78 if ((mode != CreateDatabase) && !indexInfo.exists()) {
79 return false;
80 }
81
82 if (mode == CreateDatabase) {
83 if (!QFileInfo(dir.absolutePath()).permission(permissions: QFile::WriteOwner)) {
84 qCCritical(ENGINE) << m_path << "does not have write permissions. Aborting";
85 return false;
86 }
87
88 if (!indexInfo.exists()) {
89 FSUtils::disableCoW(path: m_path);
90 }
91 }
92
93 int rc = mdb_env_create(env: &m_env);
94 if (rc) {
95 m_env = nullptr;
96 return false;
97 }
98
99 /**
100 * maximal number of allowed named databases, must match number of databases we create below
101 * each additional one leads to overhead
102 */
103 mdb_env_set_maxdbs(env: m_env, dbs: 12);
104
105 /**
106 * size limit for database == size limit of mmap
107 * use 1 GB on 32-bit, use 256 GB on 64-bit
108 * Valgrind by default (without recompiling) limits the mmap size:
109 * <= 3.9: 32 GByte, 3.9 to 3.12: 64 GByte, 3.13: 128 GByte
110 */
111 size_t sizeInGByte = 256;
112 if (sizeof(void*) == 4) {
113 sizeInGByte = 1;
114 qCWarning(ENGINE) << "Running on 32 bit arch, limiting DB mmap to" << sizeInGByte << "GByte";
115 } else if (RUNNING_ON_VALGRIND) {
116 // valgrind lacks a runtime version check, assume valgrind >= 3.9, and allow for some other mmaps
117 sizeInGByte = 40;
118 qCWarning(ENGINE) << "Valgrind detected, limiting DB mmap to" << sizeInGByte << "GByte";
119 }
120 const size_t maximalSizeInBytes = sizeInGByte * size_t(1024) * size_t(1024) * size_t(1024);
121 mdb_env_set_mapsize(env: m_env, size: maximalSizeInBytes);
122
123 // Set MDB environment flags
124 auto mdbEnvFlags = MDB_NOSUBDIR | MDB_NOMEMINIT;
125 if (mode == ReadOnlyDatabase) {
126 mdbEnvFlags |= MDB_RDONLY;
127 }
128
129 // The directory needs to be created before opening the environment
130 QByteArray arr = QFile::encodeName(fileName: indexInfo.absoluteFilePath());
131 rc = mdb_env_open(env: m_env, path: arr.constData(), flags: mdbEnvFlags, mode: 0664);
132 if (rc) {
133 mdb_env_close(env: m_env);
134 m_env = nullptr;
135 return false;
136 }
137
138 rc = mdb_reader_check(env: m_env, dead: nullptr);
139
140 if (rc) {
141 qCWarning(ENGINE) << "Database::open reader_check" << mdb_strerror(err: rc);
142 mdb_env_close(env: m_env);
143 m_env = nullptr;
144 return false;
145 }
146
147 //
148 // Individual Databases
149 //
150 MDB_txn* txn;
151 if (mode != CreateDatabase) {
152 int rc = mdb_txn_begin(env: m_env, parent: nullptr, MDB_RDONLY, txn: &txn);
153 if (rc) {
154 qCWarning(ENGINE) << "Database::transaction ro begin" << mdb_strerror(err: rc);
155 mdb_env_close(env: m_env);
156 m_env = nullptr;
157 return false;
158 }
159
160 m_dbis.postingDbi = PostingDB::open(txn);
161 m_dbis.positionDBi = PositionDB::open(txn);
162
163 m_dbis.docTermsDbi = DocumentDB::open(name: "docterms", txn);
164 m_dbis.docFilenameTermsDbi = DocumentDB::open(name: "docfilenameterms", txn);
165 m_dbis.docXattrTermsDbi = DocumentDB::open(name: "docxatrrterms", txn);
166
167 m_dbis.idTreeDbi = IdTreeDB::open(txn);
168 m_dbis.idFilenameDbi = IdFilenameDB::open(txn);
169
170 m_dbis.docTimeDbi = DocumentTimeDB::open(txn);
171 m_dbis.docDataDbi = DocumentDataDB::open(txn);
172
173 m_dbis.contentIndexingDbi = DocumentIdDB::open(name: "indexingleveldb", txn);
174 m_dbis.failedIdDbi = DocumentIdDB::open(name: "failediddb", txn);
175
176 m_dbis.mtimeDbi = MTimeDB::open(txn);
177
178 if (!m_dbis.isValid()) {
179 qCWarning(ENGINE) << "dbis is invalid";
180 mdb_txn_abort(txn);
181 mdb_env_close(env: m_env);
182 m_env = nullptr;
183 return false;
184 }
185
186 rc = mdb_txn_commit(txn);
187 if (rc) {
188 qCWarning(ENGINE) << "Database::transaction ro commit" << mdb_strerror(err: rc);
189 mdb_env_close(env: m_env);
190 m_env = nullptr;
191 return false;
192 }
193 } else {
194 int rc = mdb_txn_begin(env: m_env, parent: nullptr, flags: 0, txn: &txn);
195 if (rc) {
196 qCWarning(ENGINE) << "Database::transaction begin" << mdb_strerror(err: rc);
197 mdb_env_close(env: m_env);
198 m_env = nullptr;
199 return false;
200 }
201
202 m_dbis.postingDbi = PostingDB::create(txn);
203 m_dbis.positionDBi = PositionDB::create(txn);
204
205 m_dbis.docTermsDbi = DocumentDB::create(name: "docterms", txn);
206 m_dbis.docFilenameTermsDbi = DocumentDB::create(name: "docfilenameterms", txn);
207 m_dbis.docXattrTermsDbi = DocumentDB::create(name: "docxatrrterms", txn);
208
209 m_dbis.idTreeDbi = IdTreeDB::create(txn);
210 m_dbis.idFilenameDbi = IdFilenameDB::create(txn);
211
212 m_dbis.docTimeDbi = DocumentTimeDB::create(txn);
213 m_dbis.docDataDbi = DocumentDataDB::create(txn);
214
215 m_dbis.contentIndexingDbi = DocumentIdDB::create(name: "indexingleveldb", txn);
216 m_dbis.failedIdDbi = DocumentIdDB::create(name: "failediddb", txn);
217
218 m_dbis.mtimeDbi = MTimeDB::create(txn);
219
220 if (!m_dbis.isValid()) {
221 qCWarning(ENGINE) << "dbis is invalid";
222 mdb_txn_abort(txn);
223 mdb_env_close(env: m_env);
224 m_env = nullptr;
225 return false;
226 }
227
228 rc = mdb_txn_commit(txn);
229 if (rc) {
230 qCWarning(ENGINE) << "Database::transaction commit" << mdb_strerror(err: rc);
231 mdb_env_close(env: m_env);
232 m_env = nullptr;
233 return false;
234 }
235 }
236
237 Q_ASSERT(m_env);
238 return true;
239}
240
241bool Database::isOpen() const
242{
243 QMutexLocker locker(&m_mutex);
244 return m_env != nullptr;
245}
246
247QString Database::path() const
248{
249 QMutexLocker locker(&m_mutex);
250 return m_path;
251}
252
253bool Database::isAvailable() const
254{
255 QMutexLocker locker(&m_mutex);
256 return QFileInfo::exists(file: m_path + QStringLiteral("/index"));
257}
258

source code of baloo/src/engine/database.cpp