1/*
2 This file is part of the KDE Baloo project.
3 SPDX-FileCopyrightText: 2015 Vishesh Handa <vhanda@kde.org>
4 SPDX-FileCopyrightText: 2016 Christoph Cullmann <cullmann@kde.org>
5
6 SPDX-License-Identifier: LGPL-2.1-or-later
7*/
8
9#include "database.h"
10#include "transaction.h"
11#include "postingdb.h"
12#include "documentdb.h"
13#include "documenturldb.h"
14#include "documentiddb.h"
15#include "positiondb.h"
16#include "documenttimedb.h"
17#include "documentdatadb.h"
18#include "mtimedb.h"
19
20#include "enginequery.h"
21
22#include "andpostingiterator.h"
23#include "orpostingiterator.h"
24#include "phraseanditerator.h"
25
26#include "writetransaction.h"
27#include "idutils.h"
28#include "fsutils.h"
29
30#include "enginedebug.h"
31
32// MSVC does not understand the inline assembly in valgrind.h
33// Defining NVALGRIND stubs out all definitions, so we can use
34// the macros without ifdef'ing these in place
35#if defined _MSC_VER && !defined NVALGRIND
36#define NVALGRIND 1
37#endif
38#include "valgrind.h"
39
40#include <QFile>
41#include <QFileInfo>
42#include <QDir>
43#include <QMutexLocker>
44
45using namespace Baloo;
46
47Database::Database(const QString& path)
48 : m_path(path)
49 , m_env(nullptr)
50{
51}
52
53Database::~Database()
54{
55 // try only to close if we did open the DB successfully
56 if (m_env) {
57 mdb_env_close(env: m_env);
58 m_env = nullptr;
59 }
60}
61
62Database::OpenResult Database::open(OpenMode mode)
63{
64 QMutexLocker locker(&m_mutex);
65
66 // nop if already open!
67 if (m_env) {
68 return OpenResult::Success;
69 }
70
71 MDB_env* env = nullptr;
72
73 QDir dir(m_path);
74 if (!dir.exists()) {
75 dir.mkpath(QStringLiteral("."));
76 dir.refresh();
77 }
78 QFileInfo indexInfo(dir, QStringLiteral("index"));
79
80 if ((mode != CreateDatabase) && !indexInfo.exists()) {
81 return OpenResult::InvalidPath;
82 }
83
84 if (mode == CreateDatabase) {
85 if (!QFileInfo(dir.absolutePath()).permission(permissions: QFile::WriteOwner)) {
86 qCCritical(ENGINE) << m_path << "does not have write permissions. Aborting";
87 return OpenResult::InvalidPath;
88 }
89
90 if (!indexInfo.exists()) {
91 FSUtils::disableCoW(path: m_path);
92 }
93 }
94
95 int rc = mdb_env_create(env: &env);
96 if (rc) {
97 return OpenResult::InternalError;
98 }
99
100 /**
101 * maximal number of allowed named databases, must match number of databases we create below
102 * each additional one leads to overhead
103 */
104 mdb_env_set_maxdbs(env, dbs: 12);
105
106 /**
107 * size limit for database == size limit of mmap
108 * use 1 GB on 32-bit, use 256 GB on 64-bit
109 * Valgrind by default (without recompiling) limits the mmap size:
110 * <= 3.9: 32 GByte, 3.9 to 3.12: 64 GByte, 3.13: 128 GByte
111 */
112 size_t sizeInGByte = 256;
113 if (sizeof(void*) == 4) {
114 sizeInGByte = 1;
115 qCWarning(ENGINE) << "Running on 32 bit arch, limiting DB mmap to" << sizeInGByte << "GByte";
116 } else if (RUNNING_ON_VALGRIND) {
117 // valgrind lacks a runtime version check, assume valgrind >= 3.9, and allow for some other mmaps
118 sizeInGByte = 40;
119 qCWarning(ENGINE) << "Valgrind detected, limiting DB mmap to" << sizeInGByte << "GByte";
120 }
121 const size_t maximalSizeInBytes = sizeInGByte * size_t(1024) * size_t(1024) * size_t(1024);
122 mdb_env_set_mapsize(env, size: maximalSizeInBytes);
123
124 // Set MDB environment flags
125 auto mdbEnvFlags = MDB_NOSUBDIR | MDB_NOMEMINIT;
126 if (mode == ReadOnlyDatabase) {
127 mdbEnvFlags |= MDB_RDONLY;
128 }
129
130 // The directory needs to be created before opening the environment
131 QByteArray arr = QFile::encodeName(fileName: indexInfo.absoluteFilePath());
132 rc = mdb_env_open(env, path: arr.constData(), flags: mdbEnvFlags, mode: 0664);
133 if (rc) {
134 // mdb_env_close must be called when mdb_env_open fails
135 mdb_env_close(env);
136 if ((rc == ENOENT) || (rc == EACCES)) {
137 return OpenResult::InvalidPath;
138 }
139 return OpenResult::InternalError;
140 }
141
142 rc = mdb_reader_check(env, dead: nullptr);
143
144 if (rc) {
145 qCWarning(ENGINE) << "Database::open reader_check" << mdb_strerror(err: rc);
146 mdb_env_close(env);
147 return OpenResult::InternalError;
148 }
149
150 //
151 // Individual Databases
152 //
153 MDB_txn* txn;
154 if (mode != CreateDatabase) {
155 int rc = mdb_txn_begin(env, parent: nullptr, MDB_RDONLY, txn: &txn);
156 if (rc) {
157 qCWarning(ENGINE) << "Database::transaction ro begin" << mdb_strerror(err: rc);
158 mdb_env_close(env);
159 return OpenResult::InternalError;
160 }
161
162 m_dbis.postingDbi = PostingDB::open(txn);
163 m_dbis.positionDBi = PositionDB::open(txn);
164
165 m_dbis.docTermsDbi = DocumentDB::open(name: "docterms", txn);
166 m_dbis.docFilenameTermsDbi = DocumentDB::open(name: "docfilenameterms", txn);
167 m_dbis.docXattrTermsDbi = DocumentDB::open(name: "docxatrrterms", txn);
168
169 m_dbis.idTreeDbi = IdTreeDB::open(txn);
170 m_dbis.idFilenameDbi = IdFilenameDB::open(txn);
171
172 m_dbis.docTimeDbi = DocumentTimeDB::open(txn);
173 m_dbis.docDataDbi = DocumentDataDB::open(txn);
174
175 m_dbis.contentIndexingDbi = DocumentIdDB::open(name: "indexingleveldb", txn);
176 m_dbis.failedIdDbi = DocumentIdDB::open(name: "failediddb", txn);
177
178 m_dbis.mtimeDbi = MTimeDB::open(txn);
179
180 if (!m_dbis.isValid()) {
181 qCWarning(ENGINE) << "dbis is invalid";
182 mdb_txn_abort(txn);
183 mdb_env_close(env);
184 return OpenResult::InvalidDatabase;
185 }
186
187 rc = mdb_txn_commit(txn);
188 if (rc) {
189 qCWarning(ENGINE) << "Database::transaction ro commit" << mdb_strerror(err: rc);
190 mdb_env_close(env);
191 return OpenResult::InternalError;
192 }
193 } else {
194 int rc = mdb_txn_begin(env, parent: nullptr, flags: 0, txn: &txn);
195 if (rc) {
196 qCWarning(ENGINE) << "Database::transaction begin" << mdb_strerror(err: rc);
197 mdb_env_close(env);
198 return OpenResult::InternalError;
199 }
200
201 m_dbis.postingDbi = PostingDB::create(txn);
202 m_dbis.positionDBi = PositionDB::create(txn);
203
204 m_dbis.docTermsDbi = DocumentDB::create(name: "docterms", txn);
205 m_dbis.docFilenameTermsDbi = DocumentDB::create(name: "docfilenameterms", txn);
206 m_dbis.docXattrTermsDbi = DocumentDB::create(name: "docxatrrterms", txn);
207
208 m_dbis.idTreeDbi = IdTreeDB::create(txn);
209 m_dbis.idFilenameDbi = IdFilenameDB::create(txn);
210
211 m_dbis.docTimeDbi = DocumentTimeDB::create(txn);
212 m_dbis.docDataDbi = DocumentDataDB::create(txn);
213
214 m_dbis.contentIndexingDbi = DocumentIdDB::create(name: "indexingleveldb", txn);
215 m_dbis.failedIdDbi = DocumentIdDB::create(name: "failediddb", txn);
216
217 m_dbis.mtimeDbi = MTimeDB::create(txn);
218
219 if (!m_dbis.isValid()) {
220 qCWarning(ENGINE) << "dbis is invalid";
221 mdb_txn_abort(txn);
222 mdb_env_close(env);
223 return OpenResult::InvalidDatabase;
224 }
225
226 rc = mdb_txn_commit(txn);
227 if (rc) {
228 qCWarning(ENGINE) << "Database::transaction commit" << mdb_strerror(err: rc);
229 mdb_env_close(env);
230 return OpenResult::InternalError;
231 }
232 }
233
234 Q_ASSERT(env);
235 m_env = env;
236 return OpenResult::Success;
237}
238

source code of baloo/src/engine/database.cpp