1#include <mbgl/storage/offline_database.hpp>
2#include <mbgl/storage/response.hpp>
3#include <mbgl/util/compression.hpp>
4#include <mbgl/util/io.hpp>
5#include <mbgl/util/string.hpp>
6#include <mbgl/util/chrono.hpp>
7#include <mbgl/util/logging.hpp>
8
9#include "offline_schema.hpp"
10
11#include "sqlite3.hpp"
12
13namespace mbgl {
14
15OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
16 : path(std::move(path_)),
17 maximumCacheSize(maximumCacheSize_) {
18 ensureSchema();
19}
20
21OfflineDatabase::~OfflineDatabase() {
22 // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
23 // can't throw anything.
24 try {
25 statements.clear();
26 db.reset();
27 } catch (mapbox::sqlite::Exception& ex) {
28 Log::Error(event: Event::Database, args: (int)ex.code, args: ex.what());
29 }
30}
31
32void OfflineDatabase::ensureSchema() {
33 auto result = mapbox::sqlite::Database::tryOpen(filename: path, flags: mapbox::sqlite::ReadWriteCreate);
34 if (result.is<mapbox::sqlite::Exception>()) {
35 const auto& ex = result.get<mapbox::sqlite::Exception>();
36 if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
37 // Corrupted; blow it away.
38 removeExisting();
39 result = mapbox::sqlite::Database::open(filename: path, flags: mapbox::sqlite::ReadWriteCreate);
40 } else {
41 Log::Error(event: Event::Database, args: "Unexpected error connecting to database: %s", args: ex.what());
42 throw ex;
43 }
44 }
45
46 try {
47 assert(result.is<mapbox::sqlite::Database>());
48 db = std::make_unique<mapbox::sqlite::Database>(args: std::move(result.get<mapbox::sqlite::Database>()));
49 db->setBusyTimeout(Milliseconds::max());
50 db->exec(sql: "PRAGMA foreign_keys = ON");
51
52 switch (userVersion()) {
53 case 0:
54 case 1:
55 // Newly created database, or old cache-only database; remove old table if it exists.
56 removeOldCacheTable();
57 break;
58 case 2:
59 migrateToVersion3();
60 // fall through
61 case 3:
62 case 4:
63 migrateToVersion5();
64 // fall through
65 case 5:
66 migrateToVersion6();
67 // fall through
68 case 6:
69 // happy path; we're done
70 return;
71 default:
72 // downgrade, delete the database
73 removeExisting();
74 break;
75 }
76 } catch (const mapbox::sqlite::Exception& ex) {
77 // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database.
78 // Apparently sometimes it is delayed until the first read operation.
79 if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
80 removeExisting();
81 } else {
82 throw;
83 }
84 }
85
86 try {
87 // When downgrading the database, or when the database is corrupt, we've deleted the old database handle,
88 // so we need to reopen it.
89 if (!db) {
90 db = std::make_unique<mapbox::sqlite::Database>(args: mapbox::sqlite::Database::open(filename: path, flags: mapbox::sqlite::ReadWriteCreate));
91 db->setBusyTimeout(Milliseconds::max());
92 db->exec(sql: "PRAGMA foreign_keys = ON");
93 }
94
95 db->exec(sql: "PRAGMA auto_vacuum = INCREMENTAL");
96 db->exec(sql: "PRAGMA journal_mode = DELETE");
97 db->exec(sql: "PRAGMA synchronous = FULL");
98 db->exec(sql: offlineDatabaseSchema);
99 db->exec(sql: "PRAGMA user_version = 6");
100 } catch (...) {
101 Log::Error(event: Event::Database, args: "Unexpected error creating database schema: %s", args: util::toString(error: std::current_exception()).c_str());
102 throw;
103 }
104}
105
106int OfflineDatabase::userVersion() {
107 return static_cast<int>(getPragma<int64_t>(sql: "PRAGMA user_version"));
108}
109
110void OfflineDatabase::removeExisting() {
111 Log::Warning(event: Event::Database, args: "Removing existing incompatible offline database");
112
113 statements.clear();
114 db.reset();
115
116 try {
117 util::deleteFile(filename: path);
118 } catch (util::IOException& ex) {
119 Log::Error(event: Event::Database, args: ex.code, args: ex.what());
120 }
121}
122
123void OfflineDatabase::removeOldCacheTable() {
124 db->exec(sql: "DROP TABLE IF EXISTS http_cache");
125 db->exec(sql: "VACUUM");
126}
127
128void OfflineDatabase::migrateToVersion3() {
129 db->exec(sql: "PRAGMA auto_vacuum = INCREMENTAL");
130 db->exec(sql: "VACUUM");
131 db->exec(sql: "PRAGMA user_version = 3");
132}
133
134// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre-
135// release development and the migration was removed entirely to avoid potential
136// conflicts from quickly (and needlessly) switching journal and sync modes.
137//
138// See: https://github.com/mapbox/mapbox-gl-native/pull/6320
139
140void OfflineDatabase::migrateToVersion5() {
141 db->exec(sql: "PRAGMA journal_mode = DELETE");
142 db->exec(sql: "PRAGMA synchronous = FULL");
143 db->exec(sql: "PRAGMA user_version = 5");
144}
145
146void OfflineDatabase::migrateToVersion6() {
147 mapbox::sqlite::Transaction transaction(*db);
148 db->exec(sql: "ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
149 db->exec(sql: "ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
150 db->exec(sql: "PRAGMA user_version = 6");
151 transaction.commit();
152}
153
154mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
155 auto it = statements.find(x: sql);
156 if (it == statements.end()) {
157 it = statements.emplace(args&: sql, args: std::make_unique<mapbox::sqlite::Statement>(args&: *db, args&: sql)).first;
158 }
159 return *it->second;
160}
161
162optional<Response> OfflineDatabase::get(const Resource& resource) {
163 auto result = getInternal(resource);
164 return result ? result->first : optional<Response>();
165}
166
167optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
168 if (resource.kind == Resource::Kind::Tile) {
169 assert(resource.tileData);
170 return getTile(*resource.tileData);
171 } else {
172 return getResource(resource);
173 }
174}
175
176optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) {
177 if (resource.kind == Resource::Kind::Tile) {
178 assert(resource.tileData);
179 return hasTile(*resource.tileData);
180 } else {
181 return hasResource(resource);
182 }
183}
184
185std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) {
186 mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
187 auto result = putInternal(resource, response, evict: true);
188 transaction.commit();
189 return result;
190}
191
192std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) {
193 if (response.error) {
194 return { false, 0 };
195 }
196
197 std::string compressedData;
198 bool compressed = false;
199 uint64_t size = 0;
200
201 if (response.data) {
202 compressedData = util::compress(raw: *response.data);
203 compressed = compressedData.size() < response.data->size();
204 size = compressed ? compressedData.size() : response.data->size();
205 }
206
207 if (evict_ && !evict(neededFreeSize: size)) {
208 Log::Info(event: Event::Database, args: "Unable to make space for entry");
209 return { false, 0 };
210 }
211
212 bool inserted;
213
214 if (resource.kind == Resource::Kind::Tile) {
215 assert(resource.tileData);
216 inserted = putTile(*resource.tileData, response,
217 compressed ? compressedData : response.data ? *response.data : "",
218 compressed);
219 } else {
220 inserted = putResource(resource, response,
221 compressed ? compressedData : response.data ? *response.data : "",
222 compressed);
223 }
224
225 return { inserted, size };
226}
227
228optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
229 // Update accessed timestamp used for LRU eviction.
230 {
231 mapbox::sqlite::Query accessedQuery{ getStatement(sql: "UPDATE resources SET accessed = ?1 WHERE url = ?2") };
232 accessedQuery.bind(offset: 1, value: util::now());
233 accessedQuery.bind(offset: 2, resource.url);
234 accessedQuery.run();
235 }
236
237 // clang-format off
238 mapbox::sqlite::Query query{ getStatement(
239 // 0 1 2 3 4 5
240 sql: "SELECT etag, expires, must_revalidate, modified, data, compressed "
241 "FROM resources "
242 "WHERE url = ?") };
243 // clang-format on
244
245 query.bind(offset: 1, resource.url);
246
247 if (!query.run()) {
248 return {};
249 }
250
251 Response response;
252 uint64_t size = 0;
253
254 response.etag = query.get<optional<std::string>>(offset: 0);
255 response.expires = query.get<optional<Timestamp>>(offset: 1);
256 response.mustRevalidate = query.get<bool>(offset: 2);
257 response.modified = query.get<optional<Timestamp>>(offset: 3);
258
259 auto data = query.get<optional<std::string>>(offset: 4);
260 if (!data) {
261 response.noContent = true;
262 } else if (query.get<bool>(offset: 5)) {
263 response.data = std::make_shared<std::string>(args: util::decompress(raw: *data));
264 size = data->length();
265 } else {
266 response.data = std::make_shared<std::string>(args&: *data);
267 size = data->length();
268 }
269
270 return std::make_pair(x&: response, y&: size);
271}
272
273optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
274 mapbox::sqlite::Query query{ getStatement(sql: "SELECT length(data) FROM resources WHERE url = ?") };
275 query.bind(offset: 1, resource.url);
276 if (!query.run()) {
277 return {};
278 }
279
280 return query.get<optional<int64_t>>(offset: 0);
281}
282
283bool OfflineDatabase::putResource(const Resource& resource,
284 const Response& response,
285 const std::string& data,
286 bool compressed) {
287 if (response.notModified) {
288 // clang-format off
289 mapbox::sqlite::Query notModifiedQuery{ getStatement(
290 sql: "UPDATE resources "
291 "SET accessed = ?1, "
292 " expires = ?2, "
293 " must_revalidate = ?3 "
294 "WHERE url = ?4 ") };
295 // clang-format on
296
297 notModifiedQuery.bind(offset: 1, value: util::now());
298 notModifiedQuery.bind(offset: 2, value: response.expires);
299 notModifiedQuery.bind(offset: 3, value: response.mustRevalidate);
300 notModifiedQuery.bind(offset: 4, resource.url);
301 notModifiedQuery.run();
302 return false;
303 }
304
305 // We can't use REPLACE because it would change the id value.
306 // clang-format off
307 mapbox::sqlite::Query updateQuery{ getStatement(
308 sql: "UPDATE resources "
309 "SET kind = ?1, "
310 " etag = ?2, "
311 " expires = ?3, "
312 " must_revalidate = ?4, "
313 " modified = ?5, "
314 " accessed = ?6, "
315 " data = ?7, "
316 " compressed = ?8 "
317 "WHERE url = ?9 ") };
318 // clang-format on
319
320 updateQuery.bind(offset: 1, value: int(resource.kind));
321 updateQuery.bind(offset: 2, value: response.etag);
322 updateQuery.bind(offset: 3, value: response.expires);
323 updateQuery.bind(offset: 4, value: response.mustRevalidate);
324 updateQuery.bind(offset: 5, value: response.modified);
325 updateQuery.bind(offset: 6, value: util::now());
326 updateQuery.bind(offset: 9, resource.url);
327
328 if (response.noContent) {
329 updateQuery.bind(offset: 7, value: nullptr);
330 updateQuery.bind(offset: 8, value: false);
331 } else {
332 updateQuery.bindBlob(offset: 7, data.data(), length: data.size(), retain: false);
333 updateQuery.bind(offset: 8, value: compressed);
334 }
335
336 updateQuery.run();
337 if (updateQuery.changes() != 0) {
338 return false;
339 }
340
341 // clang-format off
342 mapbox::sqlite::Query insertQuery{ getStatement(
343 sql: "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
344 "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
345 // clang-format on
346
347 insertQuery.bind(offset: 1, resource.url);
348 insertQuery.bind(offset: 2, value: int(resource.kind));
349 insertQuery.bind(offset: 3, value: response.etag);
350 insertQuery.bind(offset: 4, value: response.expires);
351 insertQuery.bind(offset: 5, value: response.mustRevalidate);
352 insertQuery.bind(offset: 6, value: response.modified);
353 insertQuery.bind(offset: 7, value: util::now());
354
355 if (response.noContent) {
356 insertQuery.bind(offset: 8, value: nullptr);
357 insertQuery.bind(offset: 9, value: false);
358 } else {
359 insertQuery.bindBlob(offset: 8, data.data(), length: data.size(), retain: false);
360 insertQuery.bind(offset: 9, value: compressed);
361 }
362
363 insertQuery.run();
364
365 return true;
366}
367
368optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
369 {
370 // clang-format off
371 mapbox::sqlite::Query accessedQuery{ getStatement(
372 sql: "UPDATE tiles "
373 "SET accessed = ?1 "
374 "WHERE url_template = ?2 "
375 " AND pixel_ratio = ?3 "
376 " AND x = ?4 "
377 " AND y = ?5 "
378 " AND z = ?6 ") };
379 // clang-format on
380
381 accessedQuery.bind(offset: 1, value: util::now());
382 accessedQuery.bind(offset: 2, tile.urlTemplate);
383 accessedQuery.bind(offset: 3, value: tile.pixelRatio);
384 accessedQuery.bind(offset: 4, value: tile.x);
385 accessedQuery.bind(offset: 5, value: tile.y);
386 accessedQuery.bind(offset: 6, value: tile.z);
387 accessedQuery.run();
388 }
389
390 // clang-format off
391 mapbox::sqlite::Query query{ getStatement(
392 // 0 1 2, 3, 4, 5
393 sql: "SELECT etag, expires, must_revalidate, modified, data, compressed "
394 "FROM tiles "
395 "WHERE url_template = ?1 "
396 " AND pixel_ratio = ?2 "
397 " AND x = ?3 "
398 " AND y = ?4 "
399 " AND z = ?5 ") };
400 // clang-format on
401
402 query.bind(offset: 1, tile.urlTemplate);
403 query.bind(offset: 2, value: tile.pixelRatio);
404 query.bind(offset: 3, value: tile.x);
405 query.bind(offset: 4, value: tile.y);
406 query.bind(offset: 5, value: tile.z);
407
408 if (!query.run()) {
409 return {};
410 }
411
412 Response response;
413 uint64_t size = 0;
414
415 response.etag = query.get<optional<std::string>>(offset: 0);
416 response.expires = query.get<optional<Timestamp>>(offset: 1);
417 response.mustRevalidate = query.get<bool>(offset: 2);
418 response.modified = query.get<optional<Timestamp>>(offset: 3);
419
420 optional<std::string> data = query.get<optional<std::string>>(offset: 4);
421 if (!data) {
422 response.noContent = true;
423 } else if (query.get<bool>(offset: 5)) {
424 response.data = std::make_shared<std::string>(args: util::decompress(raw: *data));
425 size = data->length();
426 } else {
427 response.data = std::make_shared<std::string>(args&: *data);
428 size = data->length();
429 }
430
431 return std::make_pair(x&: response, y&: size);
432}
433
434optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
435 // clang-format off
436 mapbox::sqlite::Query size{ getStatement(
437 sql: "SELECT length(data) "
438 "FROM tiles "
439 "WHERE url_template = ?1 "
440 " AND pixel_ratio = ?2 "
441 " AND x = ?3 "
442 " AND y = ?4 "
443 " AND z = ?5 ") };
444 // clang-format on
445
446 size.bind(offset: 1, tile.urlTemplate);
447 size.bind(offset: 2, value: tile.pixelRatio);
448 size.bind(offset: 3, value: tile.x);
449 size.bind(offset: 4, value: tile.y);
450 size.bind(offset: 5, value: tile.z);
451
452 if (!size.run()) {
453 return {};
454 }
455
456 return size.get<optional<int64_t>>(offset: 0);
457}
458
459bool OfflineDatabase::putTile(const Resource::TileData& tile,
460 const Response& response,
461 const std::string& data,
462 bool compressed) {
463 if (response.notModified) {
464 // clang-format off
465 mapbox::sqlite::Query notModifiedQuery{ getStatement(
466 sql: "UPDATE tiles "
467 "SET accessed = ?1, "
468 " expires = ?2, "
469 " must_revalidate = ?3 "
470 "WHERE url_template = ?4 "
471 " AND pixel_ratio = ?5 "
472 " AND x = ?6 "
473 " AND y = ?7 "
474 " AND z = ?8 ") };
475 // clang-format on
476
477 notModifiedQuery.bind(offset: 1, value: util::now());
478 notModifiedQuery.bind(offset: 2, value: response.expires);
479 notModifiedQuery.bind(offset: 3, value: response.mustRevalidate);
480 notModifiedQuery.bind(offset: 4, tile.urlTemplate);
481 notModifiedQuery.bind(offset: 5, value: tile.pixelRatio);
482 notModifiedQuery.bind(offset: 6, value: tile.x);
483 notModifiedQuery.bind(offset: 7, value: tile.y);
484 notModifiedQuery.bind(offset: 8, value: tile.z);
485 notModifiedQuery.run();
486 return false;
487 }
488
489 // We can't use REPLACE because it would change the id value.
490
491 // clang-format off
492 mapbox::sqlite::Query updateQuery{ getStatement(
493 sql: "UPDATE tiles "
494 "SET modified = ?1, "
495 " etag = ?2, "
496 " expires = ?3, "
497 " must_revalidate = ?4, "
498 " accessed = ?5, "
499 " data = ?6, "
500 " compressed = ?7 "
501 "WHERE url_template = ?8 "
502 " AND pixel_ratio = ?9 "
503 " AND x = ?10 "
504 " AND y = ?11 "
505 " AND z = ?12 ") };
506 // clang-format on
507
508 updateQuery.bind(offset: 1, value: response.modified);
509 updateQuery.bind(offset: 2, value: response.etag);
510 updateQuery.bind(offset: 3, value: response.expires);
511 updateQuery.bind(offset: 4, value: response.mustRevalidate);
512 updateQuery.bind(offset: 5, value: util::now());
513 updateQuery.bind(offset: 8, tile.urlTemplate);
514 updateQuery.bind(offset: 9, value: tile.pixelRatio);
515 updateQuery.bind(offset: 10, value: tile.x);
516 updateQuery.bind(offset: 11, value: tile.y);
517 updateQuery.bind(offset: 12, value: tile.z);
518
519 if (response.noContent) {
520 updateQuery.bind(offset: 6, value: nullptr);
521 updateQuery.bind(offset: 7, value: false);
522 } else {
523 updateQuery.bindBlob(offset: 6, data.data(), length: data.size(), retain: false);
524 updateQuery.bind(offset: 7, value: compressed);
525 }
526
527 updateQuery.run();
528 if (updateQuery.changes() != 0) {
529 return false;
530 }
531
532 // clang-format off
533 mapbox::sqlite::Query insertQuery{ getStatement(
534 sql: "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
535 "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
536 // clang-format on
537
538 insertQuery.bind(offset: 1, tile.urlTemplate);
539 insertQuery.bind(offset: 2, value: tile.pixelRatio);
540 insertQuery.bind(offset: 3, value: tile.x);
541 insertQuery.bind(offset: 4, value: tile.y);
542 insertQuery.bind(offset: 5, value: tile.z);
543 insertQuery.bind(offset: 6, value: response.modified);
544 insertQuery.bind(offset: 7, value: response.mustRevalidate);
545 insertQuery.bind(offset: 8, value: response.etag);
546 insertQuery.bind(offset: 9, value: response.expires);
547 insertQuery.bind(offset: 10, value: util::now());
548
549 if (response.noContent) {
550 insertQuery.bind(offset: 11, value: nullptr);
551 insertQuery.bind(offset: 12, value: false);
552 } else {
553 insertQuery.bindBlob(offset: 11, data.data(), length: data.size(), retain: false);
554 insertQuery.bind(offset: 12, value: compressed);
555 }
556
557 insertQuery.run();
558
559 return true;
560}
561
562std::vector<OfflineRegion> OfflineDatabase::listRegions() {
563 mapbox::sqlite::Query query{ getStatement(sql: "SELECT id, definition, description FROM regions") };
564
565 std::vector<OfflineRegion> result;
566
567 while (query.run()) {
568 result.push_back(x: OfflineRegion(
569 query.get<int64_t>(offset: 0),
570 decodeOfflineRegionDefinition(query.get<std::string>(offset: 1)),
571 query.get<std::vector<uint8_t>>(offset: 2)));
572 }
573
574 return result;
575}
576
577OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
578 const OfflineRegionMetadata& metadata) {
579 // clang-format off
580 mapbox::sqlite::Query query{ getStatement(
581 sql: "INSERT INTO regions (definition, description) "
582 "VALUES (?1, ?2) ") };
583 // clang-format on
584
585 query.bind(offset: 1, encodeOfflineRegionDefinition(definition));
586 query.bindBlob(offset: 2, metadata);
587 query.run();
588
589 return OfflineRegion(query.lastInsertRowId(), definition, metadata);
590}
591
592OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) {
593 // clang-format off
594 mapbox::sqlite::Query query{ getStatement(
595 sql: "UPDATE regions SET description = ?1 "
596 "WHERE id = ?2") };
597 // clang-format on
598 query.bindBlob(offset: 1, metadata);
599 query.bind(offset: 2, value: regionID);
600 query.run();
601
602 return metadata;
603}
604
605void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
606 {
607 mapbox::sqlite::Query query{ getStatement(sql: "DELETE FROM regions WHERE id = ?") };
608 query.bind(offset: 1, value: region.getID());
609 query.run();
610 }
611
612 evict(neededFreeSize: 0);
613 db->exec(sql: "PRAGMA incremental_vacuum");
614
615 // Ensure that the cached offlineTileCount value is recalculated.
616 offlineMapboxTileCount = {};
617}
618
619optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
620 auto response = getInternal(resource);
621
622 if (response) {
623 markUsed(regionID, resource);
624 }
625
626 return response;
627}
628
629optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) {
630 auto response = hasInternal(resource);
631
632 if (response) {
633 markUsed(regionID, resource);
634 }
635
636 return response;
637}
638
639uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
640 mapbox::sqlite::Transaction transaction(*db);
641 auto size = putRegionResourceInternal(regionID, resource, response);
642 transaction.commit();
643 return size;
644}
645
646void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>& resources, OfflineRegionStatus& status) {
647 mapbox::sqlite::Transaction transaction(*db);
648
649 for (const auto& elem : resources) {
650 const auto& resource = std::get<0>(t: elem);
651 const auto& response = std::get<1>(t: elem);
652
653 try {
654 uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response);
655 status.completedResourceCount++;
656 status.completedResourceSize += resourceSize;
657 if (resource.kind == Resource::Kind::Tile) {
658 status.completedTileCount += 1;
659 status.completedTileSize += resourceSize;
660 }
661 } catch (const MapboxTileLimitExceededException&) {
662 // Commit the rest of the batch and retrow
663 transaction.commit();
664 throw;
665 }
666 }
667
668 // Commit the completed batch
669 transaction.commit();
670}
671
672uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) {
673 if (exceedsOfflineMapboxTileCountLimit(resource)) {
674 throw MapboxTileLimitExceededException();
675 }
676
677 uint64_t size = putInternal(resource, response, evict_: false).second;
678 bool previouslyUnused = markUsed(regionID, resource);
679
680 if (offlineMapboxTileCount
681 && resource.kind == Resource::Kind::Tile
682 && util::mapbox::isMapboxURL(url: resource.url)
683 && previouslyUnused) {
684 *offlineMapboxTileCount += 1;
685 }
686
687 return size;
688}
689
690bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
691 if (resource.kind == Resource::Kind::Tile) {
692 // clang-format off
693 mapbox::sqlite::Query insertQuery{ getStatement(
694 sql: "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
695 "SELECT ?1, tiles.id "
696 "FROM tiles "
697 "WHERE url_template = ?2 "
698 " AND pixel_ratio = ?3 "
699 " AND x = ?4 "
700 " AND y = ?5 "
701 " AND z = ?6 ") };
702 // clang-format on
703
704 const Resource::TileData& tile = *resource.tileData;
705 insertQuery.bind(offset: 1, value: regionID);
706 insertQuery.bind(offset: 2, tile.urlTemplate);
707 insertQuery.bind(offset: 3, value: tile.pixelRatio);
708 insertQuery.bind(offset: 4, value: tile.x);
709 insertQuery.bind(offset: 5, value: tile.y);
710 insertQuery.bind(offset: 6, value: tile.z);
711 insertQuery.run();
712
713 if (insertQuery.changes() == 0) {
714 return false;
715 }
716
717 // clang-format off
718 mapbox::sqlite::Query selectQuery{ getStatement(
719 sql: "SELECT region_id "
720 "FROM region_tiles, tiles "
721 "WHERE region_id != ?1 "
722 " AND url_template = ?2 "
723 " AND pixel_ratio = ?3 "
724 " AND x = ?4 "
725 " AND y = ?5 "
726 " AND z = ?6 "
727 "LIMIT 1 ") };
728 // clang-format on
729
730 selectQuery.bind(offset: 1, value: regionID);
731 selectQuery.bind(offset: 2, tile.urlTemplate);
732 selectQuery.bind(offset: 3, value: tile.pixelRatio);
733 selectQuery.bind(offset: 4, value: tile.x);
734 selectQuery.bind(offset: 5, value: tile.y);
735 selectQuery.bind(offset: 6, value: tile.z);
736 return !selectQuery.run();
737 } else {
738 // clang-format off
739 mapbox::sqlite::Query insertQuery{ getStatement(
740 sql: "INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
741 "SELECT ?1, resources.id "
742 "FROM resources "
743 "WHERE resources.url = ?2 ") };
744 // clang-format on
745
746 insertQuery.bind(offset: 1, value: regionID);
747 insertQuery.bind(offset: 2, resource.url);
748 insertQuery.run();
749
750 if (insertQuery.changes() == 0) {
751 return false;
752 }
753
754 // clang-format off
755 mapbox::sqlite::Query selectQuery{ getStatement(
756 sql: "SELECT region_id "
757 "FROM region_resources, resources "
758 "WHERE region_id != ?1 "
759 " AND resources.url = ?2 "
760 "LIMIT 1 ") };
761 // clang-format on
762
763 selectQuery.bind(offset: 1, value: regionID);
764 selectQuery.bind(offset: 2, resource.url);
765 return !selectQuery.run();
766 }
767}
768
769OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) {
770 mapbox::sqlite::Query query{ getStatement(sql: "SELECT definition FROM regions WHERE id = ?1") };
771 query.bind(offset: 1, value: regionID);
772 query.run();
773
774 return decodeOfflineRegionDefinition(query.get<std::string>(offset: 0));
775}
776
777OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) {
778 OfflineRegionStatus result;
779
780 std::tie(args&: result.completedResourceCount, args&: result.completedResourceSize)
781 = getCompletedResourceCountAndSize(regionID);
782 std::tie(args&: result.completedTileCount, args&: result.completedTileSize)
783 = getCompletedTileCountAndSize(regionID);
784
785 result.completedResourceCount += result.completedTileCount;
786 result.completedResourceSize += result.completedTileSize;
787
788 return result;
789}
790
791std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
792 // clang-format off
793 mapbox::sqlite::Query query{ getStatement(
794 sql: "SELECT COUNT(*), SUM(LENGTH(data)) "
795 "FROM region_resources, resources "
796 "WHERE region_id = ?1 "
797 "AND resource_id = resources.id ") };
798 // clang-format on
799 query.bind(offset: 1, value: regionID);
800 query.run();
801 return { query.get<int64_t>(offset: 0), query.get<int64_t>(offset: 1) };
802}
803
804std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
805 // clang-format off
806 mapbox::sqlite::Query query{ getStatement(
807 sql: "SELECT COUNT(*), SUM(LENGTH(data)) "
808 "FROM region_tiles, tiles "
809 "WHERE region_id = ?1 "
810 "AND tile_id = tiles.id ") };
811 // clang-format on
812 query.bind(offset: 1, value: regionID);
813 query.run();
814 return { query.get<int64_t>(offset: 0), query.get<int64_t>(offset: 1) };
815}
816
817template <class T>
818T OfflineDatabase::getPragma(const char* sql) {
819 mapbox::sqlite::Query query{ getStatement(sql) };
820 query.run();
821 return query.get<T>(0);
822}
823
824// Remove least-recently used resources and tiles until the used database size,
825// as calculated by multiplying the number of in-use pages by the page size, is
826// less than the maximum cache size. Returns false if this condition cannot be
827// satisfied.
828//
829// SQLite database never shrinks in size unless we call VACCUM. We here
830// are monitoring the soft limit (i.e. number of free pages in the file)
831// and as it approaches to the hard limit (i.e. the actual file size) we
832// delete an arbitrary number of old cache entries. The free pages approach saves
833// us from calling VACCUM or keeping a running total, which can be costly.
834bool OfflineDatabase::evict(uint64_t neededFreeSize) {
835 uint64_t pageSize = getPragma<int64_t>(sql: "PRAGMA page_size");
836 uint64_t pageCount = getPragma<int64_t>(sql: "PRAGMA page_count");
837
838 auto usedSize = [&] {
839 return pageSize * (pageCount - getPragma<int64_t>(sql: "PRAGMA freelist_count"));
840 };
841
842 // The addition of pageSize is a fudge factor to account for non `data` column
843 // size, and because pages can get fragmented on the database.
844 while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
845 // clang-format off
846 mapbox::sqlite::Query accessedQuery{ getStatement(
847 sql: "SELECT max(accessed) "
848 "FROM ( "
849 " SELECT accessed "
850 " FROM resources "
851 " LEFT JOIN region_resources "
852 " ON resource_id = resources.id "
853 " WHERE resource_id IS NULL "
854 " UNION ALL "
855 " SELECT accessed "
856 " FROM tiles "
857 " LEFT JOIN region_tiles "
858 " ON tile_id = tiles.id "
859 " WHERE tile_id IS NULL "
860 " ORDER BY accessed ASC LIMIT ?1 "
861 ") "
862 ) };
863 accessedQuery.bind(offset: 1, value: 50);
864 // clang-format on
865 if (!accessedQuery.run()) {
866 return false;
867 }
868 Timestamp accessed = accessedQuery.get<Timestamp>(offset: 0);
869
870 // clang-format off
871 mapbox::sqlite::Query resourceQuery{ getStatement(
872 sql: "DELETE FROM resources "
873 "WHERE id IN ( "
874 " SELECT id FROM resources "
875 " LEFT JOIN region_resources "
876 " ON resource_id = resources.id "
877 " WHERE resource_id IS NULL "
878 " AND accessed <= ?1 "
879 ") ") };
880 // clang-format on
881 resourceQuery.bind(offset: 1, value: accessed);
882 resourceQuery.run();
883 const uint64_t resourceChanges = resourceQuery.changes();
884
885 // clang-format off
886 mapbox::sqlite::Query tileQuery{ getStatement(
887 sql: "DELETE FROM tiles "
888 "WHERE id IN ( "
889 " SELECT id FROM tiles "
890 " LEFT JOIN region_tiles "
891 " ON tile_id = tiles.id "
892 " WHERE tile_id IS NULL "
893 " AND accessed <= ?1 "
894 ") ") };
895 // clang-format on
896 tileQuery.bind(offset: 1, value: accessed);
897 tileQuery.run();
898 const uint64_t tileChanges = tileQuery.changes();
899
900 // The cached value of offlineTileCount does not need to be updated
901 // here because only non-offline tiles can be removed by eviction.
902
903 if (resourceChanges == 0 && tileChanges == 0) {
904 return false;
905 }
906 }
907
908 return true;
909}
910
911void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
912 offlineMapboxTileCountLimit = limit;
913}
914
915uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
916 return offlineMapboxTileCountLimit;
917}
918
919bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
920 return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
921}
922
923uint64_t OfflineDatabase::getOfflineMapboxTileCount() {
924 // Calculating this on every call would be much simpler than caching and
925 // manually updating the value, but it would make offline downloads an O(n²)
926 // operation, because the database query below involves an index scan of
927 // region_tiles.
928
929 if (offlineMapboxTileCount) {
930 return *offlineMapboxTileCount;
931 }
932
933 // clang-format off
934 mapbox::sqlite::Query query{ getStatement(
935 sql: "SELECT COUNT(DISTINCT id) "
936 "FROM region_tiles, tiles "
937 "WHERE tile_id = tiles.id "
938 "AND url_template LIKE 'mapbox://%' ") };
939 // clang-format on
940
941 query.run();
942
943 offlineMapboxTileCount = query.get<int64_t>(offset: 0);
944 return *offlineMapboxTileCount;
945}
946
947bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) {
948 return resource.kind == Resource::Kind::Tile
949 && util::mapbox::isMapboxURL(url: resource.url)
950 && offlineMapboxTileCountLimitExceeded();
951}
952
953} // namespace mbgl
954

source code of qtlocation/src/3rdparty/mapbox-gl-native/platform/default/mbgl/storage/offline_database.cpp