1#include <mbgl/storage/online_file_source.hpp>
2#include <mbgl/storage/http_file_source.hpp>
3#include <mbgl/storage/network_status.hpp>
4
5#include <mbgl/storage/resource_transform.hpp>
6#include <mbgl/storage/response.hpp>
7#include <mbgl/util/logging.hpp>
8
9#include <mbgl/actor/mailbox.hpp>
10#include <mbgl/util/constants.hpp>
11#include <mbgl/util/mapbox.hpp>
12#include <mbgl/util/exception.hpp>
13#include <mbgl/util/chrono.hpp>
14#include <mbgl/util/async_task.hpp>
15#include <mbgl/util/noncopyable.hpp>
16#include <mbgl/util/run_loop.hpp>
17#include <mbgl/util/timer.hpp>
18#include <mbgl/util/http_timeout.hpp>
19
20#include <algorithm>
21#include <cassert>
22#include <list>
23#include <unordered_set>
24#include <unordered_map>
25
26namespace mbgl {
27
28class OnlineFileRequest : public AsyncRequest {
29public:
30 using Callback = std::function<void (Response)>;
31
32 OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&);
33 ~OnlineFileRequest() override;
34
35 void networkIsReachableAgain();
36 void schedule();
37 void schedule(optional<Timestamp> expires);
38 void completed(Response);
39
40 void setTransformedURL(const std::string&& url);
41 ActorRef<OnlineFileRequest> actor();
42
43 OnlineFileSource::Impl& impl;
44 Resource resource;
45 std::unique_ptr<AsyncRequest> request;
46 util::Timer timer;
47 Callback callback;
48
49 std::shared_ptr<Mailbox> mailbox;
50
51 // Counts the number of times a response was already expired when received. We're using
52 // this to add a delay when making a new request so we don't keep retrying immediately
53 // in case of a server serving expired tiles.
54 uint32_t expiredRequests = 0;
55
56 // Counts the number of subsequent failed requests. We're using this value for exponential
57 // backoff when retrying requests.
58 uint32_t failedRequests = 0;
59 Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
60 optional<Timestamp> retryAfter;
61};
62
63class OnlineFileSource::Impl {
64public:
65 Impl() {
66 NetworkStatus::Subscribe(async: &reachability);
67 }
68
69 ~Impl() {
70 NetworkStatus::Unsubscribe(async: &reachability);
71 }
72
73 void add(OnlineFileRequest* request) {
74 allRequests.insert(x: request);
75 if (resourceTransform) {
76 // Request the ResourceTransform actor a new url and replace the resource url with the
77 // transformed one before proceeding to schedule the request.
78 resourceTransform->invoke(fn: &ResourceTransform::transform, args&: request->resource.kind,
79 args: std::move(request->resource.url), args: [ref = request->actor()](const std::string&& url) mutable {
80 ref.invoke(fn: &OnlineFileRequest::setTransformedURL, args: std::move(url));
81 });
82 } else {
83 request->schedule();
84 }
85 }
86
87 void remove(OnlineFileRequest* request) {
88 allRequests.erase(x: request);
89 if (activeRequests.erase(x: request)) {
90 activatePendingRequest();
91 } else {
92 auto it = pendingRequestsMap.find(x: request);
93 if (it != pendingRequestsMap.end()) {
94 pendingRequestsList.erase(position: it->second);
95 pendingRequestsMap.erase(position: it);
96 }
97 }
98 assert(pendingRequestsMap.size() == pendingRequestsList.size());
99 }
100
101 void activateOrQueueRequest(OnlineFileRequest* request) {
102 assert(allRequests.find(request) != allRequests.end());
103 assert(activeRequests.find(request) == activeRequests.end());
104 assert(!request->request);
105
106 if (activeRequests.size() >= HTTPFileSource::maximumConcurrentRequests()) {
107 queueRequest(request);
108 } else {
109 activateRequest(request);
110 }
111 }
112
113 void queueRequest(OnlineFileRequest* request) {
114 auto it = pendingRequestsList.insert(position: pendingRequestsList.end(), x: request);
115 pendingRequestsMap.emplace(args&: request, args: std::move(it));
116 assert(pendingRequestsMap.size() == pendingRequestsList.size());
117 }
118
119 void activateRequest(OnlineFileRequest* request) {
120 auto callback = [=](Response response) {
121 activeRequests.erase(x: request);
122 request->request.reset();
123 request->completed(response);
124 activatePendingRequest();
125 };
126
127 activeRequests.insert(x: request);
128
129 if (online) {
130 request->request = httpFileSource.request(request->resource, callback);
131 } else {
132 Response response;
133 response.error = std::make_unique<Response::Error>(args: Response::Error::Reason::Connection,
134 args: "Online connectivity is disabled.");
135 callback(response);
136 }
137
138 assert(pendingRequestsMap.size() == pendingRequestsList.size());
139 }
140
141 void activatePendingRequest() {
142 if (pendingRequestsList.empty()) {
143 return;
144 }
145
146 OnlineFileRequest* request = pendingRequestsList.front();
147 pendingRequestsList.pop_front();
148
149 pendingRequestsMap.erase(x: request);
150
151 activateRequest(request);
152 assert(pendingRequestsMap.size() == pendingRequestsList.size());
153 }
154
155 bool isPending(OnlineFileRequest* request) {
156 return pendingRequestsMap.find(x: request) != pendingRequestsMap.end();
157 }
158
159 bool isActive(OnlineFileRequest* request) {
160 return activeRequests.find(x: request) != activeRequests.end();
161 }
162
163 void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
164 resourceTransform = std::move(transform);
165 }
166
167 void setOnlineStatus(const bool status) {
168 online = status;
169 networkIsReachableAgain();
170 }
171
172private:
173 void networkIsReachableAgain() {
174 for (auto& request : allRequests) {
175 request->networkIsReachableAgain();
176 }
177 }
178
179 optional<ActorRef<ResourceTransform>> resourceTransform;
180
181 /**
182 * The lifetime of a request is:
183 *
184 * 1. Waiting for timeout (revalidation or retry)
185 * 2. Pending (waiting for room in the active set)
186 * 3. Active (open network connection)
187 * 4. Back to #1
188 *
189 * Requests in any state are in `allRequests`. Requests in the pending state are in
190 * `pendingRequests`. Requests in the active state are in `activeRequests`.
191 */
192 std::unordered_set<OnlineFileRequest*> allRequests;
193 std::list<OnlineFileRequest*> pendingRequestsList;
194 std::unordered_map<OnlineFileRequest*, std::list<OnlineFileRequest*>::iterator> pendingRequestsMap;
195 std::unordered_set<OnlineFileRequest*> activeRequests;
196
197 bool online = true;
198 HTTPFileSource httpFileSource;
199 util::AsyncTask reachability { std::bind(f: &Impl::networkIsReachableAgain, args: this) };
200};
201
202OnlineFileSource::OnlineFileSource()
203 : impl(std::make_unique<Impl>()) {
204}
205
206OnlineFileSource::~OnlineFileSource() = default;
207
208std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) {
209 Resource res = resource;
210
211 switch (resource.kind) {
212 case Resource::Kind::Unknown:
213 case Resource::Kind::Image:
214 break;
215
216 case Resource::Kind::Style:
217 res.url = mbgl::util::mapbox::normalizeStyleURL(baseURL: apiBaseURL, url: resource.url, accessToken);
218 break;
219
220 case Resource::Kind::Source:
221 res.url = util::mapbox::normalizeSourceURL(baseURL: apiBaseURL, url: resource.url, accessToken);
222 break;
223
224 case Resource::Kind::Glyphs:
225 res.url = util::mapbox::normalizeGlyphsURL(baseURL: apiBaseURL, url: resource.url, accessToken);
226 break;
227
228 case Resource::Kind::SpriteImage:
229 case Resource::Kind::SpriteJSON:
230 res.url = util::mapbox::normalizeSpriteURL(baseURL: apiBaseURL, url: resource.url, accessToken);
231 break;
232
233 case Resource::Kind::Tile:
234 res.url = util::mapbox::normalizeTileURL(baseURL: apiBaseURL, url: resource.url, accessToken);
235 break;
236 }
237
238 return std::make_unique<OnlineFileRequest>(args: std::move(res), args: std::move(callback), args&: *impl);
239}
240
241void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
242 impl->setResourceTransform(std::move(transform));
243}
244
245OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_)
246 : impl(impl_),
247 resource(std::move(resource_)),
248 callback(std::move(callback_)) {
249 impl.add(request: this);
250}
251
252void OnlineFileRequest::schedule() {
253 // Force an immediate first request if we don't have an expiration time.
254 if (resource.priorExpires) {
255 schedule(expires: resource.priorExpires);
256 } else {
257 schedule(expires: util::now());
258 }
259}
260
261OnlineFileRequest::~OnlineFileRequest() {
262 impl.remove(request: this);
263}
264
265Timestamp interpolateExpiration(const Timestamp& current,
266 optional<Timestamp> prior,
267 bool& expired) {
268 auto now = util::now();
269 if (current > now) {
270 return current;
271 }
272
273 if (!bool(prior)) {
274 expired = true;
275 return current;
276 }
277
278 // Expiring date is going backwards,
279 // fallback to exponential backoff.
280 if (current < *prior) {
281 expired = true;
282 return current;
283 }
284
285 auto delta = current - *prior;
286
287 // Server is serving the same expired resource
288 // over and over, fallback to exponential backoff.
289 if (delta == Duration::zero()) {
290 expired = true;
291 return current;
292 }
293
294 // Assume that either the client or server clock is wrong and
295 // try to interpolate a valid expiration date (from the client POV)
296 // observing a minimum timeout.
297 return now + std::max<Seconds>(a: delta, b: util::CLOCK_SKEW_RETRY_TIMEOUT);
298}
299
300void OnlineFileRequest::schedule(optional<Timestamp> expires) {
301 if (impl.isPending(request: this) || impl.isActive(request: this)) {
302 // There's already a request in progress; don't start another one.
303 return;
304 }
305
306 // If we're not being asked for a forced refresh, calculate a timeout that depends on how many
307 // consecutive errors we've encountered, and on the expiration time, if present.
308 Duration timeout = std::min(
309 a: http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
310 b: http::expirationTimeout(expires, expiredRequests));
311
312 if (timeout == Duration::max()) {
313 return;
314 }
315
316 // Emulate a Connection error when the Offline mode is forced with
317 // a really long timeout. The request will get re-triggered when
318 // the NetworkStatus is set back to Online.
319 if (NetworkStatus::Get() == NetworkStatus::Status::Offline) {
320 failedRequestReason = Response::Error::Reason::Connection;
321 failedRequests = 1;
322 timeout = Duration::max();
323 }
324
325 timer.start(timeout, repeat: Duration::zero(), [&] {
326 impl.activateOrQueueRequest(request: this);
327 });
328}
329
330void OnlineFileRequest::completed(Response response) {
331 // If we didn't get various caching headers in the response, continue using the
332 // previous values. Otherwise, update the previous values to the new values.
333
334 if (!response.modified) {
335 response.modified = resource.priorModified;
336 } else {
337 resource.priorModified = response.modified;
338 }
339
340 if (response.notModified && resource.priorData) {
341 // When the priorData field is set, it indicates that we had to revalidate the request and
342 // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we
343 // have send the cached data to give the requestor a chance to actually obtain the data.
344 response.data = std::move(resource.priorData);
345 response.notModified = false;
346 }
347
348 bool isExpired = false;
349
350 if (response.expires) {
351 auto prior = resource.priorExpires;
352 resource.priorExpires = response.expires;
353 response.expires = interpolateExpiration(current: *response.expires, prior, expired&: isExpired);
354 }
355
356 if (isExpired) {
357 expiredRequests++;
358 } else {
359 expiredRequests = 0;
360 }
361
362 if (!response.etag) {
363 response.etag = resource.priorEtag;
364 } else {
365 resource.priorEtag = response.etag;
366 }
367
368 if (response.error) {
369 failedRequests++;
370 failedRequestReason = response.error->reason;
371 retryAfter = response.error->retryAfter;
372 } else {
373 failedRequests = 0;
374 failedRequestReason = Response::Error::Reason::Success;
375 }
376
377 schedule(expires: response.expires);
378
379 // Calling the callback may result in `this` being deleted. It needs to be done last,
380 // and needs to make a local copy of the callback to ensure that it remains valid for
381 // the duration of the call.
382 auto callback_ = callback;
383 callback_(response);
384}
385
386void OnlineFileRequest::networkIsReachableAgain() {
387 // We need all requests to fail at least once before we are going to start retrying
388 // them, and we only immediately restart request that failed due to connection issues.
389 if (failedRequestReason == Response::Error::Reason::Connection) {
390 schedule(expires: util::now());
391 }
392}
393
394void OnlineFileRequest::setTransformedURL(const std::string&& url) {
395 resource.url = std::move(url);
396 schedule();
397}
398
399ActorRef<OnlineFileRequest> OnlineFileRequest::actor() {
400 if (!mailbox) {
401 // Lazy constructed because this can be costly and
402 // the ResourceTransform is not used by many apps.
403 mailbox = std::make_shared<Mailbox>(args&: *Scheduler::GetCurrent());
404 }
405
406 return ActorRef<OnlineFileRequest>(*this, mailbox);
407}
408
409// For testing only:
410
411void OnlineFileSource::setOnlineStatus(const bool status) {
412 impl->setOnlineStatus(status);
413}
414
415} // namespace mbgl
416

source code of qtlocation/src/3rdparty/mapbox-gl-native/platform/default/online_file_source.cpp