1 | #include <mbgl/storage/online_file_source.hpp> |
2 | #include <mbgl/storage/http_file_source.hpp> |
3 | #include <mbgl/storage/network_status.hpp> |
4 | |
5 | #include <mbgl/storage/resource_transform.hpp> |
6 | #include <mbgl/storage/response.hpp> |
7 | #include <mbgl/util/logging.hpp> |
8 | |
9 | #include <mbgl/actor/mailbox.hpp> |
10 | #include <mbgl/util/constants.hpp> |
11 | #include <mbgl/util/mapbox.hpp> |
12 | #include <mbgl/util/exception.hpp> |
13 | #include <mbgl/util/chrono.hpp> |
14 | #include <mbgl/util/async_task.hpp> |
15 | #include <mbgl/util/noncopyable.hpp> |
16 | #include <mbgl/util/run_loop.hpp> |
17 | #include <mbgl/util/timer.hpp> |
18 | #include <mbgl/util/http_timeout.hpp> |
19 | |
20 | #include <algorithm> |
21 | #include <cassert> |
22 | #include <list> |
23 | #include <unordered_set> |
24 | #include <unordered_map> |
25 | |
26 | namespace mbgl { |
27 | |
28 | class OnlineFileRequest : public AsyncRequest { |
29 | public: |
30 | using Callback = std::function<void (Response)>; |
31 | |
32 | OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&); |
33 | ~OnlineFileRequest() override; |
34 | |
35 | void networkIsReachableAgain(); |
36 | void schedule(); |
37 | void schedule(optional<Timestamp> expires); |
38 | void completed(Response); |
39 | |
40 | void setTransformedURL(const std::string&& url); |
41 | ActorRef<OnlineFileRequest> actor(); |
42 | |
43 | OnlineFileSource::Impl& impl; |
44 | Resource resource; |
45 | std::unique_ptr<AsyncRequest> request; |
46 | util::Timer timer; |
47 | Callback callback; |
48 | |
49 | std::shared_ptr<Mailbox> mailbox; |
50 | |
51 | // Counts the number of times a response was already expired when received. We're using |
52 | // this to add a delay when making a new request so we don't keep retrying immediately |
53 | // in case of a server serving expired tiles. |
54 | uint32_t expiredRequests = 0; |
55 | |
56 | // Counts the number of subsequent failed requests. We're using this value for exponential |
57 | // backoff when retrying requests. |
58 | uint32_t failedRequests = 0; |
59 | Response::Error::Reason failedRequestReason = Response::Error::Reason::Success; |
60 | optional<Timestamp> retryAfter; |
61 | }; |
62 | |
63 | class OnlineFileSource::Impl { |
64 | public: |
65 | Impl() { |
66 | NetworkStatus::Subscribe(async: &reachability); |
67 | } |
68 | |
69 | ~Impl() { |
70 | NetworkStatus::Unsubscribe(async: &reachability); |
71 | } |
72 | |
73 | void add(OnlineFileRequest* request) { |
74 | allRequests.insert(x: request); |
75 | if (resourceTransform) { |
76 | // Request the ResourceTransform actor a new url and replace the resource url with the |
77 | // transformed one before proceeding to schedule the request. |
78 | resourceTransform->invoke(fn: &ResourceTransform::transform, args&: request->resource.kind, |
79 | args: std::move(request->resource.url), args: [ref = request->actor()](const std::string&& url) mutable { |
80 | ref.invoke(fn: &OnlineFileRequest::setTransformedURL, args: std::move(url)); |
81 | }); |
82 | } else { |
83 | request->schedule(); |
84 | } |
85 | } |
86 | |
87 | void remove(OnlineFileRequest* request) { |
88 | allRequests.erase(x: request); |
89 | if (activeRequests.erase(x: request)) { |
90 | activatePendingRequest(); |
91 | } else { |
92 | auto it = pendingRequestsMap.find(x: request); |
93 | if (it != pendingRequestsMap.end()) { |
94 | pendingRequestsList.erase(position: it->second); |
95 | pendingRequestsMap.erase(position: it); |
96 | } |
97 | } |
98 | assert(pendingRequestsMap.size() == pendingRequestsList.size()); |
99 | } |
100 | |
101 | void activateOrQueueRequest(OnlineFileRequest* request) { |
102 | assert(allRequests.find(request) != allRequests.end()); |
103 | assert(activeRequests.find(request) == activeRequests.end()); |
104 | assert(!request->request); |
105 | |
106 | if (activeRequests.size() >= HTTPFileSource::maximumConcurrentRequests()) { |
107 | queueRequest(request); |
108 | } else { |
109 | activateRequest(request); |
110 | } |
111 | } |
112 | |
113 | void queueRequest(OnlineFileRequest* request) { |
114 | auto it = pendingRequestsList.insert(position: pendingRequestsList.end(), x: request); |
115 | pendingRequestsMap.emplace(args&: request, args: std::move(it)); |
116 | assert(pendingRequestsMap.size() == pendingRequestsList.size()); |
117 | } |
118 | |
119 | void activateRequest(OnlineFileRequest* request) { |
120 | auto callback = [=](Response response) { |
121 | activeRequests.erase(x: request); |
122 | request->request.reset(); |
123 | request->completed(response); |
124 | activatePendingRequest(); |
125 | }; |
126 | |
127 | activeRequests.insert(x: request); |
128 | |
129 | if (online) { |
130 | request->request = httpFileSource.request(request->resource, callback); |
131 | } else { |
132 | Response response; |
133 | response.error = std::make_unique<Response::Error>(args: Response::Error::Reason::Connection, |
134 | args: "Online connectivity is disabled." ); |
135 | callback(response); |
136 | } |
137 | |
138 | assert(pendingRequestsMap.size() == pendingRequestsList.size()); |
139 | } |
140 | |
141 | void activatePendingRequest() { |
142 | if (pendingRequestsList.empty()) { |
143 | return; |
144 | } |
145 | |
146 | OnlineFileRequest* request = pendingRequestsList.front(); |
147 | pendingRequestsList.pop_front(); |
148 | |
149 | pendingRequestsMap.erase(x: request); |
150 | |
151 | activateRequest(request); |
152 | assert(pendingRequestsMap.size() == pendingRequestsList.size()); |
153 | } |
154 | |
155 | bool isPending(OnlineFileRequest* request) { |
156 | return pendingRequestsMap.find(x: request) != pendingRequestsMap.end(); |
157 | } |
158 | |
159 | bool isActive(OnlineFileRequest* request) { |
160 | return activeRequests.find(x: request) != activeRequests.end(); |
161 | } |
162 | |
163 | void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { |
164 | resourceTransform = std::move(transform); |
165 | } |
166 | |
167 | void setOnlineStatus(const bool status) { |
168 | online = status; |
169 | networkIsReachableAgain(); |
170 | } |
171 | |
172 | private: |
173 | void networkIsReachableAgain() { |
174 | for (auto& request : allRequests) { |
175 | request->networkIsReachableAgain(); |
176 | } |
177 | } |
178 | |
179 | optional<ActorRef<ResourceTransform>> resourceTransform; |
180 | |
181 | /** |
182 | * The lifetime of a request is: |
183 | * |
184 | * 1. Waiting for timeout (revalidation or retry) |
185 | * 2. Pending (waiting for room in the active set) |
186 | * 3. Active (open network connection) |
187 | * 4. Back to #1 |
188 | * |
189 | * Requests in any state are in `allRequests`. Requests in the pending state are in |
190 | * `pendingRequests`. Requests in the active state are in `activeRequests`. |
191 | */ |
192 | std::unordered_set<OnlineFileRequest*> allRequests; |
193 | std::list<OnlineFileRequest*> pendingRequestsList; |
194 | std::unordered_map<OnlineFileRequest*, std::list<OnlineFileRequest*>::iterator> pendingRequestsMap; |
195 | std::unordered_set<OnlineFileRequest*> activeRequests; |
196 | |
197 | bool online = true; |
198 | HTTPFileSource httpFileSource; |
199 | util::AsyncTask reachability { std::bind(f: &Impl::networkIsReachableAgain, args: this) }; |
200 | }; |
201 | |
202 | OnlineFileSource::OnlineFileSource() |
203 | : impl(std::make_unique<Impl>()) { |
204 | } |
205 | |
206 | OnlineFileSource::~OnlineFileSource() = default; |
207 | |
208 | std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) { |
209 | Resource res = resource; |
210 | |
211 | switch (resource.kind) { |
212 | case Resource::Kind::Unknown: |
213 | case Resource::Kind::Image: |
214 | break; |
215 | |
216 | case Resource::Kind::Style: |
217 | res.url = mbgl::util::mapbox::normalizeStyleURL(baseURL: apiBaseURL, url: resource.url, accessToken); |
218 | break; |
219 | |
220 | case Resource::Kind::Source: |
221 | res.url = util::mapbox::normalizeSourceURL(baseURL: apiBaseURL, url: resource.url, accessToken); |
222 | break; |
223 | |
224 | case Resource::Kind::Glyphs: |
225 | res.url = util::mapbox::normalizeGlyphsURL(baseURL: apiBaseURL, url: resource.url, accessToken); |
226 | break; |
227 | |
228 | case Resource::Kind::SpriteImage: |
229 | case Resource::Kind::SpriteJSON: |
230 | res.url = util::mapbox::normalizeSpriteURL(baseURL: apiBaseURL, url: resource.url, accessToken); |
231 | break; |
232 | |
233 | case Resource::Kind::Tile: |
234 | res.url = util::mapbox::normalizeTileURL(baseURL: apiBaseURL, url: resource.url, accessToken); |
235 | break; |
236 | } |
237 | |
238 | return std::make_unique<OnlineFileRequest>(args: std::move(res), args: std::move(callback), args&: *impl); |
239 | } |
240 | |
241 | void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { |
242 | impl->setResourceTransform(std::move(transform)); |
243 | } |
244 | |
245 | OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_) |
246 | : impl(impl_), |
247 | resource(std::move(resource_)), |
248 | callback(std::move(callback_)) { |
249 | impl.add(request: this); |
250 | } |
251 | |
252 | void OnlineFileRequest::schedule() { |
253 | // Force an immediate first request if we don't have an expiration time. |
254 | if (resource.priorExpires) { |
255 | schedule(expires: resource.priorExpires); |
256 | } else { |
257 | schedule(expires: util::now()); |
258 | } |
259 | } |
260 | |
261 | OnlineFileRequest::~OnlineFileRequest() { |
262 | impl.remove(request: this); |
263 | } |
264 | |
265 | Timestamp interpolateExpiration(const Timestamp& current, |
266 | optional<Timestamp> prior, |
267 | bool& expired) { |
268 | auto now = util::now(); |
269 | if (current > now) { |
270 | return current; |
271 | } |
272 | |
273 | if (!bool(prior)) { |
274 | expired = true; |
275 | return current; |
276 | } |
277 | |
278 | // Expiring date is going backwards, |
279 | // fallback to exponential backoff. |
280 | if (current < *prior) { |
281 | expired = true; |
282 | return current; |
283 | } |
284 | |
285 | auto delta = current - *prior; |
286 | |
287 | // Server is serving the same expired resource |
288 | // over and over, fallback to exponential backoff. |
289 | if (delta == Duration::zero()) { |
290 | expired = true; |
291 | return current; |
292 | } |
293 | |
294 | // Assume that either the client or server clock is wrong and |
295 | // try to interpolate a valid expiration date (from the client POV) |
296 | // observing a minimum timeout. |
297 | return now + std::max<Seconds>(a: delta, b: util::CLOCK_SKEW_RETRY_TIMEOUT); |
298 | } |
299 | |
300 | void OnlineFileRequest::schedule(optional<Timestamp> expires) { |
301 | if (impl.isPending(request: this) || impl.isActive(request: this)) { |
302 | // There's already a request in progress; don't start another one. |
303 | return; |
304 | } |
305 | |
306 | // If we're not being asked for a forced refresh, calculate a timeout that depends on how many |
307 | // consecutive errors we've encountered, and on the expiration time, if present. |
308 | Duration timeout = std::min( |
309 | a: http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), |
310 | b: http::expirationTimeout(expires, expiredRequests)); |
311 | |
312 | if (timeout == Duration::max()) { |
313 | return; |
314 | } |
315 | |
316 | // Emulate a Connection error when the Offline mode is forced with |
317 | // a really long timeout. The request will get re-triggered when |
318 | // the NetworkStatus is set back to Online. |
319 | if (NetworkStatus::Get() == NetworkStatus::Status::Offline) { |
320 | failedRequestReason = Response::Error::Reason::Connection; |
321 | failedRequests = 1; |
322 | timeout = Duration::max(); |
323 | } |
324 | |
325 | timer.start(timeout, repeat: Duration::zero(), [&] { |
326 | impl.activateOrQueueRequest(request: this); |
327 | }); |
328 | } |
329 | |
330 | void OnlineFileRequest::completed(Response response) { |
331 | // If we didn't get various caching headers in the response, continue using the |
332 | // previous values. Otherwise, update the previous values to the new values. |
333 | |
334 | if (!response.modified) { |
335 | response.modified = resource.priorModified; |
336 | } else { |
337 | resource.priorModified = response.modified; |
338 | } |
339 | |
340 | if (response.notModified && resource.priorData) { |
341 | // When the priorData field is set, it indicates that we had to revalidate the request and |
342 | // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we |
343 | // have send the cached data to give the requestor a chance to actually obtain the data. |
344 | response.data = std::move(resource.priorData); |
345 | response.notModified = false; |
346 | } |
347 | |
348 | bool isExpired = false; |
349 | |
350 | if (response.expires) { |
351 | auto prior = resource.priorExpires; |
352 | resource.priorExpires = response.expires; |
353 | response.expires = interpolateExpiration(current: *response.expires, prior, expired&: isExpired); |
354 | } |
355 | |
356 | if (isExpired) { |
357 | expiredRequests++; |
358 | } else { |
359 | expiredRequests = 0; |
360 | } |
361 | |
362 | if (!response.etag) { |
363 | response.etag = resource.priorEtag; |
364 | } else { |
365 | resource.priorEtag = response.etag; |
366 | } |
367 | |
368 | if (response.error) { |
369 | failedRequests++; |
370 | failedRequestReason = response.error->reason; |
371 | retryAfter = response.error->retryAfter; |
372 | } else { |
373 | failedRequests = 0; |
374 | failedRequestReason = Response::Error::Reason::Success; |
375 | } |
376 | |
377 | schedule(expires: response.expires); |
378 | |
379 | // Calling the callback may result in `this` being deleted. It needs to be done last, |
380 | // and needs to make a local copy of the callback to ensure that it remains valid for |
381 | // the duration of the call. |
382 | auto callback_ = callback; |
383 | callback_(response); |
384 | } |
385 | |
386 | void OnlineFileRequest::networkIsReachableAgain() { |
387 | // We need all requests to fail at least once before we are going to start retrying |
388 | // them, and we only immediately restart request that failed due to connection issues. |
389 | if (failedRequestReason == Response::Error::Reason::Connection) { |
390 | schedule(expires: util::now()); |
391 | } |
392 | } |
393 | |
394 | void OnlineFileRequest::setTransformedURL(const std::string&& url) { |
395 | resource.url = std::move(url); |
396 | schedule(); |
397 | } |
398 | |
399 | ActorRef<OnlineFileRequest> OnlineFileRequest::actor() { |
400 | if (!mailbox) { |
401 | // Lazy constructed because this can be costly and |
402 | // the ResourceTransform is not used by many apps. |
403 | mailbox = std::make_shared<Mailbox>(args&: *Scheduler::GetCurrent()); |
404 | } |
405 | |
406 | return ActorRef<OnlineFileRequest>(*this, mailbox); |
407 | } |
408 | |
409 | // For testing only: |
410 | |
411 | void OnlineFileSource::setOnlineStatus(const bool status) { |
412 | impl->setOnlineStatus(status); |
413 | } |
414 | |
415 | } // namespace mbgl |
416 | |