Warning: This file is not a C or C++ file. It does not have highlighting.
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
---|---|
2 | /* |
3 | * DAMON api |
4 | * |
5 | * Author: SeongJae Park <sj@kernel.org> |
6 | */ |
7 | |
8 | #ifndef _DAMON_H_ |
9 | #define _DAMON_H_ |
10 | |
11 | #include <linux/memcontrol.h> |
12 | #include <linux/mutex.h> |
13 | #include <linux/time64.h> |
14 | #include <linux/types.h> |
15 | #include <linux/random.h> |
16 | |
17 | /* Minimal region size. Every damon_region is aligned by this. */ |
18 | #define DAMON_MIN_REGION PAGE_SIZE |
19 | /* Max priority score for DAMON-based operation schemes */ |
20 | #define DAMOS_MAX_SCORE (99) |
21 | |
22 | /* Get a random number in [l, r) */ |
23 | static inline unsigned long damon_rand(unsigned long l, unsigned long r) |
24 | { |
25 | return l + get_random_u32_below(r - l); |
26 | } |
27 | |
28 | /** |
29 | * struct damon_addr_range - Represents an address region of [@start, @end). |
30 | * @start: Start address of the region (inclusive). |
31 | * @end: End address of the region (exclusive). |
32 | */ |
33 | struct damon_addr_range { |
34 | unsigned long start; |
35 | unsigned long end; |
36 | }; |
37 | |
38 | /** |
39 | * struct damon_region - Represents a monitoring target region. |
40 | * @ar: The address range of the region. |
41 | * @sampling_addr: Address of the sample for the next access check. |
42 | * @nr_accesses: Access frequency of this region. |
43 | * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for |
44 | * each sampling interval. |
45 | * @list: List head for siblings. |
46 | * @age: Age of this region. |
47 | * |
48 | * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be |
49 | * increased for every &damon_attrs->sample_interval if an access to the region |
50 | * during the last sampling interval is found. The update of this field should |
51 | * not be done with direct access but with the helper function, |
52 | * damon_update_region_access_rate(). |
53 | * |
54 | * @nr_accesses_bp is another representation of @nr_accesses in basis point |
55 | * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a |
56 | * manner similar to moving sum. By the algorithm, this value becomes |
57 | * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can |
58 | * be used when the aggregation interval is too huge and therefore cannot wait |
59 | * for it before getting the access monitoring results. |
60 | * |
61 | * @age is initially zero, increased for each aggregation interval, and reset |
62 | * to zero again if the access frequency is significantly changed. If two |
63 | * regions are merged into a new region, both @nr_accesses and @age of the new |
64 | * region are set as region size-weighted average of those of the two regions. |
65 | */ |
66 | struct damon_region { |
67 | struct damon_addr_range ar; |
68 | unsigned long sampling_addr; |
69 | unsigned int nr_accesses; |
70 | unsigned int nr_accesses_bp; |
71 | struct list_head list; |
72 | |
73 | unsigned int age; |
74 | /* private: Internal value for age calculation. */ |
75 | unsigned int last_nr_accesses; |
76 | }; |
77 | |
78 | /** |
79 | * struct damon_target - Represents a monitoring target. |
80 | * @pid: The PID of the virtual address space to monitor. |
81 | * @nr_regions: Number of monitoring target regions of this target. |
82 | * @regions_list: Head of the monitoring target regions of this target. |
83 | * @list: List head for siblings. |
84 | * |
85 | * Each monitoring context could have multiple targets. For example, a context |
86 | * for virtual memory address spaces could have multiple target processes. The |
87 | * @pid should be set for appropriate &struct damon_operations including the |
88 | * virtual address spaces monitoring operations. |
89 | */ |
90 | struct damon_target { |
91 | struct pid *pid; |
92 | unsigned int nr_regions; |
93 | struct list_head regions_list; |
94 | struct list_head list; |
95 | }; |
96 | |
97 | /** |
98 | * enum damos_action - Represents an action of a Data Access Monitoring-based |
99 | * Operation Scheme. |
100 | * |
101 | * @DAMOS_WILLNEED: Call ``madvise()`` for the region with MADV_WILLNEED. |
102 | * @DAMOS_COLD: Call ``madvise()`` for the region with MADV_COLD. |
103 | * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. |
104 | * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. |
105 | * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. |
106 | * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. |
107 | * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. |
108 | * @DAMOS_STAT: Do nothing but count the stat. |
109 | * @NR_DAMOS_ACTIONS: Total number of DAMOS actions |
110 | * |
111 | * The support of each action is up to running &struct damon_operations. |
112 | * &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR supports all actions except |
113 | * &enum DAMOS_LRU_PRIO and &enum DAMOS_LRU_DEPRIO. &enum DAMON_OPS_PADDR |
114 | * supports only &enum DAMOS_PAGEOUT, &enum DAMOS_LRU_PRIO, &enum |
115 | * DAMOS_LRU_DEPRIO, and &DAMOS_STAT. |
116 | */ |
117 | enum damos_action { |
118 | DAMOS_WILLNEED, |
119 | DAMOS_COLD, |
120 | DAMOS_PAGEOUT, |
121 | DAMOS_HUGEPAGE, |
122 | DAMOS_NOHUGEPAGE, |
123 | DAMOS_LRU_PRIO, |
124 | DAMOS_LRU_DEPRIO, |
125 | DAMOS_STAT, /* Do nothing but only record the stat */ |
126 | NR_DAMOS_ACTIONS, |
127 | }; |
128 | |
129 | /** |
130 | * enum damos_quota_goal_metric - Represents the metric to be used as the goal |
131 | * |
132 | * @DAMOS_QUOTA_USER_INPUT: User-input value. |
133 | * @DAMOS_QUOTA_SOME_MEM_PSI_US: System level some memory PSI in us. |
134 | * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. |
135 | * |
136 | * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. |
137 | */ |
138 | enum damos_quota_goal_metric { |
139 | DAMOS_QUOTA_USER_INPUT, |
140 | DAMOS_QUOTA_SOME_MEM_PSI_US, |
141 | NR_DAMOS_QUOTA_GOAL_METRICS, |
142 | }; |
143 | |
144 | /** |
145 | * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. |
146 | * @metric: Metric to be used for representing the goal. |
147 | * @target_value: Target value of @metric to achieve with the tuning. |
148 | * @current_value: Current value of @metric. |
149 | * @last_psi_total: Last measured total PSI |
150 | * @list: List head for siblings. |
151 | * |
152 | * Data structure for getting the current score of the quota tuning goal. The |
153 | * score is calculated by how close @current_value and @target_value are. Then |
154 | * the score is entered to DAMON's internal feedback loop mechanism to get the |
155 | * auto-tuned quota. |
156 | * |
157 | * If @metric is DAMOS_QUOTA_USER_INPUT, @current_value should be manually |
158 | * entered by the user, probably inside the kdamond callbacks. Otherwise, |
159 | * DAMON sets @current_value with self-measured value of @metric. |
160 | */ |
161 | struct damos_quota_goal { |
162 | enum damos_quota_goal_metric metric; |
163 | unsigned long target_value; |
164 | unsigned long current_value; |
165 | /* metric-dependent fields */ |
166 | union { |
167 | u64 last_psi_total; |
168 | }; |
169 | struct list_head list; |
170 | }; |
171 | |
172 | /** |
173 | * struct damos_quota - Controls the aggressiveness of the given scheme. |
174 | * @reset_interval: Charge reset interval in milliseconds. |
175 | * @ms: Maximum milliseconds that the scheme can use. |
176 | * @sz: Maximum bytes of memory that the action can be applied. |
177 | * @goals: Head of quota tuning goals (&damos_quota_goal) list. |
178 | * @esz: Effective size quota in bytes. |
179 | * |
180 | * @weight_sz: Weight of the region's size for prioritization. |
181 | * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. |
182 | * @weight_age: Weight of the region's age for prioritization. |
183 | * |
184 | * To avoid consuming too much CPU time or IO resources for applying the |
185 | * &struct damos->action to large memory, DAMON allows users to set time and/or |
186 | * size quotas. The quotas can be set by writing non-zero values to &ms and |
187 | * &sz, respectively. If the time quota is set, DAMON tries to use only up to |
188 | * &ms milliseconds within &reset_interval for applying the action. If the |
189 | * size quota is set, DAMON tries to apply the action only up to &sz bytes |
190 | * within &reset_interval. |
191 | * |
192 | * Internally, the time quota is transformed to a size quota using estimated |
193 | * throughput of the scheme's action. DAMON then compares it against &sz and |
194 | * uses smaller one as the effective quota. |
195 | * |
196 | * If @goals is not empt, DAMON calculates yet another size quota based on the |
197 | * goals using its internal feedback loop algorithm, for every @reset_interval. |
198 | * Then, if the new size quota is smaller than the effective quota, it uses the |
199 | * new size quota as the effective quota. |
200 | * |
201 | * The resulting effective size quota in bytes is set to @esz. |
202 | * |
203 | * For selecting regions within the quota, DAMON prioritizes current scheme's |
204 | * target memory regions using the &struct damon_operations->get_scheme_score. |
205 | * You could customize the prioritization logic by setting &weight_sz, |
206 | * &weight_nr_accesses, and &weight_age, because monitoring operations are |
207 | * encouraged to respect those. |
208 | */ |
209 | struct damos_quota { |
210 | unsigned long reset_interval; |
211 | unsigned long ms; |
212 | unsigned long sz; |
213 | struct list_head goals; |
214 | unsigned long esz; |
215 | |
216 | unsigned int weight_sz; |
217 | unsigned int weight_nr_accesses; |
218 | unsigned int weight_age; |
219 | |
220 | /* private: */ |
221 | /* For throughput estimation */ |
222 | unsigned long total_charged_sz; |
223 | unsigned long total_charged_ns; |
224 | |
225 | /* For charging the quota */ |
226 | unsigned long charged_sz; |
227 | unsigned long charged_from; |
228 | struct damon_target *charge_target_from; |
229 | unsigned long charge_addr_from; |
230 | |
231 | /* For prioritization */ |
232 | unsigned long histogram[DAMOS_MAX_SCORE + 1]; |
233 | unsigned int min_score; |
234 | |
235 | /* For feedback loop */ |
236 | unsigned long esz_bp; |
237 | }; |
238 | |
239 | /** |
240 | * enum damos_wmark_metric - Represents the watermark metric. |
241 | * |
242 | * @DAMOS_WMARK_NONE: Ignore the watermarks of the given scheme. |
243 | * @DAMOS_WMARK_FREE_MEM_RATE: Free memory rate of the system in [0,1000]. |
244 | * @NR_DAMOS_WMARK_METRICS: Total number of DAMOS watermark metrics |
245 | */ |
246 | enum damos_wmark_metric { |
247 | DAMOS_WMARK_NONE, |
248 | DAMOS_WMARK_FREE_MEM_RATE, |
249 | NR_DAMOS_WMARK_METRICS, |
250 | }; |
251 | |
252 | /** |
253 | * struct damos_watermarks - Controls when a given scheme should be activated. |
254 | * @metric: Metric for the watermarks. |
255 | * @interval: Watermarks check time interval in microseconds. |
256 | * @high: High watermark. |
257 | * @mid: Middle watermark. |
258 | * @low: Low watermark. |
259 | * |
260 | * If &metric is &DAMOS_WMARK_NONE, the scheme is always active. Being active |
261 | * means DAMON does monitoring and applying the action of the scheme to |
262 | * appropriate memory regions. Else, DAMON checks &metric of the system for at |
263 | * least every &interval microseconds and works as below. |
264 | * |
265 | * If &metric is higher than &high, the scheme is inactivated. If &metric is |
266 | * between &mid and &low, the scheme is activated. If &metric is lower than |
267 | * &low, the scheme is inactivated. |
268 | */ |
269 | struct damos_watermarks { |
270 | enum damos_wmark_metric metric; |
271 | unsigned long interval; |
272 | unsigned long high; |
273 | unsigned long mid; |
274 | unsigned long low; |
275 | |
276 | /* private: */ |
277 | bool activated; |
278 | }; |
279 | |
280 | /** |
281 | * struct damos_stat - Statistics on a given scheme. |
282 | * @nr_tried: Total number of regions that the scheme is tried to be applied. |
283 | * @sz_tried: Total size of regions that the scheme is tried to be applied. |
284 | * @nr_applied: Total number of regions that the scheme is applied. |
285 | * @sz_applied: Total size of regions that the scheme is applied. |
286 | * @qt_exceeds: Total number of times the quota of the scheme has exceeded. |
287 | */ |
288 | struct damos_stat { |
289 | unsigned long nr_tried; |
290 | unsigned long sz_tried; |
291 | unsigned long nr_applied; |
292 | unsigned long sz_applied; |
293 | unsigned long qt_exceeds; |
294 | }; |
295 | |
296 | /** |
297 | * enum damos_filter_type - Type of memory for &struct damos_filter |
298 | * @DAMOS_FILTER_TYPE_ANON: Anonymous pages. |
299 | * @DAMOS_FILTER_TYPE_MEMCG: Specific memcg's pages. |
300 | * @DAMOS_FILTER_TYPE_ADDR: Address range. |
301 | * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. |
302 | * @NR_DAMOS_FILTER_TYPES: Number of filter types. |
303 | * |
304 | * The anon pages type and memcg type filters are handled by underlying |
305 | * &struct damon_operations as a part of scheme action trying, and therefore |
306 | * accounted as 'tried'. In contrast, other types are handled by core layer |
307 | * before trying of the action and therefore not accounted as 'tried'. |
308 | * |
309 | * The support of the filters that handled by &struct damon_operations depend |
310 | * on the running &struct damon_operations. |
311 | * &enum DAMON_OPS_PADDR supports both anon pages type and memcg type filters, |
312 | * while &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR don't support any of |
313 | * the two types. |
314 | */ |
315 | enum damos_filter_type { |
316 | DAMOS_FILTER_TYPE_ANON, |
317 | DAMOS_FILTER_TYPE_MEMCG, |
318 | DAMOS_FILTER_TYPE_ADDR, |
319 | DAMOS_FILTER_TYPE_TARGET, |
320 | NR_DAMOS_FILTER_TYPES, |
321 | }; |
322 | |
323 | /** |
324 | * struct damos_filter - DAMOS action target memory filter. |
325 | * @type: Type of the page. |
326 | * @matching: If the matching page should filtered out or in. |
327 | * @memcg_id: Memcg id of the question if @type is DAMOS_FILTER_MEMCG. |
328 | * @addr_range: Address range if @type is DAMOS_FILTER_TYPE_ADDR. |
329 | * @target_idx: Index of the &struct damon_target of |
330 | * &damon_ctx->adaptive_targets if @type is |
331 | * DAMOS_FILTER_TYPE_TARGET. |
332 | * @list: List head for siblings. |
333 | * |
334 | * Before applying the &damos->action to a memory region, DAMOS checks if each |
335 | * page of the region matches to this and avoid applying the action if so. |
336 | * Support of each filter type depends on the running &struct damon_operations |
337 | * and the type. Refer to &enum damos_filter_type for more detai. |
338 | */ |
339 | struct damos_filter { |
340 | enum damos_filter_type type; |
341 | bool matching; |
342 | union { |
343 | unsigned short memcg_id; |
344 | struct damon_addr_range addr_range; |
345 | int target_idx; |
346 | }; |
347 | struct list_head list; |
348 | }; |
349 | |
350 | /** |
351 | * struct damos_access_pattern - Target access pattern of the given scheme. |
352 | * @min_sz_region: Minimum size of target regions. |
353 | * @max_sz_region: Maximum size of target regions. |
354 | * @min_nr_accesses: Minimum ``->nr_accesses`` of target regions. |
355 | * @max_nr_accesses: Maximum ``->nr_accesses`` of target regions. |
356 | * @min_age_region: Minimum age of target regions. |
357 | * @max_age_region: Maximum age of target regions. |
358 | */ |
359 | struct damos_access_pattern { |
360 | unsigned long min_sz_region; |
361 | unsigned long max_sz_region; |
362 | unsigned int min_nr_accesses; |
363 | unsigned int max_nr_accesses; |
364 | unsigned int min_age_region; |
365 | unsigned int max_age_region; |
366 | }; |
367 | |
368 | /** |
369 | * struct damos - Represents a Data Access Monitoring-based Operation Scheme. |
370 | * @pattern: Access pattern of target regions. |
371 | * @action: &damo_action to be applied to the target regions. |
372 | * @apply_interval_us: The time between applying the @action. |
373 | * @quota: Control the aggressiveness of this scheme. |
374 | * @wmarks: Watermarks for automated (in)activation of this scheme. |
375 | * @filters: Additional set of &struct damos_filter for &action. |
376 | * @stat: Statistics of this scheme. |
377 | * @list: List head for siblings. |
378 | * |
379 | * For each @apply_interval_us, DAMON finds regions which fit in the |
380 | * &pattern and applies &action to those. To avoid consuming too much |
381 | * CPU time or IO resources for the &action, "a is used. |
382 | * |
383 | * If @apply_interval_us is zero, &damon_attrs->aggr_interval is used instead. |
384 | * |
385 | * To do the work only when needed, schemes can be activated for specific |
386 | * system situations using &wmarks. If all schemes that registered to the |
387 | * monitoring context are inactive, DAMON stops monitoring either, and just |
388 | * repeatedly checks the watermarks. |
389 | * |
390 | * Before applying the &action to a memory region, &struct damon_operations |
391 | * implementation could check pages of the region and skip &action to respect |
392 | * &filters |
393 | * |
394 | * After applying the &action to each region, &stat_count and &stat_sz is |
395 | * updated to reflect the number of regions and total size of regions that the |
396 | * &action is applied. |
397 | */ |
398 | struct damos { |
399 | struct damos_access_pattern pattern; |
400 | enum damos_action action; |
401 | unsigned long apply_interval_us; |
402 | /* private: internal use only */ |
403 | /* |
404 | * number of sample intervals that should be passed before applying |
405 | * @action |
406 | */ |
407 | unsigned long next_apply_sis; |
408 | /* public: */ |
409 | struct damos_quota quota; |
410 | struct damos_watermarks wmarks; |
411 | struct list_head filters; |
412 | struct damos_stat stat; |
413 | struct list_head list; |
414 | }; |
415 | |
416 | /** |
417 | * enum damon_ops_id - Identifier for each monitoring operations implementation |
418 | * |
419 | * @DAMON_OPS_VADDR: Monitoring operations for virtual address spaces |
420 | * @DAMON_OPS_FVADDR: Monitoring operations for only fixed ranges of virtual |
421 | * address spaces |
422 | * @DAMON_OPS_PADDR: Monitoring operations for the physical address space |
423 | * @NR_DAMON_OPS: Number of monitoring operations implementations |
424 | */ |
425 | enum damon_ops_id { |
426 | DAMON_OPS_VADDR, |
427 | DAMON_OPS_FVADDR, |
428 | DAMON_OPS_PADDR, |
429 | NR_DAMON_OPS, |
430 | }; |
431 | |
432 | struct damon_ctx; |
433 | |
434 | /** |
435 | * struct damon_operations - Monitoring operations for given use cases. |
436 | * |
437 | * @id: Identifier of this operations set. |
438 | * @init: Initialize operations-related data structures. |
439 | * @update: Update operations-related data structures. |
440 | * @prepare_access_checks: Prepare next access check of target regions. |
441 | * @check_accesses: Check the accesses to target regions. |
442 | * @reset_aggregated: Reset aggregated accesses monitoring results. |
443 | * @get_scheme_score: Get the score of a region for a scheme. |
444 | * @apply_scheme: Apply a DAMON-based operation scheme. |
445 | * @target_valid: Determine if the target is valid. |
446 | * @cleanup: Clean up the context. |
447 | * |
448 | * DAMON can be extended for various address spaces and usages. For this, |
449 | * users should register the low level operations for their target address |
450 | * space and usecase via the &damon_ctx.ops. Then, the monitoring thread |
451 | * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting |
452 | * the monitoring, @update after each &damon_attrs.ops_update_interval, and |
453 | * @check_accesses, @target_valid and @prepare_access_checks after each |
454 | * &damon_attrs.sample_interval. Finally, @reset_aggregated is called after |
455 | * each &damon_attrs.aggr_interval. |
456 | * |
457 | * Each &struct damon_operations instance having valid @id can be registered |
458 | * via damon_register_ops() and selected by damon_select_ops() later. |
459 | * @init should initialize operations-related data structures. For example, |
460 | * this could be used to construct proper monitoring target regions and link |
461 | * those to @damon_ctx.adaptive_targets. |
462 | * @update should update the operations-related data structures. For example, |
463 | * this could be used to update monitoring target regions for current status. |
464 | * @prepare_access_checks should manipulate the monitoring regions to be |
465 | * prepared for the next access check. |
466 | * @check_accesses should check the accesses to each region that made after the |
467 | * last preparation and update the number of observed accesses of each region. |
468 | * It should also return max number of observed accesses that made as a result |
469 | * of its update. The value will be used for regions adjustment threshold. |
470 | * @reset_aggregated should reset the access monitoring results that aggregated |
471 | * by @check_accesses. |
472 | * @get_scheme_score should return the priority score of a region for a scheme |
473 | * as an integer in [0, &DAMOS_MAX_SCORE]. |
474 | * @apply_scheme is called from @kdamond when a region for user provided |
475 | * DAMON-based operation scheme is found. It should apply the scheme's action |
476 | * to the region and return bytes of the region that the action is successfully |
477 | * applied. |
478 | * @target_valid should check whether the target is still valid for the |
479 | * monitoring. |
480 | * @cleanup is called from @kdamond just before its termination. |
481 | */ |
482 | struct damon_operations { |
483 | enum damon_ops_id id; |
484 | void (*init)(struct damon_ctx *context); |
485 | void (*update)(struct damon_ctx *context); |
486 | void (*prepare_access_checks)(struct damon_ctx *context); |
487 | unsigned int (*check_accesses)(struct damon_ctx *context); |
488 | void (*reset_aggregated)(struct damon_ctx *context); |
489 | int (*get_scheme_score)(struct damon_ctx *context, |
490 | struct damon_target *t, struct damon_region *r, |
491 | struct damos *scheme); |
492 | unsigned long (*apply_scheme)(struct damon_ctx *context, |
493 | struct damon_target *t, struct damon_region *r, |
494 | struct damos *scheme); |
495 | bool (*target_valid)(struct damon_target *t); |
496 | void (*cleanup)(struct damon_ctx *context); |
497 | }; |
498 | |
499 | /** |
500 | * struct damon_callback - Monitoring events notification callbacks. |
501 | * |
502 | * @before_start: Called before starting the monitoring. |
503 | * @after_wmarks_check: Called after each schemes' watermarks check. |
504 | * @after_sampling: Called after each sampling. |
505 | * @after_aggregation: Called after each aggregation. |
506 | * @before_damos_apply: Called before applying DAMOS action. |
507 | * @before_terminate: Called before terminating the monitoring. |
508 | * @private: User private data. |
509 | * |
510 | * The monitoring thread (&damon_ctx.kdamond) calls @before_start and |
511 | * @before_terminate just before starting and finishing the monitoring, |
512 | * respectively. Therefore, those are good places for installing and cleaning |
513 | * @private. |
514 | * |
515 | * The monitoring thread calls @after_wmarks_check after each DAMON-based |
516 | * operation schemes' watermarks check. If users need to make changes to the |
517 | * attributes of the monitoring context while it's deactivated due to the |
518 | * watermarks, this is the good place to do. |
519 | * |
520 | * The monitoring thread calls @after_sampling and @after_aggregation for each |
521 | * of the sampling intervals and aggregation intervals, respectively. |
522 | * Therefore, users can safely access the monitoring results without additional |
523 | * protection. For the reason, users are recommended to use these callback for |
524 | * the accesses to the results. |
525 | * |
526 | * If any callback returns non-zero, monitoring stops. |
527 | */ |
528 | struct damon_callback { |
529 | void *private; |
530 | |
531 | int (*before_start)(struct damon_ctx *context); |
532 | int (*after_wmarks_check)(struct damon_ctx *context); |
533 | int (*after_sampling)(struct damon_ctx *context); |
534 | int (*after_aggregation)(struct damon_ctx *context); |
535 | int (*before_damos_apply)(struct damon_ctx *context, |
536 | struct damon_target *target, |
537 | struct damon_region *region, |
538 | struct damos *scheme); |
539 | void (*before_terminate)(struct damon_ctx *context); |
540 | }; |
541 | |
542 | /** |
543 | * struct damon_attrs - Monitoring attributes for accuracy/overhead control. |
544 | * |
545 | * @sample_interval: The time between access samplings. |
546 | * @aggr_interval: The time between monitor results aggregations. |
547 | * @ops_update_interval: The time between monitoring operations updates. |
548 | * @min_nr_regions: The minimum number of adaptive monitoring |
549 | * regions. |
550 | * @max_nr_regions: The maximum number of adaptive monitoring |
551 | * regions. |
552 | * |
553 | * For each @sample_interval, DAMON checks whether each region is accessed or |
554 | * not during the last @sample_interval. If such access is found, DAMON |
555 | * aggregates the information by increasing &damon_region->nr_accesses for |
556 | * @aggr_interval time. For each @aggr_interval, the count is reset. DAMON |
557 | * also checks whether the target memory regions need update (e.g., by |
558 | * ``mmap()`` calls from the application, in case of virtual memory monitoring) |
559 | * and applies the changes for each @ops_update_interval. All time intervals |
560 | * are in micro-seconds. Please refer to &struct damon_operations and &struct |
561 | * damon_callback for more detail. |
562 | */ |
563 | struct damon_attrs { |
564 | unsigned long sample_interval; |
565 | unsigned long aggr_interval; |
566 | unsigned long ops_update_interval; |
567 | unsigned long min_nr_regions; |
568 | unsigned long max_nr_regions; |
569 | }; |
570 | |
571 | /** |
572 | * struct damon_ctx - Represents a context for each monitoring. This is the |
573 | * main interface that allows users to set the attributes and get the results |
574 | * of the monitoring. |
575 | * |
576 | * @attrs: Monitoring attributes for accuracy/overhead control. |
577 | * @kdamond: Kernel thread who does the monitoring. |
578 | * @kdamond_lock: Mutex for the synchronizations with @kdamond. |
579 | * |
580 | * For each monitoring context, one kernel thread for the monitoring is |
581 | * created. The pointer to the thread is stored in @kdamond. |
582 | * |
583 | * Once started, the monitoring thread runs until explicitly required to be |
584 | * terminated or every monitoring target is invalid. The validity of the |
585 | * targets is checked via the &damon_operations.target_valid of @ops. The |
586 | * termination can also be explicitly requested by calling damon_stop(). |
587 | * The thread sets @kdamond to NULL when it terminates. Therefore, users can |
588 | * know whether the monitoring is ongoing or terminated by reading @kdamond. |
589 | * Reads and writes to @kdamond from outside of the monitoring thread must |
590 | * be protected by @kdamond_lock. |
591 | * |
592 | * Note that the monitoring thread protects only @kdamond via @kdamond_lock. |
593 | * Accesses to other fields must be protected by themselves. |
594 | * |
595 | * @ops: Set of monitoring operations for given use cases. |
596 | * @callback: Set of callbacks for monitoring events notifications. |
597 | * |
598 | * @adaptive_targets: Head of monitoring targets (&damon_target) list. |
599 | * @schemes: Head of schemes (&damos) list. |
600 | */ |
601 | struct damon_ctx { |
602 | struct damon_attrs attrs; |
603 | |
604 | /* private: internal use only */ |
605 | /* number of sample intervals that passed since this context started */ |
606 | unsigned long passed_sample_intervals; |
607 | /* |
608 | * number of sample intervals that should be passed before next |
609 | * aggregation |
610 | */ |
611 | unsigned long next_aggregation_sis; |
612 | /* |
613 | * number of sample intervals that should be passed before next ops |
614 | * update |
615 | */ |
616 | unsigned long next_ops_update_sis; |
617 | /* for waiting until the execution of the kdamond_fn is started */ |
618 | struct completion kdamond_started; |
619 | |
620 | /* public: */ |
621 | struct task_struct *kdamond; |
622 | struct mutex kdamond_lock; |
623 | |
624 | struct damon_operations ops; |
625 | struct damon_callback callback; |
626 | |
627 | struct list_head adaptive_targets; |
628 | struct list_head schemes; |
629 | }; |
630 | |
631 | static inline struct damon_region *damon_next_region(struct damon_region *r) |
632 | { |
633 | return container_of(r->list.next, struct damon_region, list); |
634 | } |
635 | |
636 | static inline struct damon_region *damon_prev_region(struct damon_region *r) |
637 | { |
638 | return container_of(r->list.prev, struct damon_region, list); |
639 | } |
640 | |
641 | static inline struct damon_region *damon_last_region(struct damon_target *t) |
642 | { |
643 | return list_last_entry(&t->regions_list, struct damon_region, list); |
644 | } |
645 | |
646 | static inline struct damon_region *damon_first_region(struct damon_target *t) |
647 | { |
648 | return list_first_entry(&t->regions_list, struct damon_region, list); |
649 | } |
650 | |
651 | static inline unsigned long damon_sz_region(struct damon_region *r) |
652 | { |
653 | return r->ar.end - r->ar.start; |
654 | } |
655 | |
656 | |
657 | #define damon_for_each_region(r, t) \ |
658 | list_for_each_entry(r, &t->regions_list, list) |
659 | |
660 | #define damon_for_each_region_from(r, t) \ |
661 | list_for_each_entry_from(r, &t->regions_list, list) |
662 | |
663 | #define damon_for_each_region_safe(r, next, t) \ |
664 | list_for_each_entry_safe(r, next, &t->regions_list, list) |
665 | |
666 | #define damon_for_each_target(t, ctx) \ |
667 | list_for_each_entry(t, &(ctx)->adaptive_targets, list) |
668 | |
669 | #define damon_for_each_target_safe(t, next, ctx) \ |
670 | list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list) |
671 | |
672 | #define damon_for_each_scheme(s, ctx) \ |
673 | list_for_each_entry(s, &(ctx)->schemes, list) |
674 | |
675 | #define damon_for_each_scheme_safe(s, next, ctx) \ |
676 | list_for_each_entry_safe(s, next, &(ctx)->schemes, list) |
677 | |
678 | #define damos_for_each_quota_goal(goal, quota) \ |
679 | list_for_each_entry(goal, "a->goals, list) |
680 | |
681 | #define damos_for_each_quota_goal_safe(goal, next, quota) \ |
682 | list_for_each_entry_safe(goal, next, &(quota)->goals, list) |
683 | |
684 | #define damos_for_each_filter(f, scheme) \ |
685 | list_for_each_entry(f, &(scheme)->filters, list) |
686 | |
687 | #define damos_for_each_filter_safe(f, next, scheme) \ |
688 | list_for_each_entry_safe(f, next, &(scheme)->filters, list) |
689 | |
690 | #ifdef CONFIG_DAMON |
691 | |
692 | struct damon_region *damon_new_region(unsigned long start, unsigned long end); |
693 | |
694 | /* |
695 | * Add a region between two other regions |
696 | */ |
697 | static inline void damon_insert_region(struct damon_region *r, |
698 | struct damon_region *prev, struct damon_region *next, |
699 | struct damon_target *t) |
700 | { |
701 | __list_add(&r->list, &prev->list, &next->list); |
702 | t->nr_regions++; |
703 | } |
704 | |
705 | void damon_add_region(struct damon_region *r, struct damon_target *t); |
706 | void damon_destroy_region(struct damon_region *r, struct damon_target *t); |
707 | int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, |
708 | unsigned int nr_ranges); |
709 | void damon_update_region_access_rate(struct damon_region *r, bool accessed, |
710 | struct damon_attrs *attrs); |
711 | |
712 | struct damos_filter *damos_new_filter(enum damos_filter_type type, |
713 | bool matching); |
714 | void damos_add_filter(struct damos *s, struct damos_filter *f); |
715 | void damos_destroy_filter(struct damos_filter *f); |
716 | |
717 | struct damos_quota_goal *damos_new_quota_goal( |
718 | enum damos_quota_goal_metric metric, |
719 | unsigned long target_value); |
720 | void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); |
721 | void damos_destroy_quota_goal(struct damos_quota_goal *goal); |
722 | |
723 | struct damos *damon_new_scheme(struct damos_access_pattern *pattern, |
724 | enum damos_action action, |
725 | unsigned long apply_interval_us, |
726 | struct damos_quota *quota, |
727 | struct damos_watermarks *wmarks); |
728 | void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); |
729 | void damon_destroy_scheme(struct damos *s); |
730 | |
731 | struct damon_target *damon_new_target(void); |
732 | void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); |
733 | bool damon_targets_empty(struct damon_ctx *ctx); |
734 | void damon_free_target(struct damon_target *t); |
735 | void damon_destroy_target(struct damon_target *t); |
736 | unsigned int damon_nr_regions(struct damon_target *t); |
737 | |
738 | struct damon_ctx *damon_new_ctx(void); |
739 | void damon_destroy_ctx(struct damon_ctx *ctx); |
740 | int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); |
741 | void damon_set_schemes(struct damon_ctx *ctx, |
742 | struct damos **schemes, ssize_t nr_schemes); |
743 | int damon_nr_running_ctxs(void); |
744 | bool damon_is_registered_ops(enum damon_ops_id id); |
745 | int damon_register_ops(struct damon_operations *ops); |
746 | int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); |
747 | |
748 | static inline bool damon_target_has_pid(const struct damon_ctx *ctx) |
749 | { |
750 | return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; |
751 | } |
752 | |
753 | static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) |
754 | { |
755 | /* {aggr,sample}_interval are unsigned long, hence could overflow */ |
756 | return min(attrs->aggr_interval / attrs->sample_interval, |
757 | (unsigned long)UINT_MAX); |
758 | } |
759 | |
760 | |
761 | int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); |
762 | int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); |
763 | |
764 | int damon_set_region_biggest_system_ram_default(struct damon_target *t, |
765 | unsigned long *start, unsigned long *end); |
766 | |
767 | #endif /* CONFIG_DAMON */ |
768 | |
769 | #endif /* _DAMON_H */ |
770 |
Warning: This file is not a C or C++ file. It does not have highlighting.