1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/kernel.h> |
3 | #include <linux/errno.h> |
4 | #include <linux/err.h> |
5 | #include <linux/mm.h> |
6 | #include <linux/slab.h> |
7 | #include <linux/vmalloc.h> |
8 | #include <linux/pagemap.h> |
9 | #include <linux/sched.h> |
10 | |
11 | #include <media/frame_vector.h> |
12 | |
13 | /** |
14 | * get_vaddr_frames() - map virtual addresses to pfns |
15 | * @start: starting user address |
16 | * @nr_frames: number of pages / pfns from start to map |
17 | * @write: the mapped address has write permission |
18 | * @vec: structure which receives pages / pfns of the addresses mapped. |
19 | * It should have space for at least nr_frames entries. |
20 | * |
21 | * This function maps virtual addresses from @start and fills @vec structure |
22 | * with page frame numbers or page pointers to corresponding pages (choice |
23 | * depends on the type of the vma underlying the virtual address). If @start |
24 | * belongs to a normal vma, the function grabs reference to each of the pages |
25 | * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't |
26 | * touch page structures and the caller must make sure pfns aren't reused for |
27 | * anything else while he is using them. |
28 | * |
29 | * The function returns number of pages mapped which may be less than |
30 | * @nr_frames. In particular we stop mapping if there are more vmas of |
31 | * different type underlying the specified range of virtual addresses. |
32 | * When the function isn't able to map a single page, it returns error. |
33 | * |
34 | * Note that get_vaddr_frames() cannot follow VM_IO mappings. It used |
35 | * to be able to do that, but that could (racily) return non-refcounted |
36 | * pfns. |
37 | * |
38 | * This function takes care of grabbing mmap_lock as necessary. |
39 | */ |
40 | int get_vaddr_frames(unsigned long start, unsigned int nr_frames, bool write, |
41 | struct frame_vector *vec) |
42 | { |
43 | int ret; |
44 | unsigned int gup_flags = FOLL_LONGTERM; |
45 | |
46 | if (nr_frames == 0) |
47 | return 0; |
48 | |
49 | if (WARN_ON_ONCE(nr_frames > vec->nr_allocated)) |
50 | nr_frames = vec->nr_allocated; |
51 | |
52 | start = untagged_addr(start); |
53 | |
54 | if (write) |
55 | gup_flags |= FOLL_WRITE; |
56 | |
57 | ret = pin_user_pages_fast(start, nr_pages: nr_frames, gup_flags, |
58 | pages: (struct page **)(vec->ptrs)); |
59 | vec->got_ref = true; |
60 | vec->is_pfns = false; |
61 | vec->nr_frames = ret; |
62 | |
63 | if (likely(ret > 0)) |
64 | return ret; |
65 | |
66 | vec->nr_frames = 0; |
67 | return ret ? ret : -EFAULT; |
68 | } |
69 | EXPORT_SYMBOL(get_vaddr_frames); |
70 | |
71 | /** |
72 | * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired |
73 | * them |
74 | * @vec: frame vector to put |
75 | * |
76 | * Drop references to pages if get_vaddr_frames() acquired them. We also |
77 | * invalidate the frame vector so that it is prepared for the next call into |
78 | * get_vaddr_frames(). |
79 | */ |
80 | void put_vaddr_frames(struct frame_vector *vec) |
81 | { |
82 | struct page **pages; |
83 | |
84 | if (!vec->got_ref) |
85 | goto out; |
86 | pages = frame_vector_pages(vec); |
87 | /* |
88 | * frame_vector_pages() might needed to do a conversion when |
89 | * get_vaddr_frames() got pages but vec was later converted to pfns. |
90 | * But it shouldn't really fail to convert pfns back... |
91 | */ |
92 | if (WARN_ON(IS_ERR(pages))) |
93 | goto out; |
94 | |
95 | unpin_user_pages(pages, npages: vec->nr_frames); |
96 | vec->got_ref = false; |
97 | out: |
98 | vec->nr_frames = 0; |
99 | } |
100 | EXPORT_SYMBOL(put_vaddr_frames); |
101 | |
102 | /** |
103 | * frame_vector_to_pages - convert frame vector to contain page pointers |
104 | * @vec: frame vector to convert |
105 | * |
106 | * Convert @vec to contain array of page pointers. If the conversion is |
107 | * successful, return 0. Otherwise return an error. Note that we do not grab |
108 | * page references for the page structures. |
109 | */ |
110 | int frame_vector_to_pages(struct frame_vector *vec) |
111 | { |
112 | int i; |
113 | unsigned long *nums; |
114 | struct page **pages; |
115 | |
116 | if (!vec->is_pfns) |
117 | return 0; |
118 | nums = frame_vector_pfns(vec); |
119 | for (i = 0; i < vec->nr_frames; i++) |
120 | if (!pfn_valid(pfn: nums[i])) |
121 | return -EINVAL; |
122 | pages = (struct page **)nums; |
123 | for (i = 0; i < vec->nr_frames; i++) |
124 | pages[i] = pfn_to_page(nums[i]); |
125 | vec->is_pfns = false; |
126 | return 0; |
127 | } |
128 | EXPORT_SYMBOL(frame_vector_to_pages); |
129 | |
130 | /** |
131 | * frame_vector_to_pfns - convert frame vector to contain pfns |
132 | * @vec: frame vector to convert |
133 | * |
134 | * Convert @vec to contain array of pfns. |
135 | */ |
136 | void frame_vector_to_pfns(struct frame_vector *vec) |
137 | { |
138 | int i; |
139 | unsigned long *nums; |
140 | struct page **pages; |
141 | |
142 | if (vec->is_pfns) |
143 | return; |
144 | pages = (struct page **)(vec->ptrs); |
145 | nums = (unsigned long *)pages; |
146 | for (i = 0; i < vec->nr_frames; i++) |
147 | nums[i] = page_to_pfn(pages[i]); |
148 | vec->is_pfns = true; |
149 | } |
150 | EXPORT_SYMBOL(frame_vector_to_pfns); |
151 | |
152 | /** |
153 | * frame_vector_create() - allocate & initialize structure for pinned pfns |
154 | * @nr_frames: number of pfns slots we should reserve |
155 | * |
156 | * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns |
157 | * pfns. |
158 | */ |
159 | struct frame_vector *frame_vector_create(unsigned int nr_frames) |
160 | { |
161 | struct frame_vector *vec; |
162 | int size = struct_size(vec, ptrs, nr_frames); |
163 | |
164 | if (WARN_ON_ONCE(nr_frames == 0)) |
165 | return NULL; |
166 | /* |
167 | * This is absurdly high. It's here just to avoid strange effects when |
168 | * arithmetics overflows. |
169 | */ |
170 | if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2)) |
171 | return NULL; |
172 | /* |
173 | * Avoid higher order allocations, use vmalloc instead. It should |
174 | * be rare anyway. |
175 | */ |
176 | vec = kvmalloc(size, GFP_KERNEL); |
177 | if (!vec) |
178 | return NULL; |
179 | vec->nr_allocated = nr_frames; |
180 | vec->nr_frames = 0; |
181 | return vec; |
182 | } |
183 | EXPORT_SYMBOL(frame_vector_create); |
184 | |
185 | /** |
186 | * frame_vector_destroy() - free memory allocated to carry frame vector |
187 | * @vec: Frame vector to free |
188 | * |
189 | * Free structure allocated by frame_vector_create() to carry frames. |
190 | */ |
191 | void frame_vector_destroy(struct frame_vector *vec) |
192 | { |
193 | /* Make sure put_vaddr_frames() got called properly... */ |
194 | VM_BUG_ON(vec->nr_frames > 0); |
195 | kvfree(addr: vec); |
196 | } |
197 | EXPORT_SYMBOL(frame_vector_destroy); |
198 | |