Skip to content

Commit 0600b28

Browse files
johnstultz-workomasse-nxp
johnstultz-work
and
omasse-nxp
committed
ANDROID: dma-buf: heaps: Add a shrinker controlled page pool
This patch adds a simple shrinker controlled page pool to the dmabuf heaps subsystem. This replaces the use of the networking page_pool, over concerns that the lack of a shrinker for that implementation may cause additional low-memory kills TODO: Take another pass at trying to unify this w/ the ttm pool Thoughts and feedback would be greatly appreciated! Cc: Sumit Semwal <[email protected]> Cc: Liam Mark <[email protected]> Cc: Laura Abbott <[email protected]> Cc: Brian Starkey <[email protected]> Cc: Hridya Valsaraju <[email protected]> Cc: Suren Baghdasaryan <[email protected]> Cc: Sandeep Patil <[email protected]> Cc: Daniel Mentz <[email protected]> Cc: Chris Goldsworthy <[email protected]> Cc: Ørjan Eide <[email protected]> Cc: Robin Murphy <[email protected]> Cc: Ezequiel Garcia <[email protected]> Cc: Simon Ser <[email protected]> Cc: James Jones <[email protected]> Cc: Andrew Morton <[email protected]> Cc: Dave Hansen <[email protected]> Cc: [email protected] Cc: [email protected] Cc: [email protected] Co-authored-by: Olivier Masse <[email protected]> Signed-off-by: Olivier Masse <[email protected]> Bug: 168742043
1 parent 7927f02 commit 0600b28

File tree

4 files changed

+305
-0
lines changed

4 files changed

+305
-0
lines changed

‎drivers/dma-buf/heaps/Kconfig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
config DMABUF_HEAPS_DEFERRED_FREE
22
tristate
33

4+
config DMABUF_HEAPS_PAGE_POOL
5+
tristate
6+
47
config DMABUF_HEAPS_SYSTEM
58
bool "DMA-BUF System Heap"
69
depends on DMABUF_HEAPS

‎drivers/dma-buf/heaps/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# SPDX-License-Identifier: GPL-2.0
22
obj-$(CONFIG_DMABUF_HEAPS_DEFERRED_FREE) += deferred-free-helper.o
3+
obj-$(CONFIG_DMABUF_HEAPS_PAGE_POOL) += page_pool.o
34
obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += system_heap.o
45
obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o

‎drivers/dma-buf/heaps/page_pool.c

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
/*
3+
* DMA BUF page pool system
4+
*
5+
* Copyright (C) 2020 Linaro Ltd.
6+
*
7+
* Based on the ION page pool code
8+
* Copyright (C) 2011 Google, Inc.
9+
*/
10+
11+
#include <linux/freezer.h>
12+
#include <linux/list.h>
13+
#include <linux/sched/signal.h>
14+
#include <linux/slab.h>
15+
#include <linux/swap.h>
16+
#include "page_pool.h"
17+
18+
static LIST_HEAD(pool_list);
19+
static DEFINE_MUTEX(pool_list_lock);
20+
21+
static struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool)
22+
{
23+
if (fatal_signal_pending(current))
24+
return NULL;
25+
return alloc_pages(pool->gfp_mask, pool->order);
26+
}
27+
28+
static void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool,
29+
struct page *page)
30+
{
31+
__free_pages(page, pool->order);
32+
}
33+
34+
static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page)
35+
{
36+
int index;
37+
38+
if (PageHighMem(page))
39+
index = POOL_HIGHPAGE;
40+
else
41+
index = POOL_LOWPAGE;
42+
43+
mutex_lock(&pool->mutex);
44+
list_add_tail(&page->lru, &pool->items[index]);
45+
pool->count[index]++;
46+
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
47+
1 << pool->order);
48+
mutex_unlock(&pool->mutex);
49+
}
50+
51+
static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index)
52+
{
53+
struct page *page;
54+
55+
mutex_lock(&pool->mutex);
56+
page = list_first_entry_or_null(&pool->items[index], struct page, lru);
57+
if (page) {
58+
pool->count[index]--;
59+
list_del(&page->lru);
60+
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
61+
-(1 << pool->order));
62+
}
63+
mutex_unlock(&pool->mutex);
64+
65+
return page;
66+
}
67+
68+
static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool)
69+
{
70+
struct page *page = NULL;
71+
72+
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
73+
if (!page)
74+
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
75+
76+
return page;
77+
}
78+
79+
struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool)
80+
{
81+
struct page *page = NULL;
82+
83+
if (WARN_ON(!pool))
84+
return NULL;
85+
86+
page = dmabuf_page_pool_fetch(pool);
87+
if (!page)
88+
page = dmabuf_page_pool_alloc_pages(pool);
89+
90+
return page;
91+
}
92+
EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc);
93+
94+
void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page)
95+
{
96+
if (WARN_ON(pool->order != compound_order(page)))
97+
return;
98+
99+
dmabuf_page_pool_add(pool, page);
100+
}
101+
EXPORT_SYMBOL_GPL(dmabuf_page_pool_free);
102+
103+
static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high)
104+
{
105+
int count = pool->count[POOL_LOWPAGE];
106+
107+
if (high)
108+
count += pool->count[POOL_HIGHPAGE];
109+
110+
return count << pool->order;
111+
}
112+
113+
struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order)
114+
{
115+
struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
116+
int i;
117+
118+
if (!pool)
119+
return NULL;
120+
121+
for (i = 0; i < POOL_TYPE_SIZE; i++) {
122+
pool->count[i] = 0;
123+
INIT_LIST_HEAD(&pool->items[i]);
124+
}
125+
pool->gfp_mask = gfp_mask | __GFP_COMP;
126+
pool->order = order;
127+
mutex_init(&pool->mutex);
128+
129+
mutex_lock(&pool_list_lock);
130+
list_add(&pool->list, &pool_list);
131+
mutex_unlock(&pool_list_lock);
132+
133+
return pool;
134+
}
135+
EXPORT_SYMBOL_GPL(dmabuf_page_pool_create);
136+
137+
void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool)
138+
{
139+
struct page *page;
140+
int i;
141+
142+
/* Remove us from the pool list */
143+
mutex_lock(&pool_list_lock);
144+
list_del(&pool->list);
145+
mutex_unlock(&pool_list_lock);
146+
147+
/* Free any remaining pages in the pool */
148+
for (i = 0; i < POOL_TYPE_SIZE; i++) {
149+
while ((page = dmabuf_page_pool_remove(pool, i)))
150+
dmabuf_page_pool_free_pages(pool, page);
151+
}
152+
153+
kfree(pool);
154+
}
155+
EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy);
156+
157+
static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
158+
int nr_to_scan)
159+
{
160+
int freed = 0;
161+
bool high;
162+
163+
if (current_is_kswapd())
164+
high = true;
165+
else
166+
high = !!(gfp_mask & __GFP_HIGHMEM);
167+
168+
if (nr_to_scan == 0)
169+
return dmabuf_page_pool_total(pool, high);
170+
171+
while (freed < nr_to_scan) {
172+
struct page *page;
173+
174+
/* Try to free low pages first */
175+
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
176+
if (!page)
177+
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
178+
179+
if (!page)
180+
break;
181+
182+
dmabuf_page_pool_free_pages(pool, page);
183+
freed += (1 << pool->order);
184+
}
185+
186+
return freed;
187+
}
188+
189+
static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
190+
{
191+
struct dmabuf_page_pool *pool;
192+
int nr_total = 0;
193+
int nr_freed;
194+
bool only_scan = false;
195+
196+
if (!nr_to_scan)
197+
only_scan = true;
198+
199+
mutex_lock(&pool_list_lock);
200+
list_for_each_entry(pool, &pool_list, list) {
201+
if (only_scan) {
202+
nr_total += dmabuf_page_pool_do_shrink(pool,
203+
gfp_mask,
204+
nr_to_scan);
205+
} else {
206+
nr_freed = dmabuf_page_pool_do_shrink(pool,
207+
gfp_mask,
208+
nr_to_scan);
209+
nr_to_scan -= nr_freed;
210+
nr_total += nr_freed;
211+
if (nr_to_scan <= 0)
212+
break;
213+
}
214+
}
215+
mutex_unlock(&pool_list_lock);
216+
217+
return nr_total;
218+
}
219+
220+
static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker,
221+
struct shrink_control *sc)
222+
{
223+
return dmabuf_page_pool_shrink(sc->gfp_mask, 0);
224+
}
225+
226+
static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker,
227+
struct shrink_control *sc)
228+
{
229+
if (sc->nr_to_scan == 0)
230+
return 0;
231+
return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan);
232+
}
233+
234+
struct shrinker pool_shrinker = {
235+
.count_objects = dmabuf_page_pool_shrink_count,
236+
.scan_objects = dmabuf_page_pool_shrink_scan,
237+
.seeks = DEFAULT_SEEKS,
238+
.batch = 0,
239+
};
240+
241+
static int dmabuf_page_pool_init_shrinker(void)
242+
{
243+
return register_shrinker(&pool_shrinker);
244+
}
245+
module_init(dmabuf_page_pool_init_shrinker);
246+
MODULE_LICENSE("GPL v2");

‎drivers/dma-buf/heaps/page_pool.h

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* DMA BUF PagePool implementation
4+
* Based on earlier ION code by Google
5+
*
6+
* Copyright (C) 2011 Google, Inc.
7+
* Copyright (C) 2020 Linaro Ltd.
8+
*/
9+
10+
#ifndef _DMABUF_PAGE_POOL_H
11+
#define _DMABUF_PAGE_POOL_H
12+
13+
#include <linux/device.h>
14+
#include <linux/kref.h>
15+
#include <linux/mm_types.h>
16+
#include <linux/mutex.h>
17+
#include <linux/shrinker.h>
18+
#include <linux/types.h>
19+
20+
/* page types we track in the pool */
21+
enum {
22+
POOL_LOWPAGE, /* Clean lowmem pages */
23+
POOL_HIGHPAGE, /* Clean highmem pages */
24+
25+
POOL_TYPE_SIZE
26+
};
27+
28+
/**
29+
* struct dmabuf_page_pool - pagepool struct
30+
* @count[]: array of number of pages of that type in the pool
31+
* @items[]: array of list of pages of the specific type
32+
* @mutex: lock protecting this struct and especially the count
33+
* item list
34+
* @gfp_mask: gfp_mask to use from alloc
35+
* @order: order of pages in the pool
36+
* @list: list node for list of pools
37+
*
38+
* Allows you to keep a pool of pre allocated pages to use
39+
*/
40+
struct dmabuf_page_pool {
41+
int count[POOL_TYPE_SIZE];
42+
struct list_head items[POOL_TYPE_SIZE];
43+
struct mutex mutex;
44+
gfp_t gfp_mask;
45+
unsigned int order;
46+
struct list_head list;
47+
};
48+
49+
struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask,
50+
unsigned int order);
51+
void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool);
52+
struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool);
53+
void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page);
54+
55+
#endif /* _DMABUF_PAGE_POOL_H */

0 commit comments

Comments
 (0)