Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Limited memmove with inline #5891

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+147 −66
Draft

Limited memmove with inline #5891

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ChangeLog
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

This release:
- Significantly reduces the libpmem's stack usage.
- Make memmove_mov_* function non-inline to prevent huge stack consumption.

Tue Aug 8 2023 Oksana Sałyk <[email protected]>

Expand Down
2 changes: 1 addition & 1 deletion src/libpmem2/x86_64/memcpy/memcpy_t_avx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ memmove_mov_avx_bw(char *dest, const char *src, size_t len,
memmove_small_avx(dest - len, src - len, len, flush);
}

static force_inline void
static void
memmove_mov_avx(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
6 changes: 3 additions & 3 deletions src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2017-2020, Intel Corporation */
/* Copyright 2017-2023, Intel Corporation */

#include <immintrin.h>
#include <stddef.h>
Expand Down Expand Up @@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
flush64b(dest + 0 * 64);
}

static force_inline void
static void
memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down Expand Up @@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
memmove_small_avx512f(dest, src, len, flush);
}

static force_inline void
static void
memmove_mov_avx512f_bw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
4 changes: 2 additions & 2 deletions src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
flush64b(dest + 0 * 64);
}

static force_inline void
static void
memmove_mov_sse_fw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down Expand Up @@ -151,7 +151,7 @@ memmove_mov_sse_fw(char *dest, const char *src, size_t len,
memmove_small_sse2(dest, src, len, flush);
}

static force_inline void
static void
memmove_mov_sse_bw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
70 changes: 40 additions & 30 deletions src/stats/stack-usage-debug.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,10 @@
51344 memmove_movnt_avx512f_clflush : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
25344 memmove_mov_avx512f_fw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_fw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_bw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_bw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
24144 memmove_movnt_avx_noflush_nobarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
Expand Down Expand Up @@ -79,16 +73,8 @@
19024 memset_mov_avx512f_clflushopt : src/debug/libpmem/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded
19024 memset_mov_avx512f_clflushopt : src/debug/libpmem2/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded
16480 obj_vg_check_no_undef : src/debug/libpmemobj/obj.su:obj.c static
13840 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13792 memmove_mov_avx : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static
13792 memmove_mov_avx : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static
9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_noflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
Expand All @@ -109,16 +95,6 @@
9808 memset_movnt_avx_clflushopt_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9200 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
8464 out_error : src/debug/libpmempool/out.su:out.c dynamic,bounded
8464 out_error : src/debug/libpmem/out.su:out.c dynamic,bounded
8464 out_error : src/debug/libpmemobj/out.su:out.c dynamic,bounded
Expand Down Expand Up @@ -165,6 +141,10 @@
5936 memset_mov_avx_clflush : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded
5936 memset_mov_avx_clflushopt : src/debug/libpmem/memset_t_avx.su:memset_t_avx.c dynamic,bounded
5936 memset_mov_avx_clflushopt : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded
4576 memmove_mov_sse_fw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_fw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_bw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_bw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static
4384 pool_params_parse : src/debug/libpmempool/pool.su:pool.c dynamic,bounded
4352 out_init : src/debug/libpmempool/out.su:out.c dynamic,bounded
4352 out_init : src/debug/libpmem/out.su:out.c dynamic,bounded
Expand Down Expand Up @@ -952,6 +932,26 @@
112 obj_open_common : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 obj_norep_memmove : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 obj_norep_memcpy : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memblock_run_bitmap : src/debug/libpmemobj/memblock.su:memblock.c dynamic,bounded
112 list_remove_single : src/debug/libpmemobj/list.su:list.c dynamic,bounded
112 lane_boot : src/debug/libpmemobj/lane.su:lane.c static
Expand Down Expand Up @@ -1672,6 +1672,16 @@
64 obj_replica_init : src/debug/libpmemobj/obj.su:obj.c static
64 obj_pool_open : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
64 obj_ctl_init_and_load : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
64 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 membuf_alloc : src/debug/libpmempool/membuf.su:membuf.c static
64 membuf_alloc : src/debug/libpmemobj/membuf.su:membuf.c static
64 membuf_alloc : src/debug/libpmem2/membuf.su:membuf.c static
Expand Down
Loading
Loading