From 78b74c146244241f440dfea9ae2a203e295d876b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 24 Oct 2023 11:13:12 +0200 Subject: [PATCH] common: reduce stack usage by eliminating complex memmove_ inline function Signed-off-by: Tomasz Gromadzki --- ChangeLog | 1 + src/libpmem2/x86_64/memcpy/memcpy_t_avx.c | 2 +- src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c | 6 +- src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c | 4 +- src/stats/stack-usage-debug.txt | 70 ++++++---- src/stats/stack-usage-nondebug.txt | 130 ++++++++++++++---- 6 files changed, 147 insertions(+), 66 deletions(-) diff --git a/ChangeLog b/ChangeLog index b2df03d8f63..e2b2203cec8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ This release: - Significantly reduces the libpmem's stack usage. + - Make memmove_mov_* function non-inline to prevent huge stack consumption. Tue Aug 8 2023 Oksana Sałyk diff --git a/src/libpmem2/x86_64/memcpy/memcpy_t_avx.c b/src/libpmem2/x86_64/memcpy/memcpy_t_avx.c index c4780cabab2..bf6e4288e2f 100644 --- a/src/libpmem2/x86_64/memcpy/memcpy_t_avx.c +++ b/src/libpmem2/x86_64/memcpy/memcpy_t_avx.c @@ -227,7 +227,7 @@ memmove_mov_avx_bw(char *dest, const char *src, size_t len, memmove_small_avx(dest - len, src - len, len, flush); } -static force_inline void +static void memmove_mov_avx(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { diff --git a/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c b/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c index b1775283ed9..fbfef891b14 100644 --- a/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c +++ b/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2017-2020, Intel Corporation */ +/* Copyright 2017-2023, Intel Corporation */ #include #include @@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b) flush64b(dest + 0 * 64); } -static force_inline void +static void memmove_mov_avx512f_fw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { @@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len, memmove_small_avx512f(dest, src, len, flush); } -static force_inline void +static void memmove_mov_avx512f_bw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { diff --git a/src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c b/src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c index 10c03a002aa..eff23682eb3 100644 --- a/src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c +++ b/src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c @@ -107,7 +107,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b) flush64b(dest + 0 * 64); } -static force_inline void +static void memmove_mov_sse_fw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { @@ -151,7 +151,7 @@ memmove_mov_sse_fw(char *dest, const char *src, size_t len, memmove_small_sse2(dest, src, len, flush); } -static force_inline void +static void memmove_mov_sse_bw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { diff --git a/src/stats/stack-usage-debug.txt b/src/stats/stack-usage-debug.txt index 5b857ae20a4..063f1780f8b 100644 --- a/src/stats/stack-usage-debug.txt +++ b/src/stats/stack-usage-debug.txt @@ -8,16 +8,10 @@ 51344 memmove_movnt_avx512f_clflush : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded 51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded 51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded -50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +25344 memmove_mov_avx512f_fw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +25344 memmove_mov_avx512f_fw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +25344 memmove_mov_avx512f_bw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +25344 memmove_mov_avx512f_bw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static 24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded 24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded 24144 memmove_movnt_avx_noflush_nobarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded @@ -79,16 +73,8 @@ 19024 memset_mov_avx512f_clflushopt : src/debug/libpmem/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded 19024 memset_mov_avx512f_clflushopt : src/debug/libpmem2/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded 16480 obj_vg_check_no_undef : src/debug/libpmemobj/obj.su:obj.c static -13840 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded -13840 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +13792 memmove_mov_avx : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +13792 memmove_mov_avx : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static 9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded 9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded 9808 memset_movnt_avx_noflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded @@ -109,16 +95,6 @@ 9808 memset_movnt_avx_clflushopt_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded 9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded 9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded -9200 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded -9200 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded 8464 out_error : src/debug/libpmempool/out.su:out.c dynamic,bounded 8464 out_error : src/debug/libpmem/out.su:out.c dynamic,bounded 8464 out_error : src/debug/libpmemobj/out.su:out.c dynamic,bounded @@ -165,6 +141,10 @@ 5936 memset_mov_avx_clflush : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded 5936 memset_mov_avx_clflushopt : src/debug/libpmem/memset_t_avx.su:memset_t_avx.c dynamic,bounded 5936 memset_mov_avx_clflushopt : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded +4576 memmove_mov_sse_fw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +4576 memmove_mov_sse_fw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +4576 memmove_mov_sse_bw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +4576 memmove_mov_sse_bw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static 4384 pool_params_parse : src/debug/libpmempool/pool.su:pool.c dynamic,bounded 4352 out_init : src/debug/libpmempool/out.su:out.c dynamic,bounded 4352 out_init : src/debug/libpmem/out.su:out.c dynamic,bounded @@ -952,6 +932,26 @@ 112 obj_open_common : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded 112 obj_norep_memmove : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded 112 obj_norep_memcpy : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded +112 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded +112 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded +112 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded 112 memblock_run_bitmap : src/debug/libpmemobj/memblock.su:memblock.c dynamic,bounded 112 list_remove_single : src/debug/libpmemobj/list.su:list.c dynamic,bounded 112 lane_boot : src/debug/libpmemobj/lane.su:lane.c static @@ -1672,6 +1672,16 @@ 64 obj_replica_init : src/debug/libpmemobj/obj.su:obj.c static 64 obj_pool_open : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded 64 obj_ctl_init_and_load : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded +64 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded +64 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded 64 membuf_alloc : src/debug/libpmempool/membuf.su:membuf.c static 64 membuf_alloc : src/debug/libpmemobj/membuf.su:membuf.c static 64 membuf_alloc : src/debug/libpmem2/membuf.su:membuf.c static diff --git a/src/stats/stack-usage-nondebug.txt b/src/stats/stack-usage-nondebug.txt index 7ac24525825..3843bafc5a7 100644 --- a/src/stats/stack-usage-nondebug.txt +++ b/src/stats/stack-usage-nondebug.txt @@ -10,8 +10,6 @@ 8224 pmem2_get_type_from_stat : src/nondebug/libpmemobj/pmem2_utils_linux.su:pmem2_utils_linux.c static 8224 pmem2_get_type_from_stat : src/nondebug/libpmem2/pmem2_utils_linux.su:pmem2_utils_linux.c static 8224 pmem2_get_type_from_stat : src/nondebug/common/pmem2_utils_linux.su:pmem2_utils_linux.c static -7424 memmove_mov_avx512f_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -7424 memmove_mov_avx512f_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static 4352 pool_params_parse.isra.2.constprop : src/nondebug/libpmempool/pool.su:pool.c dynamic,bounded 4272 ndctl_match_fsdax : src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c static 4272 ndctl_match_fsdax : src/nondebug/libpmempool/region_namespace_ndctl.su:region_namespace_ndctl.c static @@ -48,13 +46,9 @@ 2144 util_map_hint_unused : src/nondebug/libpmemobj/mmap_posix.su:mmap_posix.c static 2144 util_map_hint_unused : src/nondebug/libpmem/mmap_posix.su:mmap_posix.c static 2144 util_map_hint_unused : src/nondebug/common/mmap_posix.su:mmap_posix.c static -2016 memmove_mov_avx_empty : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static -2016 memmove_mov_avx_empty : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static 1184 heap_init : src/nondebug/libpmemobj/heap.su:heap.c static 1088 memset_mov_avx_empty : src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c static 1088 memset_mov_avx_empty : src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c static -1088 memmove_mov_sse2_empty : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static -1088 memmove_mov_sse2_empty : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static 720 ulog_entry_buf_create : src/nondebug/libpmemobj/ulog.su:ulog.c dynamic 640 memblock_run_init : src/nondebug/libpmemobj/memblock.su:memblock.c static 592 memset_mov_sse2_empty : src/nondebug/libpmem/memset_t_sse2.su:memset_t_sse2.c static @@ -420,6 +414,10 @@ 128 memset_mov_avx_clflush : src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c static 128 memset_mov_avx_clflushopt : src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c static 128 memset_mov_avx_clflushopt : src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c static +128 memmove_mov_avx512f_fw : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +128 memmove_mov_avx512f_fw : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +128 memmove_mov_avx512f_bw : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +128 memmove_mov_avx512f_bw : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static 128 memblock_header_legacy_reinit : src/nondebug/libpmemobj/memblock.su:memblock.c static 128 memblock_from_offset_opt : src/nondebug/libpmemobj/memblock.su:memblock.c static 128 list_insert_new_user : src/nondebug/libpmemobj/list.su:list.c dynamic,bounded @@ -497,6 +495,8 @@ 96 os_open : src/nondebug/libpmem2/os_posix.su:os_posix.c static 96 os_open : src/nondebug/core/os_posix.su:os_posix.c static 96 operation_add_buffer : src/nondebug/libpmemobj/memops.su:memops.c dynamic,bounded +96 memmove_mov_avx : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +96 memmove_mov_avx : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static 96 heap_run_into_free_chunk : src/nondebug/libpmemobj/heap.su:heap.c static 96 heap_reuse_from_recycler.constprop : src/nondebug/libpmemobj/heap.su:heap.c static 96 heap_recycle_unused : src/nondebug/libpmemobj/heap.su:heap.c static @@ -608,6 +608,10 @@ 80 obj_free : src/nondebug/libpmemobj/obj.su:obj.c dynamic,bounded 80 memset_nodrain_generic : src/nondebug/libpmem/memops_generic.su:memops_generic.c static 80 memset_nodrain_generic : src/nondebug/libpmem2/memops_generic.su:memops_generic.c static +80 memmove_mov_sse_fw : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +80 memmove_mov_sse_fw : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +80 memmove_mov_sse_bw : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +80 memmove_mov_sse_bw : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static 80 memblock_header_legacy_invalidate : src/nondebug/libpmemobj/memblock.su:memblock.c static 80 memblock_header_compact_reinit : src/nondebug/libpmemobj/memblock.su:memblock.c static 80 memblock_header_compact_invalidate : src/nondebug/libpmemobj/memblock.su:memblock.c static @@ -738,22 +742,6 @@ 64 memset_mov_sse2_clflush : src/nondebug/libpmem2/memset_t_sse2.su:memset_t_sse2.c static 64 memset_mov_sse2_clflushopt : src/nondebug/libpmem/memset_t_sse2.su:memset_t_sse2.c static 64 memset_mov_sse2_clflushopt : src/nondebug/libpmem2/memset_t_sse2.su:memset_t_sse2.c static -64 memmove_mov_avx_noflush : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_noflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clwb : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clwb : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clflush : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clflushopt : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx_clflushopt : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static -64 memmove_mov_avx512f_noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static -64 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static 64 heap_set_arena_auto : src/nondebug/libpmemobj/heap.su:heap.c static 64 heap_create_alloc_class_buckets : src/nondebug/libpmemobj/heap.su:heap.c static 64 heap_arena_new : src/nondebug/libpmemobj/heap.su:heap.c static @@ -955,14 +943,6 @@ 48 memset_nodrain_libc : src/nondebug/libpmem2/persist.su:persist.c static 48 memmove_nodrain_libc : src/nondebug/libpmem/pmem.su:pmem.c static 48 memmove_nodrain_libc : src/nondebug/libpmem2/persist.su:persist.c static -48 memmove_mov_sse2_noflush : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_noflush : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clwb : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clwb : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clflush : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clflush : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clflushopt : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static -48 memmove_mov_sse2_clflushopt : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static 48 mcsafe_op_reg_write : src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c static 48 mcsafe_op_reg_read : src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c static 48 list_remove_single : src/nondebug/libpmemobj/list.su:list.c static @@ -1528,6 +1508,16 @@ 16 obj_get_nlanes : src/nondebug/libpmemobj/obj.su:obj.c static 16 obj_fini : src/nondebug/libpmemobj/obj.su:obj.c static 16 obj_ctl_init_and_load : src/nondebug/libpmemobj/obj.su:obj.c static +16 memmove_mov_avx512f_noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static +16 memmove_mov_avx512f_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static 16 memblock_rebuild_state : src/nondebug/libpmemobj/memblock.su:memblock.c static 16 memblock_header_legacy_get_size : src/nondebug/libpmemobj/memblock.su:memblock.c static 16 memblock_header_legacy_get_flags : src/nondebug/libpmemobj/memblock.su:memblock.c static @@ -1850,6 +1840,24 @@ 8 pmem_emit_log : src/nondebug/libpmem/pmem.su:pmem.c static 8 pmem_drain : src/nondebug/libpmem/pmem.su:pmem.c static 8 pmem_deep_drain : src/nondebug/libpmem/pmem.su:pmem.c static +8 pmem_clwb : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 pmem_clwb : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 pmem_clwb : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 pmem_clwb : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 pmem_clwb : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 pmem_clwb : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 pmem_clflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 pmem_clflushopt : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static 8 pmem2_vm_reservation_get_size : src/nondebug/libpmem2/vm_reservation.su:vm_reservation.c static 8 pmem2_vm_reservation_get_address : src/nondebug/libpmem2/vm_reservation.su:vm_reservation.c static 8 pmem2_region_get_next_badblock : src/nondebug/libpmempool/badblocks_ndctl.su:badblocks_ndctl.c static @@ -2252,8 +2260,40 @@ 8 obj_norep_memcpy : src/nondebug/libpmemobj/obj.su:obj.c static 8 obj_norep_drain : src/nondebug/libpmemobj/obj.su:obj.c static 8 obj_drain_empty : src/nondebug/libpmemobj/obj.su:obj.c static +8 noflush : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_memset.h static +8 noflush : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_memset.h static +8 noflush : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h static +8 noflush : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_memset.h static +8 noflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_memset.h static +8 noflush : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_memset.h static +8 noflush64b : src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h static 8 memory_barrier : src/nondebug/libpmem/init.su:init.c static 8 memory_barrier : src/nondebug/libpmem2/init.su:init.c static +8 memmove_mov_sse2_noflush : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_noflush : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_empty : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_empty : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clwb : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clwb : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clflush : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clflush : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clflushopt : src/nondebug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_sse2_clflushopt : src/nondebug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static +8 memmove_mov_avx_noflush : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_noflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_empty : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_empty : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clwb : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clwb : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clflush : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clflush : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clflushopt : src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static +8 memmove_mov_avx_clflushopt : src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static 8 membuf_ptr_user_data : src/nondebug/libpmempool/membuf.su:membuf.c static 8 membuf_ptr_user_data : src/nondebug/libpmemobj/membuf.su:membuf.c static 8 membuf_ptr_user_data : src/nondebug/libpmem2/membuf.su:membuf.c static @@ -2302,12 +2342,42 @@ 8 hash64 : src/nondebug/libpmemobj/rand.su:rand.c static 8 hash64 : src/nondebug/common/rand.su:rand.c static 8 flush_empty : src/nondebug/libpmem/pmem.su:pmem.c static +8 flush_empty_nolog : src/nondebug/libpmem/memcpy_t_sse2.su:pmem2_arch.h static +8 flush_empty_nolog : src/nondebug/libpmem/memcpy_t_avx.su:pmem2_arch.h static +8 flush_empty_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:pmem2_arch.h static +8 flush_empty_nolog : src/nondebug/libpmem2/memcpy_t_sse2.su:pmem2_arch.h static +8 flush_empty_nolog : src/nondebug/libpmem2/memcpy_t_avx.su:pmem2_arch.h static +8 flush_empty_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:pmem2_arch.h static 8 flush_clwb : src/nondebug/libpmem/init.su:init.c static 8 flush_clwb : src/nondebug/libpmem2/init.su:init.c static +8 flush_clwb_nolog : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 flush_clwb_nolog : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 flush_clwb_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 flush_clwb_nolog : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 flush_clwb_nolog : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 flush_clwb_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static 8 flush_clflush : src/nondebug/libpmem/init.su:init.c static 8 flush_clflush : src/nondebug/libpmem2/init.su:init.c static 8 flush_clflushopt : src/nondebug/libpmem/init.su:init.c static 8 flush_clflushopt : src/nondebug/libpmem2/init.su:init.c static +8 flush_clflushopt_nolog : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 flush_clflushopt_nolog : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 flush_clflushopt_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 flush_clflushopt_nolog : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 flush_clflushopt_nolog : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 flush_clflushopt_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 flush_clflush_nolog : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem/memcpy_t_sse2.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem/memcpy_t_avx.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem2/memcpy_t_sse2.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem2/memcpy_t_avx.su:flush.h static +8 flush64b_empty : src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h static 8 fence_empty : src/nondebug/libpmem/pmem.su:pmem.c static 8 debug_ctl_register : src/nondebug/libpmemobj/ctl_debug.su:ctl_debug.c static 8 ctl__worker_read : src/nondebug/libpmemobj/tx.su:tx.c static