From 413fc25bfab91a3e3138aa8475dd91a2d67efef6 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 28 Feb 2024 23:12:59 -0800 Subject: [PATCH 01/47] changes to add unit test 1. Add unit test infrastructure with Catch 2. Fix a bug in ctx manager. 3. Added Test case for testing data transfer capability. 3. Added Test case for testing mdm capability. 4. Exported some functions for unit testing. --- CMakeLists.txt | 12 +- src/dyad/core/dyad_core.c | 14 +- src/dyad/core/dyad_core.h | 26 +++- src/dyad/modules/dyad.c | 9 +- tests/CMakeLists.txt | 1 + tests/unit/CMakeLists.txt | 35 +++++ tests/unit/catch_config.h | 28 ++++ tests/unit/data_plane/CMakeLists.txt | 11 ++ tests/unit/data_plane/data_plane.cpp | 204 +++++++++++++++++++++++++++ tests/unit/mdm/CMakeLists.txt | 29 ++++ tests/unit/mdm/mdm.cpp | 152 ++++++++++++++++++++ tests/unit/script/CMakeLists.txt | 9 ++ tests/unit/script/dyad_start.sh | 2 + tests/unit/script/dyad_stop.sh | 2 + tests/unit/test_utils.h | 66 +++++++++ tests/unit/unit_test.cpp | 105 ++++++++++++++ 16 files changed, 692 insertions(+), 13 deletions(-) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/unit/CMakeLists.txt create mode 100644 tests/unit/catch_config.h create mode 100644 tests/unit/data_plane/CMakeLists.txt create mode 100644 tests/unit/data_plane/data_plane.cpp create mode 100644 tests/unit/mdm/CMakeLists.txt create mode 100644 tests/unit/mdm/mdm.cpp create mode 100644 tests/unit/script/CMakeLists.txt create mode 100755 tests/unit/script/dyad_start.sh create mode 100755 tests/unit/script/dyad_stop.sh create mode 100644 tests/unit/test_utils.h create mode 100644 tests/unit/unit_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c3610a3b..be715ab4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,12 +13,12 @@ set(DYAD_PACKAGE_VERSION_MAJOR "${DYAD_VERSION_MAJOR}.${DYAD_VERSION_MINOR}") set(DYAD_PACKAGE_VERSION_MINOR "${DYAD_VERSION_PATCH}") set(DYAD_PACKAGE_STRING "${DYAD_PACKAGE_NAME} ${DYAD_PACKAGE_VERSION}") set(DYAD_PACKAGE_TARNAME "${DYAD_PACKAGE}") - project(dyad LANGUAGES C CXX) # Convenience defines string(TOUPPER "${PROJECT_NAME}" UPPER_PROJECT_NAME) string(TOLOWER "${PROJECT_NAME}" LOWER_PROJECT_NAME) +set(DYAD_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) #------------------------------------------------------------------------------ # Internal Includes for header and libraries @@ -110,7 +110,7 @@ set(DYAD_LOGGER "NONE" CACHE STRING "Logger to use for DYAD") set_property(CACHE DYAD_LOGGER PROPERTY STRINGS FLUX CPP_LOGGER NONE) set(DYAD_LOGGER_LEVEL "NONE" CACHE STRING "Logging level to use for DYAD") set_property(CACHE DYAD_LOGGER_LEVEL PROPERTY STRINGS DEBUG INFO WARN ERROR NONE) - +option(DYAD_ENABLE_TESTS "Enable dyad tests" OFF) #------------------------------------------------------------------------------ # Compiler setup @@ -459,6 +459,8 @@ string(APPEND _str " DYAD_ENABLE_UCX_DATA: ${DYAD_ENABLE_UCX_DATA}\n") string(APPEND _str " DYAD_ENABLE_UCX_DATA_RMA: ${DYAD_ENABLE_UCX_DATA_RMA}\n") +string(APPEND _str + " DYAD_ENABLE_TESTS: ${DYAD_ENABLE_TESTS}\n") string(APPEND _str " DYAD_PROFILER: ${DYAD_PROFILER}\n") string(APPEND _str @@ -513,3 +515,9 @@ install(FILES "${CMAKE_BINARY_DIR}/dyad_module.lua.install" RENAME "${DYAD_MODULEFILE_NAME}" DESTINATION "${DYAD_INSTALL_SYSCONFDIR}") + + +if (DYAD_ENABLE_TESTS) + enable_testing() + add_subdirectory(tests) +endif () \ No newline at end of file diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c index a3e97395..f99461f2 100644 --- a/src/dyad/core/dyad_core.c +++ b/src/dyad/core/dyad_core.c @@ -27,14 +27,10 @@ #include #endif -#if DYAD_PERFFLOW -#define DYAD_CORE_FUNC_MODS __attribute__ ((annotate ("@critical_path()"))) static -#else -#define DYAD_CORE_FUNC_MODS static inline -#endif -static int gen_path_key (const char* restrict str, + +DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, char* restrict path_key, const size_t len, const uint32_t depth, @@ -176,7 +172,7 @@ publish_done:; return rc; } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, const char* restrict fname) +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* restrict ctx, const char* restrict fname) { DYAD_C_FUNCTION_START(); DYAD_C_FUNCTION_UPDATE_STR ("fname", ctx->fname); @@ -236,7 +232,7 @@ static void print_mdata (const dyad_ctx_t* restrict ctx, } } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_kvs_read (const dyad_ctx_t* restrict ctx, +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* restrict ctx, const char* restrict topic, const char* restrict upath, bool should_wait, @@ -374,7 +370,7 @@ fetch_done:; return rc; } -DYAD_CORE_FUNC_MODS dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* restrict ctx, const dyad_metadata_t* restrict mdata, char** restrict file_data, size_t* restrict file_len) diff --git a/src/dyad/core/dyad_core.h b/src/dyad/core/dyad_core.h index 4608b517..b58577ca 100644 --- a/src/dyad/core/dyad_core.h +++ b/src/dyad/core/dyad_core.h @@ -25,7 +25,11 @@ #ifdef __cplusplus extern "C" { #endif - +#if DYAD_PERFFLOW +#define DYAD_CORE_FUNC_MODS __attribute__ ((annotate ("@critical_path()"))) static +#else +#define DYAD_CORE_FUNC_MODS static inline +#endif DYAD_DLL_EXPORTED extern const struct dyad_ctx dyad_ctx_default; struct dyad_metadata { @@ -113,6 +117,26 @@ DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume (dyad_ctx_t* ctx, con DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* ctx, const char* fname, const dyad_metadata_t* mdata); + +/** + * Private Function definitions + */ +DYAD_DLL_EXPORTED dyad_rc_t dyad_get_data (const dyad_ctx_t* ctx, const dyad_metadata_t* mdata, + char** file_data, + size_t* file_len); +DYAD_DLL_EXPORTED dyad_rc_t dyad_commit (dyad_ctx_t* ctx, const char* fname); + +DYAD_DLL_EXPORTED int gen_path_key (const char* str, char* path_key, + const size_t len, + const uint32_t depth, + const uint32_t width); + +DYAD_DLL_EXPORTED dyad_rc_t dyad_kvs_read (const dyad_ctx_t* ctx, + const char* topic, + const char* upath, + bool should_wait, + dyad_metadata_t** mdata); + #if DYAD_SYNC_DIR DYAD_PFA_ANNOTATE DYAD_DLL_EXPORTED int dyad_sync_directory (dyad_ctx_t* ctx, const char* path); #endif diff --git a/src/dyad/modules/dyad.c b/src/dyad/modules/dyad.c index 1b032405..22d2db1c 100644 --- a/src/dyad/modules/dyad.c +++ b/src/dyad/modules/dyad.c @@ -355,6 +355,8 @@ static int opt_parse (opt_parse_out_t* restrict opt, const unsigned broker_rank, // mode as the option, then skip reinitializing DYAD_LOG_STDERR ("DYAD_MOD: DTL 'mode' option -m with value `%s'\n", optarg); opt->dtl_mode = optarg; + if (strcmp("UCX", optarg) == 0) *dtl_mode = DYAD_DTL_UCX; + else if (strcmp("FLUX_RPC", optarg) == 0) *dtl_mode = DYAD_DTL_FLUX_RPC; break; case 'i': #ifndef DYAD_LOGGER_NO_LOG @@ -420,7 +422,12 @@ dyad_rc_t dyad_module_ctx_init (const opt_parse_out_t* opt, flux_t* h) DYAD_LOG_STDERR ("DYAD_MOD: Did not find DTL 'mode' option. " \ "Using env %s=%s", DYAD_DTL_MODE_ENV, getenv (DYAD_DTL_MODE_ENV)); } - + char* kvs = getenv("DYAD_KVS_NAMESPACE"); + if (kvs != NULL) { + DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is set to `%s'\n", kvs); + } else { + DYAD_LOG_STDERR ("DYAD_MOD: DYAD_KVS_NAMESPACE is not set\n"); + } dyad_ctx_init (DYAD_COMM_SEND, h); mod_ctx->ctx = dyad_ctx_get (); dyad_ctx_t* ctx = mod_ctx->ctx; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 00000000..059f2a25 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(unit) \ No newline at end of file diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt new file mode 100644 index 00000000..5d7692e6 --- /dev/null +++ b/tests/unit/CMakeLists.txt @@ -0,0 +1,35 @@ +if(NOT DEFINED ENV{DYAD_TEST_MACHINE}) + message(FATAL_ERROR "-- [dyad] DYAD_TEST_MACHINE in env should be set for ${PROJECT_NAME} test build") +else() + message(STATUS "[dyad] found setting machine to $ENV{DYAD_TEST_MACHINE}") +endif() +if(NOT DEFINED ENV{DYAD_PFS_DIR}) + message(FATAL_ERROR "-- [dyad] DYAD_PFS_DIR in env should be set for ${PROJECT_NAME} paper test build") +else() + message(STATUS "[dyad] found setting pfs dir to $ENV{DYAD_PFS_DIR}") +endif() +if(NOT DEFINED ENV{DYAD_DMD_DIR}) + message(FATAL_ERROR "-- [dyad] DYAD_DMD_DIR in env should be set for ${PROJECT_NAME} paper test build") +else() + message(STATUS "[dyad] found setting DMD Dir to $ENV{DYAD_DMD_DIR}") +endif() +set(DYAD_KEYSPACE test_dyad) +set(DYAD_LOG_DIR ${CMAKE_BINARY_DIR}/logs) +file(MAKE_DIRECTORY ${DYAD_LOG_DIR}) +find_package(Catch2 REQUIRED) +find_package(MPI REQUIRED COMPONENTS CXX) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(Catch2::Catch2) +include_directories(${MPI_CXX_INCLUDE_DIRS}) +include_directories(${DYAD_PROJECT_DIR}/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +include_directories(${CMAKE_BINARY_DIR}/include) +set(TEST_LIBS Catch2::Catch2 -lstdc++fs ${MPI_CXX_LIBRARIES} -rdynamic dyad_core dyad_ctx dyad_utils flux-core ${CPP_LOGGER_LIBRARIES}) +set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.h) +add_executable(unit_test unit_test.cpp ${TEST_SRC} ) +target_link_libraries(unit_test ${TEST_LIBS}) +add_dependencies(unit_test dyad) + +add_subdirectory(script) +add_subdirectory(data_plane) +add_subdirectory(mdm) \ No newline at end of file diff --git a/tests/unit/catch_config.h b/tests/unit/catch_config.h new file mode 100644 index 00000000..58e771ce --- /dev/null +++ b/tests/unit/catch_config.h @@ -0,0 +1,28 @@ +// +// Created by haridev on 2/28/24. +// + +#ifndef DYAD_CATCH_CONFIG_H +#define DYAD_CATCH_CONFIG_H +#include +namespace cl = Catch::Clara; + +cl::Parser define_options(); + +int init(int* argc, char*** argv); +int finalize(); + +int main(int argc, char* argv[]) { + Catch::Session session; + auto cli = session.cli() | define_options(); + session.cli(cli); + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) return returnCode; + returnCode = init(&argc, &argv); + if (returnCode != 0) return returnCode; + int test_return_code = session.run(); + returnCode = finalize(); + if (returnCode != 0) return returnCode; + exit(test_return_code); +} +#endif // DYAD_CATCH_CONFIG_H diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt new file mode 100644 index 00000000..3819b23d --- /dev/null +++ b/tests/unit/data_plane/CMakeLists.txt @@ -0,0 +1,11 @@ +set(node 1) +set(ppn 1) +set(files 1) +set(test_name unit_remote_data_${node}_${ppn}) +add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteDataBandwidth) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file diff --git a/tests/unit/data_plane/data_plane.cpp b/tests/unit/data_plane/data_plane.cpp new file mode 100644 index 00000000..a4609c48 --- /dev/null +++ b/tests/unit/data_plane/data_plane.cpp @@ -0,0 +1,204 @@ +#include +#include +#include +#include + +int create_files_per_broker() { + char filename[4096], first_file[4096]; + bool is_first = true; + size_t file_size = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + bool first_rank_per_node = info.rank % args.process_per_node == 0; + if (first_rank_per_node) { + fs::create_directories (args.dyad_managed_dir); + for (size_t broker_idx = 0; broker_idx < args.brokers_per_node; ++broker_idx) { + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + size_t global_broker_idx = node_idx * args.brokers_per_node + broker_idx; + if (is_first) { + sprintf (first_file, "%s/%s_%zu_%zu.bat", + args.dyad_managed_dir.c_str (), args.filename.c_str (), + global_broker_idx, file_idx); + std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + + std::to_string (file_size) + "; } > " + first_file + " "; + int status = system (cmd.c_str ()); + (void) status; + is_first = false; + } else { + sprintf (filename, "%s/%s_%zu_%zu.bat", + args.dyad_managed_dir.c_str (), args.filename.c_str (), + global_broker_idx, file_idx); + std::string cmd = "cp " + std::string (first_file) + " " + filename; + int status = system (cmd.c_str ()); + (void) status; + } + } + } + } + MPI_Barrier (MPI_COMM_WORLD); + return 0; +} + +TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + REQUIRE (create_files_per_broker() == 0); + dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + auto ctx = dyad_ctx_get(); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; + dyad_metadata_t mdata; + mdata.owner_rank = neighour_broker_idx; + size_t data_len = args.request_size*args.iteration; + char* file_data = NULL; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (filename, "%s_%u_%zu.bat", args.filename.c_str (), + neighour_broker_idx, file_idx); + mdata.fpath = filename; + data_time.resumeTime(); + auto rc = dyad_get_data (ctx, &mdata, &file_data, &data_len); + data_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + auto rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + REQUIRE (create_files_per_broker() == 0); + dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + auto ctx = dyad_ctx_get(); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096], upath[4096]; + uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; + dyad_metadata_t mdata; + mdata.owner_rank = neighour_broker_idx; + size_t data_len = args.request_size*args.iteration; + if (info.rank % args.process_per_node != 0) + usleep (10000); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (upath, "%s_%u_%zu.bat", args.filename.c_str (), + neighour_broker_idx, file_idx); + sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), + neighour_broker_idx, file_idx); + mdata.fpath = upath; + data_time.resumeTime(); + auto rc = dyad_consume_w_metadata(ctx, filename, &mdata); + data_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + auto rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + REQUIRE (create_files_per_broker() == 0); + dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), + info.broker_idx, file_idx); + data_time.resumeTime(); + int fd = open(filename, O_RDONLY); + data_time.pauseTime(); + REQUIRE (fd != -1); + data_time.resumeTime(); + int bytes = read (fd,file_data, data_len); + data_time.pauseTime(); + REQUIRE (bytes == data_len); + data_time.resumeTime(); + int status = close(fd); + data_time.pauseTime(); + REQUIRE (status == 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + auto rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + REQUIRE (create_files_per_broker() == 0); + dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), + info.broker_idx, file_idx); + data_time.resumeTime(); + int fd = open(filename, O_RDONLY); + data_time.pauseTime(); + REQUIRE (fd != -1); + data_time.resumeTime(); + int bytes = read (fd,file_data, data_len); + data_time.pauseTime(); + REQUIRE (bytes == data_len); + data_time.resumeTime(); + int status = close(fd); + data_time.pauseTime(); + REQUIRE (status == 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + auto rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} \ No newline at end of file diff --git a/tests/unit/mdm/CMakeLists.txt b/tests/unit/mdm/CMakeLists.txt new file mode 100644 index 00000000..3717411e --- /dev/null +++ b/tests/unit/mdm/CMakeLists.txt @@ -0,0 +1,29 @@ +set(node 1) +set(ppn 1) +set(files 1) +set(test_name unit_localfs_${node}_${ppn}) +set(mpiexec flux run -N ${node} --tasks-per-node ${ppn}) +set(mpiexec ) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalFSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +set(test_name unit_localkvs_${node}_${ppn}) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalKVSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +set(test_name unit_remotekvs_${node}_${ppn}) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteKVSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file diff --git a/tests/unit/mdm/mdm.cpp b/tests/unit/mdm/mdm.cpp new file mode 100644 index 00000000..bad7ea6c --- /dev/null +++ b/tests/unit/mdm/mdm.cpp @@ -0,0 +1,152 @@ + +#include +#include + +#include +TEST_CASE("LocalFSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + struct flock exclusive_lock; + SECTION("Throughput") { + char filename[4096]; + Timer kvs_time; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (filename, + "%s/%s_%u_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + file_idx); + kvs_time.resumeTime(); + int lock_fd = open (filename, O_RDWR | O_CREAT, 0666); + kvs_time.pauseTime(); + REQUIRE (lock_fd != -1); + + kvs_time.resumeTime(); + rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + + kvs_time.resumeTime(); + auto file_size = get_file_size (lock_fd); + kvs_time.pauseTime(); + (void)file_size; + kvs_time.resumeTime(); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + int status = close (lock_fd); + kvs_time.pauseTime(); + REQUIRE (status == 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (my_filename, + "%s/%s_%u_%d_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + info.rank, + file_idx); + sprintf (lookup_filename, + "%s_%u_%d_%zu.bat", + args.filename.c_str (), + info.broker_idx, + info.rank, + file_idx); + rc = dyad_commit (ctx, my_filename); + REQUIRE (rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX+1] = {'\0'}; + gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("RemoteKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (my_filename, + "%s/%s_%u_%d_%d_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + info.rank, + info.comm_size, + file_idx); + sprintf (lookup_filename, + "%s_%u_%d_%d_%zu.bat", + args.filename.c_str (), + info.broker_idx, + info.rank, + info.comm_size, + file_idx); + rc = dyad_commit (ctx, my_filename); + REQUIRE (rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX+1] = {'\0'}; + gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (posttest() == 0); +} \ No newline at end of file diff --git a/tests/unit/script/CMakeLists.txt b/tests/unit/script/CMakeLists.txt new file mode 100644 index 00000000..2c7585a7 --- /dev/null +++ b/tests/unit/script/CMakeLists.txt @@ -0,0 +1,9 @@ +add_test(dyad_start ${CMAKE_CURRENT_SOURCE_DIR}/dyad_start.sh) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_PATH=$ENV{DYAD_DMD_DIR}) + +add_test(dyad_stop ${CMAKE_CURRENT_SOURCE_DIR}/dyad_stop.sh) +set_property(TEST dyad_start APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) \ No newline at end of file diff --git a/tests/unit/script/dyad_start.sh b/tests/unit/script/dyad_start.sh new file mode 100755 index 00000000..31d82267 --- /dev/null +++ b/tests/unit/script/dyad_start.sh @@ -0,0 +1,2 @@ +flux kvs namespace create ${DYAD_KVS_NAMESPACE} +flux exec -r all flux module load ${DYAD_MODULE_SO} --info_log=${DYAD_LOG_DIR}/dyad-broker --error_log=${DYAD_LOG_DIR}/dyad-broker --mode=${DYAD_DTL_MODE} $DYAD_PATH \ No newline at end of file diff --git a/tests/unit/script/dyad_stop.sh b/tests/unit/script/dyad_stop.sh new file mode 100755 index 00000000..d564b1e2 --- /dev/null +++ b/tests/unit/script/dyad_stop.sh @@ -0,0 +1,2 @@ +flux kvs namespace remove ${DYAD_KVS_NAMESPACE} +flux exec -r all flux module unload dyad \ No newline at end of file diff --git a/tests/unit/test_utils.h b/tests/unit/test_utils.h new file mode 100644 index 00000000..11a62970 --- /dev/null +++ b/tests/unit/test_utils.h @@ -0,0 +1,66 @@ +#ifndef DYAD_TEST_UTILS_H +#define DYAD_TEST_UTILS_H + +#include +#include +#include +#include + +const uint32_t KB = 1024; +const uint32_t MB = 1024 * 1024; +#define AGGREGATE_TIME(name) \ + double total_##name = 0.0; \ + auto name##_a = name##_time.getElapsedTime(); \ + MPI_Reduce(&name##_a, &total_##name, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + +size_t GetRandomOffset(size_t i, unsigned int offset_seed, size_t stride, + size_t total_size) { + return abs((int)(((i * rand_r(&offset_seed)) % stride) % total_size)); +} +inline std::string get_filename(int fd) { + const int kMaxSize = 256; + char proclnk[kMaxSize]; + char filename[kMaxSize]; + snprintf(proclnk, kMaxSize, "/proc/self/fd/%d", fd); + size_t r = readlink(proclnk, filename, kMaxSize); + filename[r] = '\0'; + return filename; +} + +std::string GenRandom(const int len) { + std::string tmp_s; + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + srand(100); + + tmp_s.reserve(len); + + for (int i = 0; i < len; ++i) { + tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)]; + } + + tmp_s[len - 1] = '\n'; + + return tmp_s; +} + +class Timer { + public: + Timer() : elapsed_time(0) {} + void resumeTime() { t1 = std::chrono::high_resolution_clock::now(); } + double pauseTime() { + auto t2 = std::chrono::high_resolution_clock::now(); + elapsed_time += std::chrono::duration(t2 - t1).count(); + return elapsed_time; + } + double getElapsedTime() { return elapsed_time; } + + private: + std::chrono::high_resolution_clock::time_point t1; + double elapsed_time; +}; + +#endif // DYAD_TEST_UTILS_H diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp new file mode 100644 index 00000000..c9ba53b9 --- /dev/null +++ b/tests/unit/unit_test.cpp @@ -0,0 +1,105 @@ +#include +#include +#include +#include + +#include +namespace fs = std::experimental::filesystem; + +/** + * Test data structures + */ +namespace dyad::test { +struct Info { + int rank; + int comm_size; + int num_nodes; + int num_brokers; + flux_t* flux_handle; + uint32_t broker_idx; + uint32_t broker_size; +}; +struct Arguments { + // MPI Configurations + size_t process_per_node = 1; + size_t brokers_per_node = 1; + // DYAD Configuration + fs::path dyad_managed_dir = "~/dyad/dmd"; + // Test configuration + fs::path pfs = "~/dyad/pfs"; + std::string filename = "test.dat"; + size_t number_of_files = 1; + size_t request_size = 65536; + size_t iteration = 8; + bool debug = false; +}; +} // namespace dyad::test + +dyad::test::Arguments args; +dyad::test::Info info; +/** + * Overridden methods for catch + */ + +int init(int* argc, char*** argv) { + // fprintf(stdout, "Initializing MPI\n"); + MPI_Init(argc, argv); + MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); + MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); + info.flux_handle = flux_open (NULL, 0); + if (args.debug && info.rank == 0) { + printf("%d ready for attach\n", info.comm_size); + fflush(stdout); + getchar(); + } + MPI_Barrier(MPI_COMM_WORLD); + return 0; +} +int finalize() { + MPI_Finalize(); + return 0; +} +cl::Parser define_options() { + return cl::Opt(args.filename, "filename")["-f"]["--filename"]( + "Filename to be use for I/O.") | + cl::Opt(args.pfs, "pfs")["-d"]["--pfs"]( + "Directory used for performing I/O (default pfs)") | + cl::Opt(args.dyad_managed_dir, "dmd")["-d"]["--dmd"]( + "Directory used for DYAD Managed Directory") | + cl::Opt(args.process_per_node, + "process_per_node")["-p"]["--ppn"]("Processes per node") | + cl::Opt(args.request_size, "request_size")["-r"]["--request_size"]( + "Transfer size used for performing I/O") | + cl::Opt(args.iteration, + "iteration")["-i"]["--iteration"]("Number of Iterations") | + cl::Opt(args.number_of_files, + "number_of_files")["-n"]["--number_of_files"]("Number of Files") | + cl::Opt(args.number_of_files, + "brokers_per_node")["-b"]["--brokers_per_node"]("Number of Brokers per node") | + cl::Opt(args.debug, + "debug")["-d"]["--debug"]("debug"); +} + +int pretest() { + info.num_nodes = info.comm_size / args.process_per_node; + info.num_brokers = info.num_nodes * args.brokers_per_node; + flux_get_rank (info.flux_handle, &info.broker_idx); + flux_get_size (info.flux_handle, &info.broker_size); + return 0; +} +int posttest() { + return 0; +} +int clean_directories() { + auto file_pt = args.pfs.string() + args.filename; + std::string cmd = "rm -rf " + file_pt + "*"; + int status = system (cmd.c_str ()); + (void) status; + file_pt = args.dyad_managed_dir.string() + args.filename; + cmd = "rm -rf " + file_pt + "*"; + status = system (cmd.c_str ()); + (void) status; + return 0; +} +#include "data_plane/data_plane.cpp" +#include "mdm/mdm.cpp" \ No newline at end of file From caf44f83e2274e628e9b49ab9147738cd39dca97 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 4 Mar 2024 09:13:29 -0800 Subject: [PATCH 02/47] changes to support DLIO with DYAD 1. Fix Enum for DTLMode 2. Split I/O and locking for consume_w_metadata 3. Comment the STDERR 4. Configurations for DLIO with DYAD --- pydyad/pydyad/bindings.py | 11 +++-- src/dyad/core/dyad_core.c | 37 +++++++++----- src/dyad/core/dyad_ctx.c | 2 +- .../configs/workload/dyad_resnet50.yaml | 27 +++++++++++ .../configs/workload/dyad_unet3d.yaml | 8 ++-- .../configs/workload/dyad_unet3d_large.yaml | 37 ++++++++++++++ .../configs/workload/dyad_unet3d_small.yaml | 36 +++++++------- .../configs/workload/resnet50_base.yaml | 28 +++++++++++ .../configs/workload/unet3d_base.yaml | 4 +- .../configs/workload/unet3d_base_large.yaml | 37 ++++++++++++++ tests/integration/dlio_benchmark/corona.sh | 2 +- .../dlio_benchmark/dyad_torch_data_loader.py | 21 +++++--- tests/integration/dlio_benchmark/run_dlio.sh | 16 +++---- tests/integration/dlio_benchmark/setup-env.sh | 16 ++++--- .../dlio_benchmark/torch_data_loader.py | 48 +++++++++++++------ 15 files changed, 249 insertions(+), 81 deletions(-) create mode 100644 tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_large.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/unet3d_base_large.yaml diff --git a/pydyad/pydyad/bindings.py b/pydyad/pydyad/bindings.py index 447a8a2d..2b1ab3ad 100644 --- a/pydyad/pydyad/bindings.py +++ b/pydyad/pydyad/bindings.py @@ -82,9 +82,12 @@ def __del__(self): self.dyad_bindings_obj = None -class DTLMode(enum.IntEnum): - DYAD_DTL_UCX = 0 - DYAD_DTL_FLUX_RPC = 1 +class DTLMode(enum.Enum): + DYAD_DTL_UCX = "UCX" + DYAD_DTL_FLUX_RPC = "FLUX_RPC" + + def __str__(self): + return self.value class DTLCommMode(enum.IntEnum): DYAD_COMM_NONE = 0 @@ -252,7 +255,7 @@ def init( prod_managed_path.encode() if prod_managed_path is not None else None, cons_managed_path.encode() if cons_managed_path is not None else None, ctypes.c_bool(relative_to_managed_path), - dtl_mode.encode() if dtl_mode is not None else None, + str(dtl_mode).encode() if dtl_mode is not None else None, ctypes.c_int(dtl_comm_mode), ctypes.c_void_p(flux_handle) ) diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c index f99461f2..5f7f3893 100644 --- a/src/dyad/core/dyad_core.c +++ b/src/dyad/core/dyad_core.c @@ -814,7 +814,7 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, DYAD_C_FUNCTION_START(); DYAD_C_FUNCTION_UPDATE_STR ("fname", fname); dyad_rc_t rc = DYAD_RC_OK; - int fd = -1; + int lock_fd = -1, io_fd = -1; ssize_t file_size = -1; char* file_data = NULL; size_t data_len = 0ul; @@ -840,19 +840,19 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, } // Set reenter to false to avoid recursively performing DYAD operations ctx->reenter = false; - fd = open (fname, O_RDWR | O_CREAT, 0666); + lock_fd = open (fname, O_RDWR | O_CREAT, 0666); DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); - if (fd == -1) { + if (lock_fd == -1) { DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); rc = DYAD_RC_BADFIO; goto consume_close; } - rc = dyad_excl_flock (ctx, fd, &exclusive_lock); + rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); if (DYAD_IS_ERROR (rc)) { - dyad_release_flock (ctx, fd, &exclusive_lock); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); goto consume_close; } - if ((file_size = get_file_size (fd)) <= 0) { + if ((file_size = get_file_size (lock_fd)) <= 0) { DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", \ ctx->node_idx, ctx->rank, ctx->pid, fname, fd); @@ -861,27 +861,38 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, rc = dyad_get_data (ctx, mdata, &file_data, &data_len); if (DYAD_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, "dyad_get_data failed!\n"); - dyad_release_flock (ctx, fd, &exclusive_lock); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); goto consume_done; } DYAD_C_FUNCTION_UPDATE_INT ("data_len", data_len); - + io_fd = open (fname, O_WRONLY); + DYAD_C_FUNCTION_UPDATE_INT ("io_fd", io_fd); + if (io_fd == -1) { + DYAD_LOG_ERROR (ctx, "Cannot open file (%s) in write mode for dyad_consume!\n", fname); + rc = DYAD_RC_BADFIO; + goto consume_close; + } // Call dyad_pull to fetch the data from the producer's // Flux broker - rc = dyad_cons_store (ctx, mdata, fd, data_len, file_data); + rc = dyad_cons_store (ctx, mdata, io_fd, data_len, file_data); + + if (close (io_fd) != 0) { + rc = DYAD_RC_BADFIO; + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + goto consume_done; + } // If an error occured in dyad_pull, log it // and return the corresponding DYAD return code if (DYAD_IS_ERROR (rc)) { DYAD_LOG_ERROR (ctx, "dyad_cons_store failed!\n"); - dyad_release_flock (ctx, fd, &exclusive_lock); + dyad_release_flock (ctx, io_fd, &exclusive_lock); goto consume_done; }; - fsync (fd); } - dyad_release_flock (ctx, fd, &exclusive_lock); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); DYAD_C_FUNCTION_UPDATE_INT ("file_size", file_size); - if (close (fd) != 0) { + if (close (lock_fd) != 0) { rc = DYAD_RC_BADFIO; goto consume_done; } diff --git a/src/dyad/core/dyad_ctx.c b/src/dyad/core/dyad_ctx.c index 35a2f945..c2d433e3 100644 --- a/src/dyad/core/dyad_ctx.c +++ b/src/dyad/core/dyad_ctx.c @@ -801,7 +801,7 @@ DYAD_DLL_EXPORTED dyad_rc_t dyad_finalize () { DYAD_C_FUNCTION_START(); dyad_rc_t rc = DYAD_RC_OK; - DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); + //DYAD_LOG_STDERR ("DYAD context is being destroyed!\n"); if (ctx == NULL) { rc = DYAD_RC_OK; goto finalize_region_finish; diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml new file mode 100644 index 00000000..2b79edf3 --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml @@ -0,0 +1,27 @@ +model: unet3d + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + num_files_train: 10240000 + num_samples_per_file: 1 + record_length: 150528 + data_folder: data/resnet50 + format: png + +train: + computation_time: 0.317 # this is for A100 + epochs: 100 + +reader: + data_loader: pytorch + read_threads: 6 + computation_threads: 8 + batch_size: 1 + multiprocessing_context: spawn + data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader + data_loader_sampler: index \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d.yaml index 1edf7819..ae23db4b 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d.yaml @@ -5,12 +5,12 @@ framework: pytorch workflow: generate_data: False train: True - checkpoint: True + checkpoint: False dataset: data_folder: data/unet3d/ format: npz - num_files_train: 168 + num_files_train: 320 num_samples_per_file: 1 record_length: 146600628 record_length_stdev: 68341808 @@ -19,7 +19,7 @@ dataset: reader: data_loader: pytorch batch_size: 4 - read_threads: 3 + read_threads: 6 file_shuffle: seed sample_shuffle: seed multiprocessing_context: spawn @@ -28,7 +28,7 @@ reader: train: epochs: 10 - computation_time: 1.3604 + computation_time: 0 checkpoint: checkpoint_folder: checkpoints/unet3d diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_large.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_large.yaml new file mode 100644 index 00000000..21cca045 --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_large.yaml @@ -0,0 +1,37 @@ +model: unet3d + +framework: pytorch + +workflow: + generate_data: False + train: True + checkpoint: False + +dataset: + data_folder: data/unet3d/ + format: npz + num_files_train: 10240 + num_samples_per_file: 1 + record_length: 146600628 + record_length_stdev: 68341808 + record_length_resize: 2097152 + +reader: + data_loader: pytorch + batch_size: 4 + read_threads: 1 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader + data_loader_sampler: index + +train: + epochs: 20 + computation_time: 0.188 + +checkpoint: + checkpoint_folder: checkpoints/unet3d + checkpoint_after_epoch: 5 + epochs_between_checkpoints: 2 + model_size: 499153191 diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_small.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_small.yaml index 3818c2eb..752af3c1 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_small.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_unet3d_small.yaml @@ -5,35 +5,33 @@ framework: pytorch workflow: generate_data: False train: True - evaluation: True + checkpoint: False dataset: - data_folder: data/dyad_unet3d + data_folder: data/unet3d/ format: npz - num_files_train: 16 - num_files_eval: 1 + num_files_train: 168 num_samples_per_file: 1 - record_length: 4096 - file_shuffle: SEED + record_length: 146600628 + record_length_stdev: 68341808 + record_length_resize: 2097152 reader: data_loader: pytorch batch_size: 1 - batch_size_eval: 1 + read_threads: 1 + file_shuffle: seed + sample_shuffle: seed multiprocessing_context: spawn data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader data_loader_sampler: index train: - epochs: 2 - computation_time: 1.00 - seed: 100 - seed_change_epoch: True - - -evaluation: - eval_time: 0.5 - epochs_between_evals: 1 - -profiling: - profiler: iostat \ No newline at end of file + epochs: 10 + computation_time: 1 + +checkpoint: + checkpoint_folder: checkpoints/unet3d + checkpoint_after_epoch: 5 + epochs_between_checkpoints: 2 + model_size: 499153191 diff --git a/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml b/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml new file mode 100644 index 00000000..d1fda034 --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml @@ -0,0 +1,28 @@ +model: resnet50 + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + num_files_train: 10240000 + num_samples_per_file: 1 + record_length: 150528 + data_folder: data/resnet50 + format: png + +train: + computation_time: 0.317 # this is for A100 + epochs: 100 + + +reader: + data_loader: pytorch + read_threads: 6 + computation_threads: 8 + batch_size: 1 + multiprocessing_context: spawn + data_loader_classname: torch_data_loader.BaseTorchDataLoader + data_loader_sampler: index \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/configs/workload/unet3d_base.yaml b/tests/integration/dlio_benchmark/configs/workload/unet3d_base.yaml index 89875d91..95941848 100644 --- a/tests/integration/dlio_benchmark/configs/workload/unet3d_base.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/unet3d_base.yaml @@ -5,7 +5,7 @@ framework: pytorch workflow: generate_data: False train: True - checkpoint: True + checkpoint: False dataset: data_folder: data/unet3d/ @@ -28,7 +28,7 @@ reader: train: epochs: 10 - computation_time: 0 #1.3604 + computation_time: 0 checkpoint: checkpoint_folder: checkpoints/unet3d diff --git a/tests/integration/dlio_benchmark/configs/workload/unet3d_base_large.yaml b/tests/integration/dlio_benchmark/configs/workload/unet3d_base_large.yaml new file mode 100644 index 00000000..0f092c2d --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/unet3d_base_large.yaml @@ -0,0 +1,37 @@ +model: unet3d + +framework: pytorch + +workflow: + generate_data: False + train: True + checkpoint: False + +dataset: + data_folder: data/unet3d/ + format: npz + num_files_train: 10240 + num_samples_per_file: 1 + record_length: 146600628 + record_length_stdev: 68341808 + record_length_resize: 2097152 + +reader: + data_loader: pytorch + batch_size: 4 + read_threads: 6 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: torch_data_loader.BaseTorchDataLoader + data_loader_sampler: index + +train: + epochs: 20 + computation_time: 0.188 + +checkpoint: + checkpoint_folder: checkpoints/unet3d + checkpoint_after_epoch: 5 + epochs_between_checkpoints: 2 + model_size: 499153191 diff --git a/tests/integration/dlio_benchmark/corona.sh b/tests/integration/dlio_benchmark/corona.sh index f9dbf708..a561ccbe 100755 --- a/tests/integration/dlio_benchmark/corona.sh +++ b/tests/integration/dlio_benchmark/corona.sh @@ -2,4 +2,4 @@ source ./setup-env.sh rm *.core flux.log rm -rf logs/* profiler/* -flux alloc -q pdebug -N $NUM_NODES -o per-resource.count=${BROKERS_PER_NODE} --exclusive --broker-opts=--setattr=log-filename=./logs/flux.log ./run_dlio.sh +flux alloc -q $QUEUE -t $TIME -N $NUM_NODES -o per-resource.count=${BROKERS_PER_NODE} --exclusive --broker-opts=--setattr=log-filename=./logs/flux.log ./run_dlio.sh diff --git a/tests/integration/dlio_benchmark/dyad_torch_data_loader.py b/tests/integration/dlio_benchmark/dyad_torch_data_loader.py index 5842c28e..cf10afef 100644 --- a/tests/integration/dlio_benchmark/dyad_torch_data_loader.py +++ b/tests/integration/dlio_benchmark/dyad_torch_data_loader.py @@ -30,7 +30,7 @@ from dlio_profiler.logger import fn_interceptor as Profile from pydyad import Dyad, dyad_open -from pydyad.bindings import DTLMode +from pydyad.bindings import DTLMode, DTLCommMode import numpy as np import flux import os @@ -83,15 +83,15 @@ def worker_init(self, worker_id): dtl_str = os.getenv("DYAD_DTL_MODE", "FLUX_RPC") mode = DTLMode.DYAD_DTL_FLUX_RPC namespace = os.getenv("DYAD_KVS_NAMESPACE") - logging.info(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} init dyad {self.dyad_managed_directory} {dtl_str} {namespace}") + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} init dyad {self.dyad_managed_directory} {dtl_str} {namespace}") if dtl_str == "UCX": mode = DTLMode.DYAD_DTL_UCX - self.dyad_io.init(debug=self._args.debug, check=False, shared_storage=False, reinit=True, - async_publish=False, fsync_write=False, key_depth=3, + self.dyad_io.init(debug=self._args.debug, check=False, shared_storage=False, reinit=False, + async_publish=True, fsync_write=False, key_depth=3, service_mux=self.broker_per_node, key_bins=1024, kvs_namespace=os.getenv("DYAD_KVS_NAMESPACE"), prod_managed_path=self.dyad_managed_directory, cons_managed_path=self.dyad_managed_directory, - dtl_mode=mode, dtl_comm_mode=DYAD_COMM_RECV) + dtl_mode=mode, dtl_comm_mode=DTLCommMode.DYAD_COMM_RECV) def __del__(self): if self.dlp_logger: @@ -102,6 +102,7 @@ def __len__(self): @dlp.log def __getitem__(self, image_idx): + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} image") self.num_images_read += 1 step = int(math.ceil(self.num_images_read / self.batch_size)) filename, sample_index = self._args.global_index_map[image_idx] @@ -111,7 +112,7 @@ def __getitem__(self, image_idx): dlp.update(args={"fname":filename}) dlp.update(args={"image_idx":image_idx}) if self.dyad_managed_directory != "": - logging.info(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading metadata") + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading metadata") base_fname = os.path.join(self.dyad_managed_directory, os.path.basename(filename)) file_obj = self.dyad_io.get_metadata(fname=base_fname, should_wait=False, raw=True) logging.debug(f"Using managed directory {self.dyad_managed_directory} {base_fname} {file_obj}") @@ -128,7 +129,12 @@ def __getitem__(self, image_idx): logging.info(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") logging.debug(f"Reading from managed directory {base_fname}") with dyad_open(base_fname, "rb", dyad_ctx=self.dyad_io, metadata_wrapper=file_obj) as f: - data = np.load(f, allow_pickle=True)["x"] + try: + data = np.load(f, allow_pickle=True)["x"] + except: + logging.info(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} got wierd {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") + data = self._args.resized_image + logging.info(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} read {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") self.dyad_io.free_metadata(file_obj) else: dlp.update(args={"mode":"pfs"}) @@ -140,6 +146,7 @@ def __getitem__(self, image_idx): logging.debug(f"Writing to managed_directory {base_fname}") with dyad_open(base_fname, "wb", dyad_ctx=self.dyad_io) as f: np.savez(f, x=data) + logging.debug(f"Read from pfs {base_fname}") dlp.update(step=step) dlp.update(image_size=data.nbytes) diff --git a/tests/integration/dlio_benchmark/run_dlio.sh b/tests/integration/dlio_benchmark/run_dlio.sh index 021e6a73..207bcbf2 100755 --- a/tests/integration/dlio_benchmark/run_dlio.sh +++ b/tests/integration/dlio_benchmark/run_dlio.sh @@ -4,10 +4,9 @@ source ./setup-env.sh # Setup DYAD echo Setting up Dyad flux kvs namespace create ${DYAD_KVS_NAMESPACE} -for (( c=0; c= 1: prefetch_factor = math.ceil(self._args.prefetch_size / self._args.read_threads) else: @@ -147,7 +166,6 @@ def read(self): pin_memory=True, drop_last=True, worker_init_fn=dataset.worker_init, - generator=g, **kwargs) # 2 is the default value logging.debug(f"{utcnow()} Rank {self._args.my_rank} will read {len(self._dataset) * self.batch_size} files") From 21a39f89936714a34f305156034d840b937911af Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 4 Mar 2024 14:02:04 -0500 Subject: [PATCH 03/47] Adds HDF5 API to pydyad --- pydyad/pydyad/hdf.py | 41 +++++++++++++++++++++++++++++++++++++++++ pydyad/setup.cfg | 1 + 2 files changed, 42 insertions(+) create mode 100644 pydyad/pydyad/hdf.py diff --git a/pydyad/pydyad/hdf.py b/pydyad/pydyad/hdf.py new file mode 100644 index 00000000..50bd4ccf --- /dev/null +++ b/pydyad/pydyad/hdf.py @@ -0,0 +1,41 @@ +from pydyad.bindings import Dyad + +from pathlib import Path + +import h5py + + +class File(h5py.File): + + def __init__(self, *args, **kwargs, dyad_ctx=None, metadata_wrapper=None): + # According to H5PY, the first positional argument to File.__init__ is fname + self.fname = args[0] + if not isinstance(self.fname, Path): + self.fname = Path(self.fname) + self.fname = self.fname.expanduser().resolve() + self.mode = None + if "mode" in kwargs: + self.mode = kwargs["mode"] + elif len(args) > 1: + self.mode = args[1] + else: + raise NameError("'mode' argument not provided to pydyad.hdf.File constructor") + if dyad_ctx is None: + raise NameError("'dyad_ctx' argument not provided to pydyad.hdf.File constructor") + self.dyad_ctx = dyad_ctx + if self.mode in ("r", "rb", "rt"): + if (self.dyad_ctx.cons_path is not None and + self.dyad_ctx.cons_path in self.fname.parents): + if metadata_wrapper: + self.dyad_ctx.consume_w_metadata(str(self.fname), meatadata_wrapper) + else: + dyad_ctx.consume(str(self.fname)) + super().__init__(*args, **kwargs) + + + def close(self): + super().close() + if self.mode in ("w", "wb", "wt"): + if (self.dyad_ctx.prod_path is not None and + self.dyad_ctx.prod_path in self.fname.parents): + self.dyad_ctx.produce(str(self.fname)) \ No newline at end of file diff --git a/pydyad/setup.cfg b/pydyad/setup.cfg index 6daa7cdd..710db93b 100644 --- a/pydyad/setup.cfg +++ b/pydyad/setup.cfg @@ -8,4 +8,5 @@ classifier = python_requires = >=3.7 install_requires = numpy + h5py dlio_profiler_py @ git+https://github.com/hariharan-devarajan/dlio-profiler.git \ No newline at end of file From 2fdc2ce7b3cad4b0d8a65d66f1d31486cc720a95 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 4 Mar 2024 14:08:56 -0500 Subject: [PATCH 04/47] Changes pydyad.hdf.File to pydyad.hdf.DyadFile to avoid name conflicts --- pydyad/pydyad/hdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydyad/pydyad/hdf.py b/pydyad/pydyad/hdf.py index 50bd4ccf..fdc43f64 100644 --- a/pydyad/pydyad/hdf.py +++ b/pydyad/pydyad/hdf.py @@ -5,7 +5,7 @@ import h5py -class File(h5py.File): +class DyadFile(h5py.File): def __init__(self, *args, **kwargs, dyad_ctx=None, metadata_wrapper=None): # According to H5PY, the first positional argument to File.__init__ is fname From dc796ec3b68643e8a77f7cf95a844a1b496f5f73 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Mon, 4 Mar 2024 20:26:01 -0800 Subject: [PATCH 05/47] changes to support DLIO with DYAD 1. Fix Enum for DTLMode 2. Split I/O and locking for consume_w_metadata 3. Comment the STDERR 4. Configurations for DLIO with DYAD --- .../configs/workload/dyad_resnet50.yaml | 9 ++--- .../configs/workload/dyad_resnet50_small.yaml | 27 ++++++++++++++ .../configs/workload/mummi_base.yaml | 36 +++++++++++++++++++ .../configs/workload/resnet50_base.yaml | 8 ++--- .../configs/workload/resnet50_base_small.yaml | 28 +++++++++++++++ tests/integration/dlio_benchmark/setup-env.sh | 5 +-- 6 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 tests/integration/dlio_benchmark/configs/workload/dyad_resnet50_small.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/resnet50_base_small.yaml diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml index 2b79edf3..a767cbf1 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50.yaml @@ -7,7 +7,8 @@ workflow: train: True dataset: - num_files_train: 10240000 + num_subfolders_train: 21843 + num_files_train: 218368 num_samples_per_file: 1 record_length: 150528 data_folder: data/resnet50 @@ -15,12 +16,12 @@ dataset: train: computation_time: 0.317 # this is for A100 - epochs: 100 + epochs: 20 reader: data_loader: pytorch - read_threads: 6 - computation_threads: 8 + read_threads: 1 + computation_threads: 1 batch_size: 1 multiprocessing_context: spawn data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50_small.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50_small.yaml new file mode 100644 index 00000000..b9297585 --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_resnet50_small.yaml @@ -0,0 +1,27 @@ +model: unet3d + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + num_files_train: 1024 + num_samples_per_file: 1 + record_length: 150528 + data_folder: data/resnet50 + format: png + +train: + computation_time: 0.317 # this is for A100 + epochs: 100 + +reader: + data_loader: pytorch + read_threads: 6 + computation_threads: 8 + batch_size: 1 + multiprocessing_context: spawn + data_loader_classname: dyad_torch_data_loader.DyadTorchDataLoader + data_loader_sampler: index \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml new file mode 100644 index 00000000..4317e30a --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml @@ -0,0 +1,36 @@ +model: mummi + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + data_folder: data/mummi/ + format: hdf5 + num_files_train: 600 + num_samples_per_file: 8000 + record_length: 69528 + enable_chunking: True + chunk_size: 17799168 + +reader: + data_loader: pytorch + batch_size: 256 + read_threads: 6 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: torch_data_loader.BaseTorchDataLoader + data_loader_sampler: index + +train: + epochs: 10 + computation_time: 0 + +checkpoint: + checkpoint_folder: checkpoints/unet3d + checkpoint_after_epoch: 5 + epochs_between_checkpoints: 2 + model_size: 499153191 diff --git a/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml b/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml index d1fda034..09b832b2 100644 --- a/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/resnet50_base.yaml @@ -7,7 +7,8 @@ workflow: train: True dataset: - num_files_train: 10240000 + num_subfolders_train: 21843 + num_files_train: 218430 num_samples_per_file: 1 record_length: 150528 data_folder: data/resnet50 @@ -15,14 +16,13 @@ dataset: train: computation_time: 0.317 # this is for A100 - epochs: 100 + epochs: 20 reader: data_loader: pytorch read_threads: 6 - computation_threads: 8 - batch_size: 1 + batch_size: 512 multiprocessing_context: spawn data_loader_classname: torch_data_loader.BaseTorchDataLoader data_loader_sampler: index \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/configs/workload/resnet50_base_small.yaml b/tests/integration/dlio_benchmark/configs/workload/resnet50_base_small.yaml new file mode 100644 index 00000000..f0c1db1e --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/resnet50_base_small.yaml @@ -0,0 +1,28 @@ +model: resnet50 + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + num_files_train: 1024 + num_samples_per_file: 1 + record_length: 150528 + data_folder: data/resnet50 + format: png + +train: + computation_time: 0.317 # this is for A100 + epochs: 20 + + +reader: + data_loader: pytorch + read_threads: 6 + computation_threads: 8 + batch_size: 1 + multiprocessing_context: spawn + data_loader_classname: torch_data_loader.BaseTorchDataLoader + data_loader_sampler: index \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/setup-env.sh b/tests/integration/dlio_benchmark/setup-env.sh index b7175d59..17c3db55 100644 --- a/tests/integration/dlio_benchmark/setup-env.sh +++ b/tests/integration/dlio_benchmark/setup-env.sh @@ -4,7 +4,7 @@ module load python/3.9.12 module load openmpi/4.1.2 # Configurations -export DLIO_WORKLOAD=resnet50_base #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small +export DLIO_WORKLOAD=dyad_resnet50 #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large export NUM_NODES=32 export PPN=8 export QUEUE=pbatch @@ -21,7 +21,8 @@ export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad export SPACK_DIR=/usr/workspace/haridev/spack export SPACK_ENV=/usr/workspace/haridev/dyad/env/spack export PYTHON_ENV=/usr/workspace/haridev/dyad/env/python -export DLIO_DATA_DIR=/p/lustre2/haridev/dyad/dlio_benchmark/dyad_resnet50 # dyad_unet3d_basic +export DLIO_DATA_DIR=/p/lustre1/iopp/dyad/dlio_benchmark/dyad_resnet50 # dyad_resnet50 dyad_unet3d_basic +#export DLIO_DATA_DIR=/p/lustre2/haridev/dyad/dlio_benchmark/dyad_unet3d_basic # dyad_resnet50 # DLIO Profiler Configurations export DLIO_PROFILER_ENABLE=1 From f30904c2d98760b4caab0c8bbf09f38f5640665c Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Tue, 5 Mar 2024 01:56:25 -0800 Subject: [PATCH 06/47] changes to support DLIO with MuMMI and DYAD --- pydyad/pydyad/hdf.py | 26 +- .../configs/workload/dyad_mummi.yaml | 30 +++ .../configs/workload/dyad_mummi_small.yaml | 30 +++ .../configs/workload/mummi_base.yaml | 6 - .../dyad_h5_torch_data_loader.py | 234 ++++++++++++++++++ tests/integration/dlio_benchmark/setup-env.sh | 11 +- 6 files changed, 311 insertions(+), 26 deletions(-) create mode 100644 tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/dyad_mummi_small.yaml create mode 100644 tests/integration/dlio_benchmark/dyad_h5_torch_data_loader.py diff --git a/pydyad/pydyad/hdf.py b/pydyad/pydyad/hdf.py index fdc43f64..880d6c1e 100644 --- a/pydyad/pydyad/hdf.py +++ b/pydyad/pydyad/hdf.py @@ -7,35 +7,31 @@ class DyadFile(h5py.File): - def __init__(self, *args, **kwargs, dyad_ctx=None, metadata_wrapper=None): + def __init__(self, fname, mode, file=None, dyad_ctx=None, metadata_wrapper=None): # According to H5PY, the first positional argument to File.__init__ is fname - self.fname = args[0] + self.fname = fname if not isinstance(self.fname, Path): - self.fname = Path(self.fname) + self.fname = Path(fname) self.fname = self.fname.expanduser().resolve() - self.mode = None - if "mode" in kwargs: - self.mode = kwargs["mode"] - elif len(args) > 1: - self.mode = args[1] - else: - raise NameError("'mode' argument not provided to pydyad.hdf.File constructor") + self.m = mode if dyad_ctx is None: raise NameError("'dyad_ctx' argument not provided to pydyad.hdf.File constructor") self.dyad_ctx = dyad_ctx - if self.mode in ("r", "rb", "rt"): + if self.m in ("r"): if (self.dyad_ctx.cons_path is not None and self.dyad_ctx.cons_path in self.fname.parents): if metadata_wrapper: - self.dyad_ctx.consume_w_metadata(str(self.fname), meatadata_wrapper) + self.dyad_ctx.consume_w_metadata(str(self.fname), metadata_wrapper) else: dyad_ctx.consume(str(self.fname)) - super().__init__(*args, **kwargs) - + if file: + super().__init__(file, mode) + else: + super().__init__(fname, mode) def close(self): super().close() - if self.mode in ("w", "wb", "wt"): + if self.m in ("w", "r+"): if (self.dyad_ctx.prod_path is not None and self.dyad_ctx.prod_path in self.fname.parents): self.dyad_ctx.produce(str(self.fname)) \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml new file mode 100644 index 00000000..92dab9c4 --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml @@ -0,0 +1,30 @@ +model: mummi + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + data_folder: data/mummi/ + format: hdf5 + num_files_train: 600 + num_samples_per_file: 8000 + record_length: 69528 + enable_chunking: True + chunk_size: 17799168 + +reader: + data_loader: pytorch + batch_size: 256 + read_threads: 6 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: dyad_h5_torch_data_loader.DyadH5TorchDataLoader + data_loader_sampler: index + +train: + epochs: 10 + computation_time: 0 diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_mummi_small.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi_small.yaml new file mode 100644 index 00000000..ec40905b --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi_small.yaml @@ -0,0 +1,30 @@ +model: mummi + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + data_folder: data/mummi/ + format: hdf5 + num_files_train: 1 + num_samples_per_file: 100 + record_length: 69528 + enable_chunking: True + chunk_size: 17799168 + +reader: + data_loader: pytorch + batch_size: 1 + read_threads: 2 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: dyad_h5_torch_data_loader.DyadH5TorchDataLoader + data_loader_sampler: index + +train: + epochs: 10 + computation_time: .133 diff --git a/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml index 4317e30a..f7f9aeb0 100644 --- a/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml @@ -28,9 +28,3 @@ reader: train: epochs: 10 computation_time: 0 - -checkpoint: - checkpoint_folder: checkpoints/unet3d - checkpoint_after_epoch: 5 - epochs_between_checkpoints: 2 - model_size: 499153191 diff --git a/tests/integration/dlio_benchmark/dyad_h5_torch_data_loader.py b/tests/integration/dlio_benchmark/dyad_h5_torch_data_loader.py new file mode 100644 index 00000000..34e35e2d --- /dev/null +++ b/tests/integration/dlio_benchmark/dyad_h5_torch_data_loader.py @@ -0,0 +1,234 @@ +""" + Copyright (c) 2022, UChicago Argonne, LLC + All Rights Reserved + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +from time import time +import logging +import math +import pickle +import torch +from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler + +from dlio_benchmark.common.constants import MODULE_DATA_LOADER +from dlio_benchmark.common.enumerations import Shuffle, DatasetType, DataLoaderType +from dlio_benchmark.data_loader.base_data_loader import BaseDataLoader +from dlio_benchmark.reader.reader_factory import ReaderFactory +from dlio_benchmark.utils.utility import utcnow, DLIOMPI +from dlio_benchmark.utils.config import ConfigArguments +from dlio_profiler.logger import fn_interceptor as Profile + +from pydyad import Dyad +from pydyad.hdf import DyadFile +from pydyad.bindings import DTLMode, DTLCommMode +import numpy as np +import flux +import os +import h5py +import fcntl +dlp = Profile(MODULE_DATA_LOADER) + + +class DYADH5TorchDataset(Dataset): + """ + Currently, we only support loading one sample per file + TODO: support multiple samples per file + """ + @dlp.log_init + def __init__(self, format_type, dataset_type, epoch, num_samples, num_workers, batch_size): + self.format_type = format_type + self.dataset_type = dataset_type + self.epoch_number = epoch + self.num_samples = num_samples + self.reader = None + self.num_images_read = 0 + self.batch_size = batch_size + args = ConfigArguments.get_instance() + self.serial_args = pickle.dumps(args) + self.dlp_logger = None + if num_workers == 0: + self.worker_init(-1) + + @dlp.log + def worker_init(self, worker_id): + pickle.loads(self.serial_args) + self._args = ConfigArguments.get_instance() + self._args.configure_dlio_logging(is_child=True) + self.dlp_logger = self._args.configure_dlio_profiler(is_child=True, use_pid=True) + logging.debug(f"{utcnow()} worker initialized {worker_id} with format {self.format_type}") + self.reader = ReaderFactory.get_reader(type=self.format_type, + dataset_type=self.dataset_type, + thread_index=worker_id, + epoch_number=self.epoch_number) + self.dyad_io = Dyad() + is_local = os.getenv("DYAD_LOCAL_TEST", "0") == "1" + self.broker_per_node = int(os.getenv("BROKERS_PER_NODE", "1")) + + self.f = flux.Flux() + self.broker_rank = self.f.get_rank() + if is_local: + self.dyad_managed_directory = os.path.join(os.getenv("DYAD_PATH", ""), str(self.f.get_rank())) + else: + self.dyad_managed_directory = os.getenv("DYAD_PATH", "") + self.my_node_index = int(self.broker_rank*1.0 / self.broker_per_node) + dtl_str = os.getenv("DYAD_DTL_MODE", "FLUX_RPC") + mode = DTLMode.DYAD_DTL_FLUX_RPC + namespace = os.getenv("DYAD_KVS_NAMESPACE") + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} init dyad {self.dyad_managed_directory} {dtl_str} {namespace}") + if dtl_str == "UCX": + mode = DTLMode.DYAD_DTL_UCX + self.dyad_io.init(debug=self._args.debug, check=False, shared_storage=False, reinit=False, + async_publish=True, fsync_write=False, key_depth=3, + service_mux=self.broker_per_node, + key_bins=1024, kvs_namespace=os.getenv("DYAD_KVS_NAMESPACE"), + prod_managed_path=self.dyad_managed_directory, cons_managed_path=self.dyad_managed_directory, + dtl_mode=mode, dtl_comm_mode=DTLCommMode.DYAD_COMM_RECV) + + def __del__(self): + if self.dlp_logger: + self.dlp_logger.finalize() + @dlp.log + def __len__(self): + return self.num_samples + + @dlp.log + def __getitem__(self, image_idx): + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} image") + self.num_images_read += 1 + step = int(math.ceil(self.num_images_read / self.batch_size)) + filename, sample_index = self._args.global_index_map[image_idx] + is_present = False + file_obj = None + base_fname = filename + dlp.update(args={"fname":filename}) + dlp.update(args={"image_idx":image_idx}) + if self.dyad_managed_directory != "": + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading metadata") + base_fname = os.path.join(self.dyad_managed_directory, os.path.basename(filename)) + file_obj = self.dyad_io.get_metadata(fname=base_fname, should_wait=False, raw=True) + logging.debug(f"Using managed directory {self.dyad_managed_directory} {base_fname} {file_obj}") + is_present = True + if file_obj: + access_mode = "remote" + file_node_index = int(file_obj.contents.owner_rank*1.0 / self.broker_per_node) + if self.my_node_index == file_node_index: + access_mode = "local" + dlp.update(args={"owner_rank":str(file_obj.contents.owner_rank)}) + dlp.update(args={"my_broker":str(self.broker_rank)}) + dlp.update(args={"mode":"dyad"}) + dlp.update(args={"access":access_mode}) + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") + logging.debug(f"Reading from managed directory {base_fname}") + hf = DyadFile(base_fname, "r", dyad_ctx=self.dyad_io, metadata_wrapper=file_obj) + try: + data = hf["records"][sample_index] + except: + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} got wierd {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") + data = self._args.resized_image + hf.close() + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} read {image_idx} sample from {access_mode} dyad {file_obj.contents.owner_rank}") + self.dyad_io.free_metadata(file_obj) + else: + dlp.update(args={"mode":"pfs"}) + dlp.update(args={"access":"remote"}) + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} sample from pfs {base_fname}") + logging.debug(f"Reading from pfs {base_fname}") + dyad_f = open(base_fname, "wb") + fcntl.lockf(dyad_f, fcntl.LOCK_EX) + dyad_f.seek(0, 2) + size = dyad_f.tell() + if size == 0: + pfs_f = open(filename, "rb") + data = pfs_f.read() + dyad_f.write(data) + pfs_f.close() + fcntl.lockf(dyad_f, fcntl.LOCK_UN) + dyad_f.close() + hf = DyadFile(base_fname, "r+", dyad_ctx=self.dyad_io) + data = hf["records"][sample_index] + hf.close() + logging.debug(f"Read from pfs {base_fname}") + dlp.update(step=step) + dlp.update(image_size=data.nbytes) + return data + +class DyadH5TorchDataLoader(BaseDataLoader): + @dlp.log_init + def __init__(self, format_type, dataset_type, epoch_number): + super().__init__(format_type, dataset_type, epoch_number, DataLoaderType.PYTORCH) + + @dlp.log + def read(self): + do_shuffle = True if self._args.sample_shuffle != Shuffle.OFF else False + dataset = DYADH5TorchDataset(self.format_type, self.dataset_type, self.epoch_number, self.num_samples, self._args.read_threads, self.batch_size) + if do_shuffle: + sampler = RandomSampler(dataset) + else: + sampler = SequentialSampler(dataset) + if self._args.read_threads >= 1: + prefetch_factor = math.ceil(self._args.prefetch_size / self._args.read_threads) + else: + prefetch_factor = self._args.prefetch_size + if prefetch_factor > 0: + if self._args.my_rank == 0: + logging.debug( + f"{utcnow()} Prefetch size is {self._args.prefetch_size}; prefetch factor of {prefetch_factor} will be set to Torch DataLoader.") + else: + prefetch_factor = 2 + if self._args.my_rank == 0: + logging.debug( + f"{utcnow()} Prefetch size is 0; a default prefetch factor of 2 will be set to Torch DataLoader.") + logging.debug(f"{utcnow()} Setup dataloader with {self._args.read_threads} workers {torch.__version__}") + if self._args.read_threads==0: + kwargs={} + else: + kwargs={'multiprocessing_context':self._args.multiprocessing_context, + 'prefetch_factor': prefetch_factor} + if torch.__version__ != '1.3.1': + kwargs['persistent_workers'] = True + if torch.__version__ == '1.3.1': + if 'prefetch_factor' in kwargs: + del kwargs['prefetch_factor'] + self._dataset = DataLoader(dataset, + batch_size=self.batch_size, + sampler=sampler, + num_workers=self._args.read_threads, + pin_memory=True, + drop_last=True, + worker_init_fn=dataset.worker_init, + **kwargs) + else: + self._dataset = DataLoader(dataset, + batch_size=self.batch_size, + sampler=sampler, + num_workers=self._args.read_threads, + pin_memory=True, + drop_last=True, + worker_init_fn=dataset.worker_init, + **kwargs) # 2 is the default value + logging.debug(f"{utcnow()} Rank {self._args.my_rank} will read {len(self._dataset) * self.batch_size} files") + + # self._dataset.sampler.set_epoch(epoch_number) + + @dlp.log + def next(self): + super().next() + total = self._args.training_steps if self.dataset_type is DatasetType.TRAIN else self._args.eval_steps + logging.debug(f"{utcnow()} Rank {self._args.my_rank} should read {total} batches") + for batch in self._dataset: + yield batch + + @dlp.log + def finalize(self): + pass diff --git a/tests/integration/dlio_benchmark/setup-env.sh b/tests/integration/dlio_benchmark/setup-env.sh index 17c3db55..cb89dfcd 100644 --- a/tests/integration/dlio_benchmark/setup-env.sh +++ b/tests/integration/dlio_benchmark/setup-env.sh @@ -4,10 +4,10 @@ module load python/3.9.12 module load openmpi/4.1.2 # Configurations -export DLIO_WORKLOAD=dyad_resnet50 #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large -export NUM_NODES=32 +export DLIO_WORKLOAD=dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi +export NUM_NODES=4 export PPN=8 -export QUEUE=pbatch +export QUEUE=pdebug export TIME=$((60)) export BROKERS_PER_NODE=1 export GENERATE_DATA="0" @@ -21,11 +21,11 @@ export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad export SPACK_DIR=/usr/workspace/haridev/spack export SPACK_ENV=/usr/workspace/haridev/dyad/env/spack export PYTHON_ENV=/usr/workspace/haridev/dyad/env/python -export DLIO_DATA_DIR=/p/lustre1/iopp/dyad/dlio_benchmark/dyad_resnet50 # dyad_resnet50 dyad_unet3d_basic +export DLIO_DATA_DIR=/p/lustre1/iopp/dyad/dlio_benchmark/dyad_mummi # dyad_resnet50 dyad_unet3d_basic #export DLIO_DATA_DIR=/p/lustre2/haridev/dyad/dlio_benchmark/dyad_unet3d_basic # dyad_resnet50 # DLIO Profiler Configurations -export DLIO_PROFILER_ENABLE=1 +export DLIO_PROFILER_ENABLE=0 export DLIO_PROFILER_INC_METADATA=1 export DLIO_PROFILER_DATA_DIR=${DLIO_DATA_DIR}:${DYAD_PATH} export DLIO_PROFILER_LOG_FILE=/usr/workspace/haridev/dyad/tests/integration/dlio_benchmark/profiler/dyad @@ -35,6 +35,7 @@ export DLIO_PROFILER_LOG_LEVEL=ERROR export DLIO_PROFILER_BIND_SIGNALS=0 export MV2_BCAST_HWLOC_TOPOLOGY=0 +export HDF5_USE_FILE_LOCKING=0 #mkdir -p ${DYAD_PATH} mkdir -p ${DLIO_PROFILER_LOG_FILE} From 9c6bb666bffdbda7b58dbf7241657e30ddaf386f Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Tue, 5 Mar 2024 23:40:14 -0800 Subject: [PATCH 07/47] changes to support DLIO with MuMMI and DYAD --- .../dlio_benchmark/configs/workload/dyad_mummi.yaml | 2 +- .../dlio_benchmark/configs/workload/mummi_base.yaml | 2 +- tests/integration/dlio_benchmark/setup-env.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml index 92dab9c4..55e8d027 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dyad_mummi.yaml @@ -27,4 +27,4 @@ reader: train: epochs: 10 - computation_time: 0 + computation_time: .133 diff --git a/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml index f7f9aeb0..da8022d1 100644 --- a/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/mummi_base.yaml @@ -27,4 +27,4 @@ reader: train: epochs: 10 - computation_time: 0 + computation_time: .133 diff --git a/tests/integration/dlio_benchmark/setup-env.sh b/tests/integration/dlio_benchmark/setup-env.sh index cb89dfcd..debccdc3 100644 --- a/tests/integration/dlio_benchmark/setup-env.sh +++ b/tests/integration/dlio_benchmark/setup-env.sh @@ -5,9 +5,9 @@ module load openmpi/4.1.2 # Configurations export DLIO_WORKLOAD=dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi -export NUM_NODES=4 +export NUM_NODES=64 export PPN=8 -export QUEUE=pdebug +export QUEUE=pbatch export TIME=$((60)) export BROKERS_PER_NODE=1 export GENERATE_DATA="0" From e5f511946131608c0b8bf0963c05aaed64fa0c4b Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 18 Mar 2024 16:43:56 -0400 Subject: [PATCH 08/47] Does initial changes for DataSpaces baseline plus reworks the data plane tests for DataSpaces --- tests/dspaces_perf/CMakeLists.txt | 35 +++ tests/dspaces_perf/catch_config.h | 24 ++ tests/dspaces_perf/data_plane/CMakeLists.txt | 11 + tests/dspaces_perf/data_plane/data_plane.cpp | 303 +++++++++++++++++++ tests/dspaces_perf/mdm/CMakeLists.txt | 29 ++ tests/dspaces_perf/mdm/mdm.cpp | 151 +++++++++ tests/dspaces_perf/script/CMakeLists.txt | 9 + tests/dspaces_perf/script/dspaces_start.sh | 11 + tests/dspaces_perf/script/dspaces_stop.sh | 1 + tests/dspaces_perf/test_utils.h | 66 ++++ tests/dspaces_perf/unit_test.cpp | 85 ++++++ 11 files changed, 725 insertions(+) create mode 100644 tests/dspaces_perf/CMakeLists.txt create mode 100644 tests/dspaces_perf/catch_config.h create mode 100644 tests/dspaces_perf/data_plane/CMakeLists.txt create mode 100644 tests/dspaces_perf/data_plane/data_plane.cpp create mode 100644 tests/dspaces_perf/mdm/CMakeLists.txt create mode 100644 tests/dspaces_perf/mdm/mdm.cpp create mode 100644 tests/dspaces_perf/script/CMakeLists.txt create mode 100755 tests/dspaces_perf/script/dspaces_start.sh create mode 100755 tests/dspaces_perf/script/dspaces_stop.sh create mode 100644 tests/dspaces_perf/test_utils.h create mode 100644 tests/dspaces_perf/unit_test.cpp diff --git a/tests/dspaces_perf/CMakeLists.txt b/tests/dspaces_perf/CMakeLists.txt new file mode 100644 index 00000000..5d7692e6 --- /dev/null +++ b/tests/dspaces_perf/CMakeLists.txt @@ -0,0 +1,35 @@ +if(NOT DEFINED ENV{DYAD_TEST_MACHINE}) + message(FATAL_ERROR "-- [dyad] DYAD_TEST_MACHINE in env should be set for ${PROJECT_NAME} test build") +else() + message(STATUS "[dyad] found setting machine to $ENV{DYAD_TEST_MACHINE}") +endif() +if(NOT DEFINED ENV{DYAD_PFS_DIR}) + message(FATAL_ERROR "-- [dyad] DYAD_PFS_DIR in env should be set for ${PROJECT_NAME} paper test build") +else() + message(STATUS "[dyad] found setting pfs dir to $ENV{DYAD_PFS_DIR}") +endif() +if(NOT DEFINED ENV{DYAD_DMD_DIR}) + message(FATAL_ERROR "-- [dyad] DYAD_DMD_DIR in env should be set for ${PROJECT_NAME} paper test build") +else() + message(STATUS "[dyad] found setting DMD Dir to $ENV{DYAD_DMD_DIR}") +endif() +set(DYAD_KEYSPACE test_dyad) +set(DYAD_LOG_DIR ${CMAKE_BINARY_DIR}/logs) +file(MAKE_DIRECTORY ${DYAD_LOG_DIR}) +find_package(Catch2 REQUIRED) +find_package(MPI REQUIRED COMPONENTS CXX) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(Catch2::Catch2) +include_directories(${MPI_CXX_INCLUDE_DIRS}) +include_directories(${DYAD_PROJECT_DIR}/src) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +include_directories(${CMAKE_BINARY_DIR}/include) +set(TEST_LIBS Catch2::Catch2 -lstdc++fs ${MPI_CXX_LIBRARIES} -rdynamic dyad_core dyad_ctx dyad_utils flux-core ${CPP_LOGGER_LIBRARIES}) +set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.h) +add_executable(unit_test unit_test.cpp ${TEST_SRC} ) +target_link_libraries(unit_test ${TEST_LIBS}) +add_dependencies(unit_test dyad) + +add_subdirectory(script) +add_subdirectory(data_plane) +add_subdirectory(mdm) \ No newline at end of file diff --git a/tests/dspaces_perf/catch_config.h b/tests/dspaces_perf/catch_config.h new file mode 100644 index 00000000..9eb9ac24 --- /dev/null +++ b/tests/dspaces_perf/catch_config.h @@ -0,0 +1,24 @@ +#ifndef DYAD_CATCH_CONFIG_H +#define DYAD_CATCH_CONFIG_H +#include +namespace cl = Catch::Clara; + +cl::Parser define_options(); + +int init(int* argc, char*** argv); +int finalize(); + +int main(int argc, char* argv[]) { + Catch::Session session; + auto cli = session.cli() | define_options(); + session.cli(cli); + int returnCode = session.applyCommandLine(argc, argv); + if (returnCode != 0) return returnCode; + returnCode = init(&argc, &argv); + if (returnCode != 0) return returnCode; + int test_return_code = session.run(); + returnCode = finalize(); + if (returnCode != 0) return returnCode; + exit(test_return_code); +} +#endif // DYAD_CATCH_CONFIG_H diff --git a/tests/dspaces_perf/data_plane/CMakeLists.txt b/tests/dspaces_perf/data_plane/CMakeLists.txt new file mode 100644 index 00000000..3819b23d --- /dev/null +++ b/tests/dspaces_perf/data_plane/CMakeLists.txt @@ -0,0 +1,11 @@ +set(node 1) +set(ppn 1) +set(files 1) +set(test_name unit_remote_data_${node}_${ppn}) +add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteDataBandwidth) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp new file mode 100644 index 00000000..8d60f641 --- /dev/null +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -0,0 +1,303 @@ +#include +#include + +#include + +int create_files_per_server_process(dspaces_client_t* client, bool use_local) { + int rc = 0; + char filename[4096]; + size_t file_size = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + size_t server_proc_local_idx = info.rank % args.server_ppn; + // Clients are connected round-robin to server processes on the same node. + // To work out which processes should write "files", we first get a node-local rank for + // each client process by modulo dividing the global rank (info.rank) with the processes per node + // (args.process_per_node). Then, we floor divide that node-local rank by args.server_ppn. This + // floor division should only equal 0 for the first "args.server_ppn" ranks to connect to local + // server processes. Since DataSpaces connects to local processes round-robin, these ranks for which the + // floor division equals 0 are the "first" processes to connect to each local server processes. + bool first_rank_per_server_proc = (info.rank % args.process_per_node) / args.server_ppn == 0; + if (first_rank_per_server_proc) { + std::string rand_data = GenRandom(file_size); + size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + uint64_t lb = 0; + uint64_t ub = rand_data.size()-1; + uint64_t buf_size = ub + 1; + dspaces_define_gdim(*client, filename, 1, &buf_size); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + if (use_local) { + rc = dspaces_put_local(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); + } else { + rc = dspaces_put(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); + } + if (rc != 0) { + // TODO log error? + } + } + } + MPI_Barrier (MPI_COMM_WORLD); + return rc; +} + +TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_server_process(&client, false) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t neighbor_node_idx = ((info.rank / args.process_per_node) + 1) % info.num_nodes; + size_t server_proc_local_idx = info.rank % args.server_ppn; + uint32_t neighour_server_process_idx = neighbor_node_idx * args.server_ppn + server_proc_local_idx; + size_t data_len = args.request_size*args.iteration; + char* file_data = NULL; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), neighbour_server_process_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} + +TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_broker(&client, false) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t neighbor_node_idx = ((info.rank / args.process_per_node) + 1) % info.num_nodes; + size_t server_proc_local_idx = info.rank % args.server_ppn; + uint32_t neighour_server_process_idx = neighbor_node_idx * args.server_ppn + server_proc_local_idx; + size_t data_len = args.request_size*args.iteration; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), neighbour_server_process_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + if (info.rank % args.process_per_node != 0) + usleep (10000); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Unlike the previous test, we set the timeout to -1 so it will do any blocking that it might want to do + // TODO: confirm that the timeout is actually needed to guarantee this type of behavior + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_broker(&client, true) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalServerProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_broker(&client, false) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalClientNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_broker(&client, true) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t read_server_proc_local_idx = (server_proc_local_idx + 1) % args.server_ppn; + size_t global_server_proc_idx = node_idx * args.server_ppn + read_server_proc_local_idx; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalServerNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_broker(&client, false) == 0); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size*args.iteration; + size_t node_idx = info.rank / args.process_per_node; + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t read_server_proc_local_idx = (server_proc_local_idx + 1) % args.server_ppn; + size_t global_server_proc_idx = node_idx * args.server_ppn + read_server_proc_local_idx; + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + int ndim = 0; + uint64_t buf_size = 0; + dspaces_get_gdim(client, filename, &ndim, &buf_size); + uint64_t lb = 0; + uint64_t ub = buf_size - 1; + char* file_data =(char*)malloc(data_len); + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the buffer + // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + data_time.pauseTime(); + REQUIRE (rc == dspaces_SUCCESS); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, data_len*args.number_of_files, + total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); + } + } + rc = dspaces_fini(); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (posttest() == 0); +} diff --git a/tests/dspaces_perf/mdm/CMakeLists.txt b/tests/dspaces_perf/mdm/CMakeLists.txt new file mode 100644 index 00000000..3717411e --- /dev/null +++ b/tests/dspaces_perf/mdm/CMakeLists.txt @@ -0,0 +1,29 @@ +set(node 1) +set(ppn 1) +set(files 1) +set(test_name unit_localfs_${node}_${ppn}) +set(mpiexec flux run -N ${node} --tasks-per-node ${ppn}) +set(mpiexec ) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalFSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +set(test_name unit_localkvs_${node}_${ppn}) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalKVSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +set(test_name unit_remotekvs_${node}_${ppn}) +add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteKVSLookup) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) +set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file diff --git a/tests/dspaces_perf/mdm/mdm.cpp b/tests/dspaces_perf/mdm/mdm.cpp new file mode 100644 index 00000000..d4cc2563 --- /dev/null +++ b/tests/dspaces_perf/mdm/mdm.cpp @@ -0,0 +1,151 @@ +#include +#include + +#include +TEST_CASE("LocalFSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + REQUIRE (clean_directories() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + struct flock exclusive_lock; + SECTION("Throughput") { + char filename[4096]; + Timer kvs_time; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (filename, + "%s/%s_%u_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + file_idx); + kvs_time.resumeTime(); + int lock_fd = open (filename, O_RDWR | O_CREAT, 0666); + kvs_time.pauseTime(); + REQUIRE (lock_fd != -1); + + kvs_time.resumeTime(); + rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + + kvs_time.resumeTime(); + auto file_size = get_file_size (lock_fd); + kvs_time.pauseTime(); + (void)file_size; + kvs_time.resumeTime(); + dyad_release_flock (ctx, lock_fd, &exclusive_lock); + int status = close (lock_fd); + kvs_time.pauseTime(); + REQUIRE (status == 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (clean_directories() == 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("LocalKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (my_filename, + "%s/%s_%u_%d_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + info.rank, + file_idx); + sprintf (lookup_filename, + "%s_%u_%d_%zu.bat", + args.filename.c_str (), + info.broker_idx, + info.rank, + file_idx); + rc = dyad_commit (ctx, my_filename); + REQUIRE (rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX+1] = {'\0'}; + gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (posttest() == 0); +} + +TEST_CASE("RemoteKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { + REQUIRE (pretest() == 0); + dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); + REQUIRE (rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + sprintf (my_filename, + "%s/%s_%u_%d_%d_%zu.bat", + args.dyad_managed_dir.c_str (), + args.filename.c_str (), + info.broker_idx, + info.rank, + info.comm_size, + file_idx); + sprintf (lookup_filename, + "%s_%u_%d_%d_%zu.bat", + args.filename.c_str (), + info.broker_idx, + info.rank, + info.comm_size, + file_idx); + rc = dyad_commit (ctx, my_filename); + REQUIRE (rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX+1] = {'\0'}; + gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE (rc >= 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", + info.comm_size, args.number_of_files, + total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); + } + } + rc = dyad_finalize(); + REQUIRE (rc >= 0); + REQUIRE (posttest() == 0); +} \ No newline at end of file diff --git a/tests/dspaces_perf/script/CMakeLists.txt b/tests/dspaces_perf/script/CMakeLists.txt new file mode 100644 index 00000000..85ada8a4 --- /dev/null +++ b/tests/dspaces_perf/script/CMakeLists.txt @@ -0,0 +1,9 @@ +add_test(dspaces_start ${CMAKE_CURRENT_SOURCE_DIR}/dspaces_start.sh) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_DEFAULT_VAR_SIZE=${DSPACES_DEFAULT_VAR_SIZE}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_MAX_VERSIONS=${DSPACES_MAX_VERSIONS}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_PPN=${DSPACES_PPN}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_ROOT=${DSPACES_ROOT}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_HG_CONNECTION_STR=${DSPACES_HG_CONNECTION_STR}) + +add_test(dspaces_stop ${CMAKE_CURRENT_SOURCE_DIR}/dspaces_stop.sh) +set_property(TEST dspaces_stop APPEND PROPERTY ENVIRONMENT DSPACES_ROOT=${DSPACES_ROOT}) diff --git a/tests/dspaces_perf/script/dspaces_start.sh b/tests/dspaces_perf/script/dspaces_start.sh new file mode 100755 index 00000000..e3f3a96e --- /dev/null +++ b/tests/dspaces_perf/script/dspaces_start.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "## Config file for DataSpaces server +ndim = 1 +dims = ${DSPACES_DEFAULT_VAR_SIZE} +max_versions = ${DSPACES_MAX_VAR_VERSIONS} +num_apps = 1" > dataspaces.conf + +dspaces_num_nodes=$(flux resource info | grep -oP "\d+ Nodes" | grep -oP "^\d+") + +flux submit -N ${dspaces_num_nodes} --tasks-per-node ${DSPACES_PPN} ${DSPACES_ROOT}/bin/dspaces_server ${DSPACES_HG_CONNECTION_STR} \ No newline at end of file diff --git a/tests/dspaces_perf/script/dspaces_stop.sh b/tests/dspaces_perf/script/dspaces_stop.sh new file mode 100755 index 00000000..1a6b4ccf --- /dev/null +++ b/tests/dspaces_perf/script/dspaces_stop.sh @@ -0,0 +1 @@ +flux run --ntasks=1 ${DSPACES_ROOT}/bin/terminator \ No newline at end of file diff --git a/tests/dspaces_perf/test_utils.h b/tests/dspaces_perf/test_utils.h new file mode 100644 index 00000000..11a62970 --- /dev/null +++ b/tests/dspaces_perf/test_utils.h @@ -0,0 +1,66 @@ +#ifndef DYAD_TEST_UTILS_H +#define DYAD_TEST_UTILS_H + +#include +#include +#include +#include + +const uint32_t KB = 1024; +const uint32_t MB = 1024 * 1024; +#define AGGREGATE_TIME(name) \ + double total_##name = 0.0; \ + auto name##_a = name##_time.getElapsedTime(); \ + MPI_Reduce(&name##_a, &total_##name, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); + +size_t GetRandomOffset(size_t i, unsigned int offset_seed, size_t stride, + size_t total_size) { + return abs((int)(((i * rand_r(&offset_seed)) % stride) % total_size)); +} +inline std::string get_filename(int fd) { + const int kMaxSize = 256; + char proclnk[kMaxSize]; + char filename[kMaxSize]; + snprintf(proclnk, kMaxSize, "/proc/self/fd/%d", fd); + size_t r = readlink(proclnk, filename, kMaxSize); + filename[r] = '\0'; + return filename; +} + +std::string GenRandom(const int len) { + std::string tmp_s; + static const char alphanum[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + srand(100); + + tmp_s.reserve(len); + + for (int i = 0; i < len; ++i) { + tmp_s += alphanum[rand() % (sizeof(alphanum) - 1)]; + } + + tmp_s[len - 1] = '\n'; + + return tmp_s; +} + +class Timer { + public: + Timer() : elapsed_time(0) {} + void resumeTime() { t1 = std::chrono::high_resolution_clock::now(); } + double pauseTime() { + auto t2 = std::chrono::high_resolution_clock::now(); + elapsed_time += std::chrono::duration(t2 - t1).count(); + return elapsed_time; + } + double getElapsedTime() { return elapsed_time; } + + private: + std::chrono::high_resolution_clock::time_point t1; + double elapsed_time; +}; + +#endif // DYAD_TEST_UTILS_H diff --git a/tests/dspaces_perf/unit_test.cpp b/tests/dspaces_perf/unit_test.cpp new file mode 100644 index 00000000..cb49e734 --- /dev/null +++ b/tests/dspaces_perf/unit_test.cpp @@ -0,0 +1,85 @@ +#include +#include +#include +#include + +// TODO add a check so that we don't use experimental/filesystem on fully C++17 compliant compilers +// because they may have removed the experimental header +#include +namespace fs = std::experimental::filesystem; + +/** + * Test data structures + */ +namespace dyad::test { +struct Info { + int rank; + int comm_size; + int num_nodes; + size_t num_server_procs; + uint32_t +}; +struct Arguments { + // MPI Configurations + size_t process_per_node = 1; + // DataSpaces Configuration + size_t server_ppn = 1; + // Test configuration + std::string filename = "test.dat"; + size_t number_of_files = 1; + size_t request_size = 65536; + size_t iteration = 8; + bool debug = false; +}; +} // namespace dyad::test + +dyad::test::Arguments args; +dyad::test::Info info; +/** + * Overridden methods for catch + */ + +int init(int* argc, char*** argv) { + // fprintf(stdout, "Initializing MPI\n"); + MPI_Init(argc, argv); + MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); + MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); + if (args.debug && info.rank == 0) { + printf("%d ready for attach\n", info.comm_size); + fflush(stdout); + getchar(); + } + MPI_Barrier(MPI_COMM_WORLD); + return 0; +} +int finalize() { + MPI_Finalize(); + return 0; +} +cl::Parser define_options() { + return cl::Opt(args.filename, "filename")["-f"]["--filename"]( + "Filename to be use for I/O.") | + cl::Opt(args.process_per_node, + "process_per_node")["-p"]["--ppn"]("Processes per node") | + cl::Opt(args.request_size, "request_size")["-r"]["--request_size"]( + "Transfer size used for performing I/O") | + cl::Opt(args.iteration, + "iteration")["-i"]["--iteration"]("Number of Iterations") | + cl::Opt(args.number_of_files, + "number_of_files")["-n"]["--number_of_files"]("Number of Files") | + cl::Opt(args.server_ppn, + "server_ppn")["-s"]["--server_ppn"]("Number of DataSpaces server processes per node") | + cl::Opt(args.debug, + "debug")["-d"]["--debug"]("debug"); +} + +int pretest() { + info.num_nodes = info.comm_size / args.process_per_node; + info.num_server_procs = info.num_nodes * args.server_ppn; + return 0; +} +int posttest() { + return 0; +} +#include "data_plane/data_plane.cpp" +#include "mdm/mdm.cpp" \ No newline at end of file From 6e588c072f84dd97f0ab8e39fe6351997e31f6f2 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 19 Mar 2024 13:58:01 -0400 Subject: [PATCH 09/47] Updates DataSpaces data plane tests to account for better understanding of DHT --- tests/dspaces_perf/data_plane/data_plane.cpp | 197 ++++++------------- tests/dspaces_perf/unit_test.cpp | 6 - 2 files changed, 58 insertions(+), 145 deletions(-) diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp index 8d60f641..f5aedf36 100644 --- a/tests/dspaces_perf/data_plane/data_plane.cpp +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -3,12 +3,34 @@ #include -int create_files_per_server_process(dspaces_client_t* client, bool use_local) { +void gen_var_name(char* filename, bool is_local, bool add_rank_if_remote, bool next_local_rank, bool next_node) { + size_t node_idx = info.rank / args.process_per_node; + size_t local_rank = info.rank % args.process_per_node; + if (next_local_rank) { + local_rank = (local_rank + 1) % args.process_per_node; + } + if (next_node) { + node_idx = (node_idx + 1) % info.num_nodes; + } + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; + if (is_local) { + size_t global_rank_from_local = node_idx * args.process_per_node + local_rank; + sprintf(filename, "%s_%zu.bat", args.filename.c_str(), global_rank_from_local); + } else { + if (add_rank_if_remote) { + sprintf (filename, "%s_%zu_%zu.bat", args.filename.c_str(), global_server_proc_idx, local_rank); + } else { + sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); + } + } +} + +int create_files_per_server_process(dspaces_client_t* client, bool is_local, bool add_rank) { int rc = 0; char filename[4096]; size_t file_size = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - size_t server_proc_local_idx = info.rank % args.server_ppn; + gen_var_name (filename, is_local, add_rank, false, false); // Clients are connected round-robin to server processes on the same node. // To work out which processes should write "files", we first get a node-local rank for // each client process by modulo dividing the global rank (info.rank) with the processes per node @@ -17,16 +39,14 @@ int create_files_per_server_process(dspaces_client_t* client, bool use_local) { // server processes. Since DataSpaces connects to local processes round-robin, these ranks for which the // floor division equals 0 are the "first" processes to connect to each local server processes. bool first_rank_per_server_proc = (info.rank % args.process_per_node) / args.server_ppn == 0; - if (first_rank_per_server_proc) { + if (is_local || add_rank || first_rank_per_server_proc) { std::string rand_data = GenRandom(file_size); - size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); uint64_t lb = 0; uint64_t ub = rand_data.size()-1; uint64_t buf_size = ub + 1; dspaces_define_gdim(*client, filename, 1, &buf_size); for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { - if (use_local) { + if (is_local) { rc = dspaces_put_local(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); } else { rc = dspaces_put(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); @@ -48,28 +68,24 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, false) == 0); + REQUIRE (create_files_per_server_process(&client, false, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; - size_t neighbor_node_idx = ((info.rank / args.process_per_node) + 1) % info.num_nodes; - size_t server_proc_local_idx = info.rank % args.server_ppn; - uint32_t neighour_server_process_idx = neighbor_node_idx * args.server_ppn + server_proc_local_idx; + gen_var_name(filename, false, true, false, true); size_t data_len = args.request_size*args.iteration; char* file_data = NULL; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), neighbour_server_process_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); + int ndim = 1; uint64_t lb = 0; - uint64_t ub = buf_size - 1; + uint64_t ub = data_len - 1; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); + free(file_data); } AGGREGATE_TIME(data); if (info.rank == 0) { @@ -91,20 +107,16 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, false) == 0); + REQUIRE (create_files_per_broker(&client, false, false) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; - size_t neighbor_node_idx = ((info.rank / args.process_per_node) + 1) % info.num_nodes; - size_t server_proc_local_idx = info.rank % args.server_ppn; - uint32_t neighour_server_process_idx = neighbor_node_idx * args.server_ppn + server_proc_local_idx; + gen_var_name(filename, false, false, false, true); + char* file_data = NULL; size_t data_len = args.request_size*args.iteration; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), neighbour_server_process_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); + int ndim = 1; uint64_t lb = 0; - uint64_t ub = buf_size - 1; + uint64_t ub = data_len - 1; if (info.rank % args.process_per_node != 0) usleep (10000); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { @@ -115,6 +127,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); + free(file_data); } AGGREGATE_TIME(data); if (info.rank == 0) { @@ -128,6 +141,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f REQUIRE (posttest() == 0); } + TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" @@ -136,28 +150,24 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, true) == 0); + REQUIRE (create_files_per_broker(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; + gen_var_name(filename, true, false, false, false); size_t data_len = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - size_t server_proc_local_idx = info.rank % args.server_ppn; - size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); + int ndim = 1; uint64_t lb = 0; - uint64_t ub = buf_size - 1; - char* file_data =(char*)malloc(data_len); + uint64_t ub = data_len - 1; + char* file_data = NULL; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); + free(file_data); } AGGREGATE_TIME(data); if (info.rank == 0) { @@ -171,124 +181,33 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o REQUIRE (posttest() == 0); } -TEST_CASE("LocalServerProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, false) == 0); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096]; - size_t data_len = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - size_t server_proc_local_idx = info.rank % args.server_ppn; - size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); - uint64_t lb = 0; - uint64_t ub = buf_size - 1; - char* file_data =(char*)malloc(data_len); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } - } - rc = dspaces_fini(); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); -} - -TEST_CASE("LocalClientNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, true) == 0); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096]; - size_t data_len = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - size_t server_proc_local_idx = info.rank % args.server_ppn; - size_t read_server_proc_local_idx = (server_proc_local_idx + 1) % args.server_ppn; - size_t global_server_proc_idx = node_idx * args.server_ppn + read_server_proc_local_idx; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); - uint64_t lb = 0; - uint64_t ub = buf_size - 1; - char* file_data =(char*)malloc(data_len); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } - } - rc = dspaces_fini(); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); -} -TEST_CASE("LocalServerNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { +TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" + "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" + "[parallel_req= " + std::to_string(info.comm_size) +"]" + "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { REQUIRE (pretest() == 0); dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, false) == 0); + REQUIRE (create_files_per_broker(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; + gen_var_name(filename, true, false, true, false); size_t data_len = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - size_t server_proc_local_idx = info.rank % args.server_ppn; - size_t read_server_proc_local_idx = (server_proc_local_idx + 1) % args.server_ppn; - size_t global_server_proc_idx = node_idx * args.server_ppn + read_server_proc_local_idx; - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); - int ndim = 0; - uint64_t buf_size = 0; - dspaces_get_gdim(client, filename, &ndim, &buf_size); + int ndim = 1; uint64_t lb = 0; - uint64_t ub = buf_size - 1; - char* file_data =(char*)malloc(data_len); + uint64_t ub = data_len - 1; + char* file_data = NULL; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, 0); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); + free(file_data); } AGGREGATE_TIME(data); if (info.rank == 0) { diff --git a/tests/dspaces_perf/unit_test.cpp b/tests/dspaces_perf/unit_test.cpp index cb49e734..92f06039 100644 --- a/tests/dspaces_perf/unit_test.cpp +++ b/tests/dspaces_perf/unit_test.cpp @@ -3,11 +3,6 @@ #include #include -// TODO add a check so that we don't use experimental/filesystem on fully C++17 compliant compilers -// because they may have removed the experimental header -#include -namespace fs = std::experimental::filesystem; - /** * Test data structures */ @@ -17,7 +12,6 @@ struct Info { int comm_size; int num_nodes; size_t num_server_procs; - uint32_t }; struct Arguments { // MPI Configurations From 780f95e5d23b90d1102cffcc20a91bf182db773c Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 19 Mar 2024 13:58:23 -0400 Subject: [PATCH 10/47] Updates CMake to build DataSpaces test --- tests/dspaces_perf/CMakeLists.txt | 42 ++++++++++-------------- tests/dspaces_perf/script/CMakeLists.txt | 6 ++-- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/tests/dspaces_perf/CMakeLists.txt b/tests/dspaces_perf/CMakeLists.txt index 5d7692e6..137da424 100644 --- a/tests/dspaces_perf/CMakeLists.txt +++ b/tests/dspaces_perf/CMakeLists.txt @@ -1,35 +1,29 @@ -if(NOT DEFINED ENV{DYAD_TEST_MACHINE}) - message(FATAL_ERROR "-- [dyad] DYAD_TEST_MACHINE in env should be set for ${PROJECT_NAME} test build") -else() - message(STATUS "[dyad] found setting machine to $ENV{DYAD_TEST_MACHINE}") -endif() -if(NOT DEFINED ENV{DYAD_PFS_DIR}) - message(FATAL_ERROR "-- [dyad] DYAD_PFS_DIR in env should be set for ${PROJECT_NAME} paper test build") -else() - message(STATUS "[dyad] found setting pfs dir to $ENV{DYAD_PFS_DIR}") -endif() -if(NOT DEFINED ENV{DYAD_DMD_DIR}) - message(FATAL_ERROR "-- [dyad] DYAD_DMD_DIR in env should be set for ${PROJECT_NAME} paper test build") -else() - message(STATUS "[dyad] found setting DMD Dir to $ENV{DYAD_DMD_DIR}") -endif() -set(DYAD_KEYSPACE test_dyad) -set(DYAD_LOG_DIR ${CMAKE_BINARY_DIR}/logs) -file(MAKE_DIRECTORY ${DYAD_LOG_DIR}) +# TODO uncomment and update when adding the "add_test" directives +# if(NOT DEFINED ENV{DYAD_TEST_MACHINE}) +# message(FATAL_ERROR "-- [dyad] DYAD_TEST_MACHINE in env should be set for ${PROJECT_NAME} test build") +# else() +# message(STATUS "[dyad] found setting machine to $ENV{DYAD_TEST_MACHINE}") +# endif() +# if (NOT DEFINED ENV{DSPACES_MAX_VERSIONS}) +# message(FATAL_ERROR "-- [dyad] DSPACES_MAX_VERSIONS in env should be set for ${PROJECT_NAME} paper test build") +# else () +# message(STATUS "[dyad] found setting DataSpaces max versions to $ENV{DSPACES_MAX_VERSIONS}") +# endif () + find_package(Catch2 REQUIRED) find_package(MPI REQUIRED COMPONENTS CXX) +find_package(dspaces REQUIRED CONFIG) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(Catch2::Catch2) include_directories(${MPI_CXX_INCLUDE_DIRS}) -include_directories(${DYAD_PROJECT_DIR}/src) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) include_directories(${CMAKE_BINARY_DIR}/include) -set(TEST_LIBS Catch2::Catch2 -lstdc++fs ${MPI_CXX_LIBRARIES} -rdynamic dyad_core dyad_ctx dyad_utils flux-core ${CPP_LOGGER_LIBRARIES}) +set(TEST_LIBS Catch2::Catch2 ${MPI_CXX_LIBRARIES} -rdynamic dspaces::dspaces) set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.h) add_executable(unit_test unit_test.cpp ${TEST_SRC} ) target_link_libraries(unit_test ${TEST_LIBS}) -add_dependencies(unit_test dyad) -add_subdirectory(script) -add_subdirectory(data_plane) -add_subdirectory(mdm) \ No newline at end of file +# TODO add the "add_test" directives by going into the subdirectories +# add_subdirectory(script) +# add_subdirectory(data_plane) +# add_subdirectory(mdm) \ No newline at end of file diff --git a/tests/dspaces_perf/script/CMakeLists.txt b/tests/dspaces_perf/script/CMakeLists.txt index 85ada8a4..355bb72c 100644 --- a/tests/dspaces_perf/script/CMakeLists.txt +++ b/tests/dspaces_perf/script/CMakeLists.txt @@ -1,9 +1,9 @@ add_test(dspaces_start ${CMAKE_CURRENT_SOURCE_DIR}/dspaces_start.sh) set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_DEFAULT_VAR_SIZE=${DSPACES_DEFAULT_VAR_SIZE}) -set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_MAX_VERSIONS=${DSPACES_MAX_VERSIONS}) -set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_PPN=${DSPACES_PPN}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_MAX_VERSIONS=$ENV{DSPACES_MAX_VERSIONS}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_PPN=$ENV{DSPACES_PPN}) set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_ROOT=${DSPACES_ROOT}) -set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_HG_CONNECTION_STR=${DSPACES_HG_CONNECTION_STR}) +set_property(TEST dspaces_start APPEND PROPERTY ENVIRONMENT DSPACES_HG_CONNECTION_STR=$ENV{DSPACES_HG_CONNECTION_STR}) add_test(dspaces_stop ${CMAKE_CURRENT_SOURCE_DIR}/dspaces_stop.sh) set_property(TEST dspaces_stop APPEND PROPERTY ENVIRONMENT DSPACES_ROOT=${DSPACES_ROOT}) From 0b4e9787c444e88540e5f22ea5f429e51db3719c Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Tue, 19 Mar 2024 11:07:44 -0700 Subject: [PATCH 11/47] Update unit_test.cpp --- tests/unit/unit_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index c9ba53b9..0d910229 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -74,7 +74,7 @@ cl::Parser define_options() { "iteration")["-i"]["--iteration"]("Number of Iterations") | cl::Opt(args.number_of_files, "number_of_files")["-n"]["--number_of_files"]("Number of Files") | - cl::Opt(args.number_of_files, + cl::Opt(args.brokers_per_node, "brokers_per_node")["-b"]["--brokers_per_node"]("Number of Brokers per node") | cl::Opt(args.debug, "debug")["-d"]["--debug"]("debug"); @@ -102,4 +102,4 @@ int clean_directories() { return 0; } #include "data_plane/data_plane.cpp" -#include "mdm/mdm.cpp" \ No newline at end of file +#include "mdm/mdm.cpp" From 9483d21c02634afba1a1ae15cded990771178c0e Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 11:38:25 -0700 Subject: [PATCH 12/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 68 +++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 3819b23d..3af7088d 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -1,11 +1,57 @@ -set(node 1) -set(ppn 1) -set(files 1) -set(test_name unit_remote_data_${node}_${ppn}) -add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteDataBandwidth) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file + +set(files 16) +set(ts 16384) +set(ops 4) +set(ppns 1 2 4 8 16 32 64) + +function(add_dp_test node ppn files ts ops) + # Remote Streaming RPC over RDMA + set(test_name unit_remote_data_${node}_${ppn}) + add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) + # Process Local + set(test_name unit_process_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) + # Node Local + set(test_name unit_node_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) + # Remote Agg + set(test_name unit_remote_agg_data_${node}_${ppn}) + add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +endfunction() + + +foreach (ppn ${ppns}) + add_dp_test(1 ${ppn} ${files} ${ts} ${ops}) +endforeach () +set(nodes 1 2 4 8 16 32 64) +set(ppns 16 32 64) +foreach (node ${nodes}) + foreach (ppn ${ppns}) + add_dp_test(${node} ${ppn} ${files} ${ts} ${ops}) + endforeach () +endforeach () + From b309e3d7264fa098bce44aa0c6ed3aa8e88f1d5a Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 11:44:56 -0700 Subject: [PATCH 13/47] fix compile issue --- src/dyad/core/dyad_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c index 5f7f3893..d3d22b3b 100644 --- a/src/dyad/core/dyad_core.c +++ b/src/dyad/core/dyad_core.c @@ -841,7 +841,7 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, // Set reenter to false to avoid recursively performing DYAD operations ctx->reenter = false; lock_fd = open (fname, O_RDWR | O_CREAT, 0666); - DYAD_C_FUNCTION_UPDATE_INT ("fd", fd); + DYAD_C_FUNCTION_UPDATE_INT ("lock_fd", lock_fd); if (lock_fd == -1) { DYAD_LOG_ERROR (ctx, "Cannot create file (%s) for dyad_consume_w_metadata!\n", fname); rc = DYAD_RC_BADFIO; @@ -854,7 +854,7 @@ dyad_rc_t dyad_consume_w_metadata (dyad_ctx_t* restrict ctx, const char* fname, } if ((file_size = get_file_size (lock_fd)) <= 0) { DYAD_LOG_INFO (ctx, "[node %u rank %u pid %d] File (%s with fd %d) is not fetched yet", \ - ctx->node_idx, ctx->rank, ctx->pid, fname, fd); + ctx->node_idx, ctx->rank, ctx->pid, fname, lock_fd); // Call dyad_get_data to dispatch a RPC to the producer's Flux broker // and retrieve the data associated with the file From 1d5cccb07036d41705b558dbdf5b358f87d5c9f7 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 20 Mar 2024 14:47:59 -0400 Subject: [PATCH 14/47] Adds freopen to DataSpaces tests so that we can get the internal breakdown of metadata vs data times --- tests/CMakeLists.txt | 1 + tests/dspaces_perf/data_plane/data_plane.cpp | 41 ++++++++++++++++++++ tests/dspaces_perf/unit_test.cpp | 6 ++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 059f2a25..952d1106 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1 +1,2 @@ +add_subdirectory(dspaces_perf) add_subdirectory(unit) \ No newline at end of file diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp index f5aedf36..42c4d0e9 100644 --- a/tests/dspaces_perf/data_plane/data_plane.cpp +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -1,8 +1,33 @@ #include #include +#include +#include #include +FILE* redirect_stdout(const char* filename) +{ + size_t dir_len = strlen(args.dspaces_timing_dir); + bool ends_with_separator = (args.dspaces_timing_dir[dir_len-1] == '/'); + size_t filename_len = dir_len + strlen(filename) + 1; + if (!ends_with_separator) { + filename_len += 1; + } + char* full_filename = malloc(filename_len * sizeof(char)); + memset(full_filename, 0, filename_len*sizeof(char)); + strcpy(full_filename, args.dspaces_timing_dir); + if (!ends_with_separator) { + strcat(full_filename, "/"); + } + strcat(full_filename, filename); + return freopen(full_filename, "a", stdout); +} + +int restore_stdout(FILE* freopen_fp) +{ + return fclose(freopen_fp); +} + void gen_var_name(char* filename, bool is_local, bool add_rank_if_remote, bool next_local_rank, bool next_node) { size_t node_idx = info.rank / args.process_per_node; size_t local_rank = info.rank % args.process_per_node; @@ -78,6 +103,9 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file int ndim = 1; uint64_t lb = 0; uint64_t ub = data_len - 1; + FILE* fp = redirect_stdout("remote_data_bandwidth.csv"); + REQUIRE(fp != NULL); + printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer @@ -87,6 +115,7 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file REQUIRE (rc == dspaces_SUCCESS); free(file_data); } + restore_stdout(fp); AGGREGATE_TIME(data); if (info.rank == 0) { printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", @@ -117,6 +146,9 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f int ndim = 1; uint64_t lb = 0; uint64_t ub = data_len - 1; + FILE* fp = redirect_stdout("remote_data_agg_bandwidth.csv"); + REQUIRE(fp != NULL); + printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); if (info.rank % args.process_per_node != 0) usleep (10000); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { @@ -129,6 +161,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f REQUIRE (rc == dspaces_SUCCESS); free(file_data); } + restore_stdout(fp); AGGREGATE_TIME(data); if (info.rank == 0) { printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", @@ -160,6 +193,9 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o uint64_t lb = 0; uint64_t ub = data_len - 1; char* file_data = NULL; + FILE* fp = redirect_stdout("local_process_data_bandwidth.csv"); + REQUIRE(fp != NULL); + printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer @@ -169,6 +205,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o REQUIRE (rc == dspaces_SUCCESS); free(file_data); } + restore_stdout(fp); AGGREGATE_TIME(data); if (info.rank == 0) { printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", @@ -200,6 +237,9 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f uint64_t lb = 0; uint64_t ub = data_len - 1; char* file_data = NULL; + FILE* fp = redirect_stdout("local_node_data_bandwidth.csv"); + REQUIRE(fp != NULL); + printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer @@ -209,6 +249,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f REQUIRE (rc == dspaces_SUCCESS); free(file_data); } + restore_stdout(fp); AGGREGATE_TIME(data); if (info.rank == 0) { printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", diff --git a/tests/dspaces_perf/unit_test.cpp b/tests/dspaces_perf/unit_test.cpp index 92f06039..6970de66 100644 --- a/tests/dspaces_perf/unit_test.cpp +++ b/tests/dspaces_perf/unit_test.cpp @@ -14,6 +14,7 @@ struct Info { size_t num_server_procs; }; struct Arguments { + const char* dspaces_timing_dir; // MPI Configurations size_t process_per_node = 1; // DataSpaces Configuration @@ -63,6 +64,8 @@ cl::Parser define_options() { "number_of_files")["-n"]["--number_of_files"]("Number of Files") | cl::Opt(args.server_ppn, "server_ppn")["-s"]["--server_ppn"]("Number of DataSpaces server processes per node") | + cl::Opt(args.dspaces_timing_dir, + "dspaces_timing_dir")["-t"]["--timing_dir"]("Directory to write DataSpaces internal timings") | cl::Opt(args.debug, "debug")["-d"]["--debug"]("debug"); } @@ -76,4 +79,5 @@ int posttest() { return 0; } #include "data_plane/data_plane.cpp" -#include "mdm/mdm.cpp" \ No newline at end of file +// Temporarily disable mdm tests +// #include "mdm/mdm.cpp" \ No newline at end of file From 9745f7fc4f82f0f82ce2a34113984b36bc88a6f3 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 12:18:49 -0700 Subject: [PATCH 15/47] fixed tests --- scripts/corona.sh | 8 +++++++ scripts/run.sh | 22 +++++++++++++++++ scripts/setup-env.sh | 35 ++++++++++++++++++++++++++++ tests/unit/data_plane/CMakeLists.txt | 8 +++---- 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100755 scripts/corona.sh create mode 100755 scripts/run.sh create mode 100644 scripts/setup-env.sh diff --git a/scripts/corona.sh b/scripts/corona.sh new file mode 100755 index 00000000..2fd8eec2 --- /dev/null +++ b/scripts/corona.sh @@ -0,0 +1,8 @@ +#!/bin/bash +test_case=$1 +NUM_NODES=$2 +PPN=$3 +source ./setup-env.sh ${test_case} $NUM_NODES $PPN +rm *.core flux.log +rm -rf logs/* profiler/* +flux alloc -q $QUEUE -t $TIME -N $NUM_NODES -o per-resource.count=${BROKERS_PER_NODE} --exclusive --broker-opts=--setattr=log-filename=./logs/flux.log ./run.sh $test_case $NUM_NODES $PPN diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 00000000..1b17f4e2 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +test_case=$1 +NUM_NODES=$2 +PPN=$3 +source ./setup-env.sh $test_case $NUM_NODES $PPN + + +pushd ${GITHUB_WORKSPACE}/build +echo Starting DYAD +# Setup DYAD +ctest -R dyad_start -VV + +# Setup Local Directories +echo Setting up local directories +flux run -N $((NUM_NODES*BROKERS_PER_NODE)) --tasks-per-node=1 mkdir -p $DYAD_PATH +flux run -N $((NUM_NODES*BROKERS_PER_NODE)) --tasks-per-node=1 rm -rf $DYAD_PATH/* + +echo "Running Test $WORKLOAD" +ctest -R ${WORKLOAD}$ -VV + +echo Stopping DYAD +ctest -R dyad_stop -VV diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh new file mode 100644 index 00000000..ead3031d --- /dev/null +++ b/scripts/setup-env.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +module load python/3.9.12 +module load openmpi/4.1.2 +test_case=$1 +NUM_NODES=$2 +PPN=$3 +# Configurations +export WORKLOAD=${test_case}_${NUM_NODES}_${PPN} +export QUEUE=pdebug +export TIME=$((60)) +export BROKERS_PER_NODE=1 + +export DYAD_INSTALL_PREFIX=/usr/workspace/haridev/dyad/env/spack/.spack-env/view +export DYAD_KVS_NAMESPACE=dyad +export DYAD_DTL_MODE=UCX +export DYAD_PATH="/l/ssd/haridev/dyad" +#export DYAD_PATH=/dev/shm/haridev/dyad +export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad +export SPACK_DIR=/usr/workspace/haridev/spack +export SPACK_ENV=/usr/workspace/haridev/dyad/env/spack +export PYTHON_ENV=/usr/workspace/haridev/dyad/env/python + +# Activate Environments +. ${SPACK_DIR}/share/spack/setup-env.sh +spack env activate -p ${SPACK_ENV} +source ${PYTHON_ENV}/bin/activate + +# Derived PATHS +export PATH=${PATH}:${DYAD_INSTALL_PREFIX}/bin:${DYAD_INSTALL_PREFIX}/sbin +export LD_LIBRARY_PATH=/usr/lib64:${DYAD_INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH} +export PYTHONPATH=${GITHUB_WORKSPACE}/tests/integration/dlio_benchmark:${GITHUB_WORKSPACE}/pydyad:$PYTHONPATH + +unset LUA_PATH +unset LUA_CPATH diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 3af7088d..259bd067 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -16,7 +16,7 @@ function(add_dp_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Process Local set(test_name unit_process_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -25,7 +25,7 @@ function(add_dp_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Node Local set(test_name unit_node_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -34,7 +34,7 @@ function(add_dp_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Remote Agg set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N $node --tasks-per-node $ppn ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) From c8117ef306a9225fee3e7ee037d779c652415b30 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 12:20:53 -0700 Subject: [PATCH 16/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 35 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 259bd067..5cbc4686 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -4,7 +4,7 @@ set(ts 16384) set(ops 4) set(ppns 1 2 4 8 16 32 64) -function(add_dp_test node ppn files ts ops) +function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) @@ -14,27 +14,30 @@ function(add_dp_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Process Local - set(test_name unit_process_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + # Remote Agg + set(test_name unit_remote_agg_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Node Local - set(test_name unit_node_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) +endfunction() + +function(add_dp_local_test node ppn files ts ops) + # Process Local + set(test_name unit_process_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Remote Agg - set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + # Node Local + set(test_name unit_node_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -45,13 +48,19 @@ endfunction() foreach (ppn ${ppns}) - add_dp_test(1 ${ppn} ${files} ${ts} ${ops}) + add_dp_remote_test(2 ${ppn} ${files} ${ts} ${ops}) + add_dp_local_test(1 ${ppn} ${files} ${ts} ${ops}) endforeach () -set(nodes 1 2 4 8 16 32 64) +set(nodes 2 4 8 16 32 64) set(ppns 16 32 64) +foreach (node ${nodes}) + foreach (ppn ${ppns}) + add_dp_local_test(${node} ${ppn} ${files} ${ts} ${ops}) + endforeach () +endforeach () +set(nodes 4 8 16 32 64) foreach (node ${nodes}) foreach (ppn ${ppns}) add_dp_test(${node} ${ppn} ${files} ${ts} ${ops}) endforeach () endforeach () - From b7fba320b445ca3cebb2d6ece02900d3b85b3902 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 12:20:53 -0700 Subject: [PATCH 17/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 37 +++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 259bd067..26ed2b05 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -4,7 +4,7 @@ set(ts 16384) set(ops 4) set(ppns 1 2 4 8 16 32 64) -function(add_dp_test node ppn files ts ops) +function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) @@ -14,27 +14,30 @@ function(add_dp_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Process Local - set(test_name unit_process_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + # Remote Agg + set(test_name unit_remote_agg_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Node Local - set(test_name unit_node_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) +endfunction() + +function(add_dp_local_test node ppn files ts ops) + # Process Local + set(test_name unit_process_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) - # Remote Agg - set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + # Node Local + set(test_name unit_node_local_data_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -45,13 +48,19 @@ endfunction() foreach (ppn ${ppns}) - add_dp_test(1 ${ppn} ${files} ${ts} ${ops}) + add_dp_remote_test(2 ${ppn} ${files} ${ts} ${ops}) + add_dp_local_test(1 ${ppn} ${files} ${ts} ${ops}) endforeach () -set(nodes 1 2 4 8 16 32 64) +set(nodes 2 4 8 16 32 64) set(ppns 16 32 64) foreach (node ${nodes}) foreach (ppn ${ppns}) - add_dp_test(${node} ${ppn} ${files} ${ts} ${ops}) + add_dp_local_test(${node} ${ppn} ${files} ${ts} ${ops}) + endforeach () +endforeach () +set(nodes 4 8 16 32 64) +foreach (node ${nodes}) + foreach (ppn ${ppns}) + add_dp_remote_test(${node} ${ppn} ${files} ${ts} ${ops}) endforeach () endforeach () - From c02892a7398001a6952385e6744b5ad34e1a47d2 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 12:33:00 -0700 Subject: [PATCH 18/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 26ed2b05..23b8549a 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -16,7 +16,7 @@ function(add_dp_remote_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Remote Agg set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -28,7 +28,7 @@ endfunction() function(add_dp_local_test node ppn files ts ops) # Process Local set(test_name unit_process_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -37,7 +37,7 @@ function(add_dp_local_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Node Local set(test_name unit_node_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) From 3e092bde466864a8b1344587c3bdfdcceaa911a1 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 12:35:13 -0700 Subject: [PATCH 19/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 23b8549a..33e68643 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -16,7 +16,7 @@ function(add_dp_remote_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Remote Agg set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -28,7 +28,7 @@ endfunction() function(add_dp_local_test node ppn files ts ops) # Process Local set(test_name unit_process_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalProcessDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -37,7 +37,7 @@ function(add_dp_local_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Node Local set(test_name unit_node_local_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --process_per_node ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalNodeDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) From 0038eafe5c68003213ad947b7a2b3dea21e0e23c Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 12:42:11 -0700 Subject: [PATCH 20/47] add test cases for reproducability --- tests/unit/data_plane/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 33e68643..eb78d129 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -1,7 +1,7 @@ set(files 16) -set(ts 16384) -set(ops 4) +set(ts 65536) +set(ops 1) set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) From 5c2e17f5091d69109afa3fe7476ba853987e1b6a Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 12:54:38 -0700 Subject: [PATCH 21/47] fine tune parameters for reproducability --- tests/unit/data_plane/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index eb78d129..b57e2e4e 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -1,7 +1,7 @@ set(files 16) set(ts 65536) -set(ops 1) +set(ops 16) set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) From 77bf89b6bda8e48e5b9d3faf1104e5e97d10ba0f Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 13:06:05 -0700 Subject: [PATCH 22/47] add test cases for reproducability --- tests/unit/mdm/CMakeLists.txt | 66 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/tests/unit/mdm/CMakeLists.txt b/tests/unit/mdm/CMakeLists.txt index 3717411e..696d2cd4 100644 --- a/tests/unit/mdm/CMakeLists.txt +++ b/tests/unit/mdm/CMakeLists.txt @@ -1,29 +1,37 @@ -set(node 1) -set(ppn 1) -set(files 1) -set(test_name unit_localfs_${node}_${ppn}) -set(mpiexec flux run -N ${node} --tasks-per-node ${ppn}) -set(mpiexec ) -add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalFSLookup) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) -set(test_name unit_localkvs_${node}_${ppn}) -add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact LocalKVSLookup) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) -set(test_name unit_remotekvs_${node}_${ppn}) -add_test(${test_name} ${mpiexec} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration 10 --number_of_files ${files} --reporter compact RemoteKVSLookup) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) -set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) \ No newline at end of file +set(files 16) +set(ts 65536) +set(ops 16) + +function(add_mdm_test node ppn files ts ops) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalFSLookup) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) + set(test_name unit_localkvs_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalKVSLookup) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) + set(test_name unit_remotekvs_${node}_${ppn}) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteKVSLookup) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_DTL_MODE=UCX) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) + set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) +endfunction() + +set(ppns 64) +set(nodes 1 2 4 8 16 32 64) +foreach (node ${nodes}) + foreach (ppn ${ppns}) + add_mdm_test(${node} ${ppn} ${files} ${ts} ${ops}) + endforeach () +endforeach () \ No newline at end of file From 003517060c1b5626841fb370c479ddf71410b99b Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 13:17:19 -0700 Subject: [PATCH 23/47] fixed tests --- tests/unit/mdm/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/mdm/CMakeLists.txt b/tests/unit/mdm/CMakeLists.txt index 696d2cd4..424374ec 100644 --- a/tests/unit/mdm/CMakeLists.txt +++ b/tests/unit/mdm/CMakeLists.txt @@ -3,6 +3,7 @@ set(ts 65536) set(ops 16) function(add_mdm_test node ppn files ts ops) + set(test_name unit_localfs_${node}_${ppn}) add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalFSLookup) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) @@ -34,4 +35,4 @@ foreach (node ${nodes}) foreach (ppn ${ppns}) add_mdm_test(${node} ${ppn} ${files} ${ts} ${ops}) endforeach () -endforeach () \ No newline at end of file +endforeach () From c59ff79c984b004e20894ded4c6b2cc1a1f0b8e7 Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 13:17:30 -0700 Subject: [PATCH 24/47] add test cases for reproducability --- tests/unit/unit_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 0d910229..1203f100 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -91,11 +91,11 @@ int posttest() { return 0; } int clean_directories() { - auto file_pt = args.pfs.string() + args.filename; + auto file_pt = args.pfs.string() + "/" + args.filename; std::string cmd = "rm -rf " + file_pt + "*"; int status = system (cmd.c_str ()); (void) status; - file_pt = args.dyad_managed_dir.string() + args.filename; + file_pt = args.dyad_managed_dir.string() + "/" + args.filename; cmd = "rm -rf " + file_pt + "*"; status = system (cmd.c_str ()); (void) status; From 29a8ba14d8932a9b75e6bb895572a4748622f16d Mon Sep 17 00:00:00 2001 From: Hariharan Devarajan Date: Wed, 20 Mar 2024 13:20:18 -0700 Subject: [PATCH 25/47] add test cases for reproducability --- tests/unit/unit_test.cpp | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 1203f100..9086ad2e 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -85,20 +85,25 @@ int pretest() { info.num_brokers = info.num_nodes * args.brokers_per_node; flux_get_rank (info.flux_handle, &info.broker_idx); flux_get_size (info.flux_handle, &info.broker_size); + MPI_Barrier(MPI_COMM_WORLD); return 0; } int posttest() { + MPI_Barrier(MPI_COMM_WORLD); return 0; } int clean_directories() { - auto file_pt = args.pfs.string() + "/" + args.filename; - std::string cmd = "rm -rf " + file_pt + "*"; - int status = system (cmd.c_str ()); - (void) status; - file_pt = args.dyad_managed_dir.string() + "/" + args.filename; - cmd = "rm -rf " + file_pt + "*"; - status = system (cmd.c_str ()); - (void) status; + if (info.rank % args.process_per_node == 0) { + auto file_pt = args.pfs.string() + "/" + args.filename; + std::string cmd = "rm -rf " + file_pt + "*"; + int status = system (cmd.c_str ()); + (void) status; + file_pt = args.dyad_managed_dir.string() + "/" + args.filename; + cmd = "rm -rf " + file_pt + "*"; + status = system (cmd.c_str ()); + (void) status; + } + MPI_Barrier(MPI_COMM_WORLD); return 0; } #include "data_plane/data_plane.cpp" From 212b56a2f4162c3d70e1e5dd753c74973cde261d Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 14:21:51 -0700 Subject: [PATCH 26/47] fixed mdm --- scripts/setup-env.sh | 2 +- tests/unit/mdm/mdm.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index ead3031d..acb05520 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -14,7 +14,7 @@ export BROKERS_PER_NODE=1 export DYAD_INSTALL_PREFIX=/usr/workspace/haridev/dyad/env/spack/.spack-env/view export DYAD_KVS_NAMESPACE=dyad export DYAD_DTL_MODE=UCX -export DYAD_PATH="/l/ssd/haridev/dyad" +export DYAD_PATH=/l/ssd/haridev/dyad/internal #export DYAD_PATH=/dev/shm/haridev/dyad export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad export SPACK_DIR=/usr/workspace/haridev/spack diff --git a/tests/unit/mdm/mdm.cpp b/tests/unit/mdm/mdm.cpp index bad7ea6c..aa7b07ee 100644 --- a/tests/unit/mdm/mdm.cpp +++ b/tests/unit/mdm/mdm.cpp @@ -22,7 +22,8 @@ TEST_CASE("LocalFSLookup", "[number_of_lookups= " + std::to_string(args.number_ args.filename.c_str (), info.broker_idx, file_idx); - kvs_time.resumeTime(); + INFO("The file " << filename << " rank " << info.rank); + kvs_time.resumeTime(); int lock_fd = open (filename, O_RDWR | O_CREAT, 0666); kvs_time.pauseTime(); REQUIRE (lock_fd != -1); @@ -149,4 +150,4 @@ TEST_CASE("RemoteKVSLookup", "[number_of_lookups= " + std::to_string(args.numbe rc = dyad_finalize(); REQUIRE (rc >= 0); REQUIRE (posttest() == 0); -} \ No newline at end of file +} From 12ae9589d0370b90d17470ad9ea1b99400fd4503 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 20 Mar 2024 23:12:21 -0700 Subject: [PATCH 27/47] Changes to make Catch MPI friendly --- .gitignore | 17 +- dyad_mod_0.err | 14 + tests/CMakeLists.txt | 4 +- tests/unit/CMakeLists.txt | 2 +- tests/unit/catch_config.h | 59 +++ tests/unit/data_plane/CMakeLists.txt | 4 +- tests/unit/data_plane/data_plane.cpp | 5 +- tests/unit/mdm/CMakeLists.txt | 6 +- tests/unit/mpi_console_reporter.cpp | 720 +++++++++++++++++++++++++++ tests/unit/mpi_console_reporter.hpp | 73 +++ 10 files changed, 891 insertions(+), 13 deletions(-) create mode 100644 dyad_mod_0.err create mode 100644 tests/unit/mpi_console_reporter.cpp create mode 100644 tests/unit/mpi_console_reporter.hpp diff --git a/.gitignore b/.gitignore index 55f3c713..12050b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -97,9 +97,10 @@ compile_flags.txt # local editor config dirs .vscode -.idea +.idea* .clangd - +.cache +cmake-build-debug # ignore installable version of dyadrun dyadrun flux_barrier @@ -112,7 +113,17 @@ flux_barrier # Python stuff **/__pycache__/ -**/build +**/build* **/*.egg-info /install/ /dyad-env/ +env +hydra_log +docs/demos/ecp_feb_2023/c_cons +docs/demos/ecp_feb_2023/cpp_cons +docs/demos/ecp_feb_2023/c_prod +docs/demos/ecp_feb_2023/cpp_prod +tests/integration/dlio_benchmark/logs +scripts/checkpoints +scripts/logs +tests/integration/dlio_benchmark/perf_analysis/.ipynb_checkpoints diff --git a/dyad_mod_0.err b/dyad_mod_0.err new file mode 100644 index 00000000..39f83ede --- /dev/null +++ b/dyad_mod_0.err @@ -0,0 +1,14 @@ +DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal +DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: Did not find DTL 'mode' option. Using env DYAD_DTL_MODE=(null)DYAD_DTL_MODE is not set. Defaulting to FLUX_RPC +[DYAD ERROR]: No KVS namespace provided! + +DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal +DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: Did not find DTL 'mode' option. Using env DYAD_DTL_MODE=(null)DYAD_DTL_MODE is not set. Defaulting to FLUX_RPC +[DYAD ERROR]: No KVS namespace provided! + +DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal +DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: DTL 'mode' option set. Setting env DYAD_DTL_MODE=UCX[DYAD ERROR]: No KVS namespace provided! + +DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal +DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: DTL 'mode' option set. Setting env DYAD_DTL_MODE=UCX[DYAD ERROR]: No KVS namespace provided! + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 952d1106..da212291 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,2 +1,2 @@ -add_subdirectory(dspaces_perf) -add_subdirectory(unit) \ No newline at end of file +#add_subdirectory(dspaces_perf) +add_subdirectory(unit) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 5d7692e6..4095c330 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -25,7 +25,7 @@ include_directories(${DYAD_PROJECT_DIR}/src) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) include_directories(${CMAKE_BINARY_DIR}/include) set(TEST_LIBS Catch2::Catch2 -lstdc++fs ${MPI_CXX_LIBRARIES} -rdynamic dyad_core dyad_ctx dyad_utils flux-core ${CPP_LOGGER_LIBRARIES}) -set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.h) +set(TEST_SRC ${CMAKE_CURRENT_SOURCE_DIR}/catch_config.h ${CMAKE_CURRENT_SOURCE_DIR}/mpi_console_reporter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/mpi_console_reporter.hpp ${CMAKE_CURRENT_SOURCE_DIR}/test_utils.h) add_executable(unit_test unit_test.cpp ${TEST_SRC} ) target_link_libraries(unit_test ${TEST_LIBS}) add_dependencies(unit_test dyad) diff --git a/tests/unit/catch_config.h b/tests/unit/catch_config.h index 58e771ce..987b6d28 100644 --- a/tests/unit/catch_config.h +++ b/tests/unit/catch_config.h @@ -5,6 +5,8 @@ #ifndef DYAD_CATCH_CONFIG_H #define DYAD_CATCH_CONFIG_H #include +#include "mpi.h" +#include namespace cl = Catch::Clara; cl::Parser define_options(); @@ -25,4 +27,61 @@ int main(int argc, char* argv[]) { if (returnCode != 0) return returnCode; exit(test_return_code); } + +#include +#include + +namespace Catch { + // Fwd decls + class TablePrinter; + + class MPIConsoleReporter final : public StreamingReporterBase { + Detail::unique_ptr m_tablePrinter; + + public: + MPIConsoleReporter(ReporterConfig&& config); + ~MPIConsoleReporter() override; + static std::string getDescription(); + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + void reportInvalidTestSpec( StringRef arg ) override; + + void assertionStarting(AssertionInfo const&) override; + + void assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionStarting(SectionInfo const& _sectionInfo) override; + void sectionEnded(SectionStats const& _sectionStats) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting(BenchmarkInfo const& info) override; + void benchmarkEnded(BenchmarkStats<> const& stats) override; + void benchmarkFailed( StringRef error ) override; + + void testCaseEnded(TestCaseStats const& _testCaseStats) override; + void testRunEnded(TestRunStats const& _testRunStats) override; + void testRunStarting(TestRunInfo const& _testRunInfo) override; + + private: + void lazyPrint(); + + void lazyPrintWithoutClosingBenchmarkTable(); + void lazyPrintRunInfo(); + void printTestCaseAndSectionHeader(); + + void printClosedHeader(std::string const& _name); + void printOpenHeader(std::string const& _name); + + // if string has a : in first line will set indent to follow it on + // subsequent lines + void printHeaderString(std::string const& _string, std::size_t indent = 0); + + void printTotalsDivider(Totals const& totals); + + bool m_headerPrinted = false; + bool m_testRunInfoPrinted = false; + }; + +} // end namespace Catch + #endif // DYAD_CATCH_CONFIG_H diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index b57e2e4e..3450d497 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -16,7 +16,7 @@ function(add_dp_remote_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) # Remote Agg set(test_name unit_remote_agg_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteDataAggBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console RemoteDataAggBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) diff --git a/tests/unit/data_plane/data_plane.cpp b/tests/unit/data_plane/data_plane.cpp index a4609c48..02ec2dc9 100644 --- a/tests/unit/data_plane/data_plane.cpp +++ b/tests/unit/data_plane/data_plane.cpp @@ -2,6 +2,7 @@ #include #include #include +#include int create_files_per_broker() { char filename[4096], first_file[4096]; @@ -142,7 +143,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o data_time.resumeTime(); int bytes = read (fd,file_data, data_len); data_time.pauseTime(); - REQUIRE (bytes == data_len); + REQUIRE ((size_t)bytes == data_len); data_time.resumeTime(); int status = close(fd); data_time.pauseTime(); @@ -184,7 +185,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f data_time.resumeTime(); int bytes = read (fd,file_data, data_len); data_time.pauseTime(); - REQUIRE (bytes == data_len); + REQUIRE ((size_t)bytes == data_len); data_time.resumeTime(); int status = close(fd); data_time.pauseTime(); diff --git a/tests/unit/mdm/CMakeLists.txt b/tests/unit/mdm/CMakeLists.txt index 424374ec..3e25fe90 100644 --- a/tests/unit/mdm/CMakeLists.txt +++ b/tests/unit/mdm/CMakeLists.txt @@ -4,7 +4,7 @@ set(ops 16) function(add_mdm_test node ppn files ts ops) set(test_name unit_localfs_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalFSLookup) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console LocalFSLookup) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -12,7 +12,7 @@ function(add_mdm_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) set(test_name unit_localkvs_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact LocalKVSLookup) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console LocalKVSLookup) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) @@ -20,7 +20,7 @@ function(add_mdm_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_CONSUMER=$ENV{DYAD_DMD_DIR}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) set(test_name unit_remotekvs_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter compact RemoteKVSLookup) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename mdm_${node}_${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --ppn ${ppn} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console RemoteKVSLookup) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) diff --git a/tests/unit/mpi_console_reporter.cpp b/tests/unit/mpi_console_reporter.cpp new file mode 100644 index 00000000..1cbbd186 --- /dev/null +++ b/tests/unit/mpi_console_reporter.cpp @@ -0,0 +1,720 @@ +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#include "mpi_console_reporter.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch + // Note that 4062 (not all labels are handled and default is missing) is enabled +#endif + +#if defined(__clang__) +# pragma clang diagnostic push +// For simplicity, benchmarking-only helpers are always enabled +# pragma clang diagnostic ignored "-Wunused-function" +#endif + + + +namespace Catch { + +namespace { + +// Formatter impl for ConsoleMPIReporter +class ConsoleAssertionPrinter { +public: + ConsoleAssertionPrinter& operator= (ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(ConsoleAssertionPrinter const&) = delete; + ConsoleAssertionPrinter(std::ostream& _stream, AssertionStats const& _stats, ColourImpl* colourImpl_, bool _printInfoMessages) + : stream(_stream), + stats(_stats), + result(_stats.assertionResult), + colour(Colour::None), + message(result.getMessage()), + messages(_stats.infoMessages), + colourImpl(colourImpl_), + printInfoMessages(_printInfoMessages) { + switch (result.getResultType()) { + case ResultWas::Ok: + colour = Colour::Success; + passOrFail = "PASSED"_sr; + //if( result.hasMessage() ) + if (_stats.infoMessages.size() == 1) + messageLabel = "with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "with messages"; + break; + case ResultWas::ExpressionFailed: + if (result.isOk()) { + colour = Colour::Success; + passOrFail = "FAILED - but was ok"_sr; + } else { + colour = Colour::Error; + passOrFail = "FAILED"_sr; + } + if (_stats.infoMessages.size() == 1) + messageLabel = "with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "with messages"; + break; + case ResultWas::ThrewException: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + messageLabel = "due to unexpected exception with "; + if (_stats.infoMessages.size() == 1) + messageLabel += "message"; + if (_stats.infoMessages.size() > 1) + messageLabel += "messages"; + break; + case ResultWas::FatalErrorCondition: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + messageLabel = "due to a fatal error condition"; + break; + case ResultWas::DidntThrowException: + colour = Colour::Error; + passOrFail = "FAILED"_sr; + messageLabel = "because no exception was thrown where one was expected"; + break; + case ResultWas::Info: + messageLabel = "info"; + break; + case ResultWas::Warning: + messageLabel = "warning"; + break; + case ResultWas::ExplicitFailure: + passOrFail = "FAILED"_sr; + colour = Colour::Error; + if (_stats.infoMessages.size() == 1) + messageLabel = "explicitly with message"; + if (_stats.infoMessages.size() > 1) + messageLabel = "explicitly with messages"; + break; + // These cases are here to prevent compiler warnings + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + passOrFail = "** internal error **"_sr; + colour = Colour::Error; + break; + } + } + + void print() const { + printSourceInfo(); + if (stats.totals.assertions.total() > 0) { + printResultType(); + printOriginalExpression(); + printReconstructedExpression(); + } else { + stream << '\n'; + } + printMessage(); + } + +private: + void printResultType() const { + if (!passOrFail.empty()) { + stream << colourImpl->guardColour(colour) << passOrFail << ":\n"; + } + } + void printOriginalExpression() const { + if (result.hasExpression()) { + stream << colourImpl->guardColour( Colour::OriginalExpression ) + << " " << result.getExpressionInMacro() << '\n'; + } + } + void printReconstructedExpression() const { + if (result.hasExpandedExpression()) { + stream << "with expansion:\n"; + stream << colourImpl->guardColour( Colour::ReconstructedExpression ) + << TextFlow::Column( result.getExpandedExpression() ) + .indent( 2 ) + << '\n'; + } + } + void printMessage() const { + if (!messageLabel.empty()) + stream << messageLabel << ':' << '\n'; + for (auto const& msg : messages) { + // If this assertion is a warning ignore any INFO messages + if (printInfoMessages || msg.type != ResultWas::Info) + stream << TextFlow::Column(msg.message).indent(2) << '\n'; + } + } + void printSourceInfo() const { + stream << colourImpl->guardColour( Colour::FileName ) + << result.getSourceInfo() << ": "; + } + + std::ostream& stream; + AssertionStats const& stats; + AssertionResult const& result; + Colour::Code colour; + StringRef passOrFail; + std::string messageLabel; + std::string message; + std::vector messages; + ColourImpl* colourImpl; + bool printInfoMessages; +}; + +std::size_t makeRatio( std::uint64_t number, std::uint64_t total ) { + const auto ratio = total > 0 ? CATCH_CONFIG_CONSOLE_WIDTH * number / total : 0; + return (ratio == 0 && number > 0) ? 1 : static_cast(ratio); +} + +std::size_t& findMax( std::size_t& i, std::size_t& j, std::size_t& k ) { + if (i > j && i > k) + return i; + else if (j > k) + return j; + else + return k; +} + +enum class Justification { Left, Right }; + +struct ColumnInfo { + std::string name; + std::size_t width; + Justification justification; +}; +struct ColumnBreak {}; +struct RowBreak {}; + +class Duration { + enum class Unit { + Auto, + Nanoseconds, + Microseconds, + Milliseconds, + Seconds, + Minutes + }; + static const uint64_t s_nanosecondsInAMicrosecond = 1000; + static const uint64_t s_nanosecondsInAMillisecond = 1000 * s_nanosecondsInAMicrosecond; + static const uint64_t s_nanosecondsInASecond = 1000 * s_nanosecondsInAMillisecond; + static const uint64_t s_nanosecondsInAMinute = 60 * s_nanosecondsInASecond; + + double m_inNanoseconds; + Unit m_units; + +public: + explicit Duration(double inNanoseconds, Unit units = Unit::Auto) + : m_inNanoseconds(inNanoseconds), + m_units(units) { + if (m_units == Unit::Auto) { + if (m_inNanoseconds < s_nanosecondsInAMicrosecond) + m_units = Unit::Nanoseconds; + else if (m_inNanoseconds < s_nanosecondsInAMillisecond) + m_units = Unit::Microseconds; + else if (m_inNanoseconds < s_nanosecondsInASecond) + m_units = Unit::Milliseconds; + else if (m_inNanoseconds < s_nanosecondsInAMinute) + m_units = Unit::Seconds; + else + m_units = Unit::Minutes; + } + + } + + auto value() const -> double { + switch (m_units) { + case Unit::Microseconds: + return m_inNanoseconds / static_cast(s_nanosecondsInAMicrosecond); + case Unit::Milliseconds: + return m_inNanoseconds / static_cast(s_nanosecondsInAMillisecond); + case Unit::Seconds: + return m_inNanoseconds / static_cast(s_nanosecondsInASecond); + case Unit::Minutes: + return m_inNanoseconds / static_cast(s_nanosecondsInAMinute); + default: + return m_inNanoseconds; + } + } + StringRef unitsAsString() const { + switch (m_units) { + case Unit::Nanoseconds: + return "ns"_sr; + case Unit::Microseconds: + return "us"_sr; + case Unit::Milliseconds: + return "ms"_sr; + case Unit::Seconds: + return "s"_sr; + case Unit::Minutes: + return "m"_sr; + default: + return "** internal error **"_sr; + } + + } + friend auto operator << (std::ostream& os, Duration const& duration) -> std::ostream& { + return os << duration.value() << ' ' << duration.unitsAsString(); + } +}; +} // end anon namespace + +class TablePrinter { + std::ostream& m_os; + std::vector m_columnInfos; + ReusableStringStream m_oss; + int m_currentColumn = -1; + bool m_isOpen = false; + +public: + TablePrinter( std::ostream& os, std::vector columnInfos ) + : m_os( os ), + m_columnInfos( CATCH_MOVE( columnInfos ) ) {} + + auto columnInfos() const -> std::vector const& { + return m_columnInfos; + } + + void open() { + if (!m_isOpen) { + m_isOpen = true; + *this << RowBreak(); + + TextFlow::Columns headerCols; + auto spacer = TextFlow::Spacer(2); + for (auto const& info : m_columnInfos) { + assert(info.width > 2); + headerCols += TextFlow::Column(info.name).width(info.width - 2); + headerCols += spacer; + } + m_os << headerCols << '\n'; + + m_os << lineOfChars('-') << '\n'; + } + } + void close() { + if (m_isOpen) { + *this << RowBreak(); + m_os << '\n' << std::flush; + m_isOpen = false; + } + } + + template + friend TablePrinter& operator << (TablePrinter& tp, T const& value) { + tp.m_oss << value; + return tp; + } + + friend TablePrinter& operator << (TablePrinter& tp, ColumnBreak) { + auto colStr = tp.m_oss.str(); + const auto strSize = colStr.size(); + tp.m_oss.str(""); + tp.open(); + if (tp.m_currentColumn == static_cast(tp.m_columnInfos.size() - 1)) { + tp.m_currentColumn = -1; + tp.m_os << '\n'; + } + tp.m_currentColumn++; + + auto colInfo = tp.m_columnInfos[tp.m_currentColumn]; + auto padding = (strSize + 1 < colInfo.width) + ? std::string(colInfo.width - (strSize + 1), ' ') + : std::string(); + if (colInfo.justification == Justification::Left) + tp.m_os << colStr << padding << ' '; + else + tp.m_os << padding << colStr << ' '; + return tp; + } + + friend TablePrinter& operator << (TablePrinter& tp, RowBreak) { + if (tp.m_currentColumn > 0) { + tp.m_os << '\n'; + tp.m_currentColumn = -1; + } + return tp; + } +}; + +ConsoleMPIReporter::ConsoleMPIReporter(ReporterConfig&& config): + StreamingReporterBase( CATCH_MOVE( config ) ), + m_tablePrinter(Detail::make_unique(m_stream, + [&config]() -> std::vector { + if (config.fullConfig()->benchmarkNoAnalysis()) + { + return{ + { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 43, Justification::Left }, + { " samples", 14, Justification::Right }, + { " iterations", 14, Justification::Right }, + { " mean", 14, Justification::Right } + }; + } + else + { + return{ + { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 43, Justification::Left }, + { "samples mean std dev", 14, Justification::Right }, + { "iterations low mean low std dev", 14, Justification::Right }, + { "estimated high mean high std dev", 14, Justification::Right } + }; + } + }())) {} +ConsoleMPIReporter::~ConsoleMPIReporter() = default; + +std::string ConsoleMPIReporter::getDescription() { + return "Reports test results as plain lines of text"; +} + +void ConsoleMPIReporter::noMatchingTestCases( StringRef unmatchedSpec ) { + m_stream << "No test cases matched '" << unmatchedSpec << "'\n"; +} + +void ConsoleMPIReporter::reportInvalidTestSpec( StringRef arg ) { + m_stream << "Invalid Filter: " << arg << '\n'; +} + +void ConsoleMPIReporter::assertionStarting(AssertionInfo const&) {} + +void ConsoleMPIReporter::assertionEnded(AssertionStats const& _assertionStats) { + AssertionResult const& result = _assertionStats.assertionResult; + + bool includeResults = m_config->includeSuccessfulResults() || !result.isOk(); + + // Drop out if result was successful but we're not printing them. + if (!includeResults && result.getResultType() != ResultWas::Warning) + return; + + lazyPrint(); + + ConsoleAssertionPrinter printer(m_stream, _assertionStats, m_colour.get(), includeResults); + printer.print(); + m_stream << '\n' << std::flush; +} + +void ConsoleMPIReporter::sectionStarting(SectionInfo const& _sectionInfo) { + m_tablePrinter->close(); + m_headerPrinted = false; + StreamingReporterBase::sectionStarting(_sectionInfo); +} +void ConsoleMPIReporter::sectionEnded(SectionStats const& _sectionStats) { + m_tablePrinter->close(); + if (_sectionStats.missingAssertions) { + lazyPrint(); + auto guard = + m_colour->guardColour( Colour::ResultError ).engage( m_stream ); + if (m_sectionStack.size() > 1) + m_stream << "\nNo assertions in section"; + else + m_stream << "\nNo assertions in test case"; + m_stream << " '" << _sectionStats.sectionInfo.name << "'\n\n" << std::flush; + } + double dur = _sectionStats.durationInSeconds; + if (shouldShowDuration(*m_config, dur)) { + m_stream << getFormattedDuration(dur) << " s: " << _sectionStats.sectionInfo.name << '\n' << std::flush; + } + if (m_headerPrinted) { + m_headerPrinted = false; + } + StreamingReporterBase::sectionEnded(_sectionStats); +} + +void ConsoleMPIReporter::benchmarkPreparing( StringRef name ) { + lazyPrintWithoutClosingBenchmarkTable(); + + auto nameCol = TextFlow::Column( static_cast( name ) ) + .width( m_tablePrinter->columnInfos()[0].width - 2 ); + + bool firstLine = true; + for (auto line : nameCol) { + if (!firstLine) + (*m_tablePrinter) << ColumnBreak() << ColumnBreak() << ColumnBreak(); + else + firstLine = false; + + (*m_tablePrinter) << line << ColumnBreak(); + } +} + +void ConsoleMPIReporter::benchmarkStarting(BenchmarkInfo const& info) { + (*m_tablePrinter) << info.samples << ColumnBreak() + << info.iterations << ColumnBreak(); + if (!m_config->benchmarkNoAnalysis()) + (*m_tablePrinter) << Duration(info.estimatedDuration) << ColumnBreak(); +} +void ConsoleMPIReporter::benchmarkEnded(BenchmarkStats<> const& stats) { + if (m_config->benchmarkNoAnalysis()) + { + (*m_tablePrinter) << Duration(stats.mean.point.count()) << ColumnBreak(); + } + else + { + (*m_tablePrinter) << ColumnBreak() + << Duration(stats.mean.point.count()) << ColumnBreak() + << Duration(stats.mean.lower_bound.count()) << ColumnBreak() + << Duration(stats.mean.upper_bound.count()) << ColumnBreak() << ColumnBreak() + << Duration(stats.standardDeviation.point.count()) << ColumnBreak() + << Duration(stats.standardDeviation.lower_bound.count()) << ColumnBreak() + << Duration(stats.standardDeviation.upper_bound.count()) << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak(); + } +} + +void ConsoleMPIReporter::benchmarkFailed( StringRef error ) { + auto guard = m_colour->guardColour( Colour::Red ).engage( m_stream ); + (*m_tablePrinter) + << "Benchmark failed (" << error << ')' + << ColumnBreak() << RowBreak(); +} + +void ConsoleMPIReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { + m_tablePrinter->close(); + StreamingReporterBase::testCaseEnded(_testCaseStats); + m_headerPrinted = false; +} +void ConsoleMPIReporter::testRunEnded(TestRunStats const& _testRunStats) { + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if(rank != 0 && _testRunStats.totals.testCases.allPassed()) + return; + printTotalsDivider(_testRunStats.totals); + printTotals(_testRunStats.totals); + m_stream << '\n' << std::flush; + StreamingReporterBase::testRunEnded(_testRunStats); +} +void ConsoleMPIReporter::testRunStarting(TestRunInfo const& _testInfo) { + StreamingReporterBase::testRunStarting(_testInfo); + int rank = -1; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if(rank != 0) return; + if ( m_config->testSpec().hasFilters() ) { + m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: " + << serializeFilters( m_config->getTestsOrTags() ) << '\n'; + } + m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n'; +} + +void ConsoleMPIReporter::lazyPrint() { + + m_tablePrinter->close(); + lazyPrintWithoutClosingBenchmarkTable(); +} + +void ConsoleMPIReporter::lazyPrintWithoutClosingBenchmarkTable() { + + if ( !m_testRunInfoPrinted ) { + lazyPrintRunInfo(); + } + if (!m_headerPrinted) { + printTestCaseAndSectionHeader(); + m_headerPrinted = true; + } +} +void ConsoleMPIReporter::lazyPrintRunInfo() { + m_stream << '\n' + << lineOfChars( '~' ) << '\n' + << m_colour->guardColour( Colour::SecondaryText ) + << currentTestRunInfo.name << " is a Catch2 v" << libraryVersion() + << " host application.\n" + << "Run with -? for options\n\n"; + + m_testRunInfoPrinted = true; +} +void ConsoleMPIReporter::printTestCaseAndSectionHeader() { + assert(!m_sectionStack.empty()); + printOpenHeader(currentTestCaseInfo->name); + + if (m_sectionStack.size() > 1) { + auto guard = m_colour->guardColour( Colour::Headers ).engage( m_stream ); + + auto + it = m_sectionStack.begin() + 1, // Skip first section (test case) + itEnd = m_sectionStack.end(); + for (; it != itEnd; ++it) + printHeaderString(it->name, 2); + } + + SourceLineInfo lineInfo = m_sectionStack.back().lineInfo; + + + m_stream << lineOfChars( '-' ) << '\n' + << m_colour->guardColour( Colour::FileName ) << lineInfo << '\n' + << lineOfChars( '.' ) << "\n\n" + << std::flush; +} + +void ConsoleMPIReporter::printClosedHeader(std::string const& _name) { + printOpenHeader(_name); + m_stream << lineOfChars('.') << '\n'; +} +void ConsoleMPIReporter::printOpenHeader(std::string const& _name) { + m_stream << lineOfChars('-') << '\n'; + { + auto guard = m_colour->guardColour( Colour::Headers ).engage( m_stream ); + printHeaderString(_name); + } +} + +void ConsoleMPIReporter::printHeaderString(std::string const& _string, std::size_t indent) { + // We want to get a bit fancy with line breaking here, so that subsequent + // lines start after ":" if one is present, e.g. + // ``` + // blablabla: Fancy + // linebreaking + // ``` + // but we also want to avoid problems with overly long indentation causing + // the text to take up too many lines, e.g. + // ``` + // blablabla: F + // a + // n + // c + // y + // . + // . + // . + // ``` + // So we limit the prefix indentation check to first quarter of the possible + // width + std::size_t idx = _string.find( ": " ); + if ( idx != std::string::npos && idx < CATCH_CONFIG_CONSOLE_WIDTH / 4 ) { + idx += 2; + } else { + idx = 0; + } + m_stream << TextFlow::Column( _string ) + .indent( indent + idx ) + .initialIndent( indent ) + << '\n'; +} + +struct SummaryColumn { + + SummaryColumn( std::string _label, Colour::Code _colour ) + : label( CATCH_MOVE( _label ) ), + colour( _colour ) {} + SummaryColumn addRow( std::uint64_t count ) { + ReusableStringStream rss; + rss << count; + std::string row = rss.str(); + for (auto& oldRow : rows) { + while (oldRow.size() < row.size()) + oldRow = ' ' + oldRow; + while (oldRow.size() > row.size()) + row = ' ' + row; + } + rows.push_back(row); + return *this; + } + + std::string label; + Colour::Code colour; + std::vector rows; + +}; + +void ConsoleMPIReporter::printTotals( Totals const& totals ) { + if (totals.testCases.total() == 0) { + m_stream << m_colour->guardColour( Colour::Warning ) + << "No tests ran\n"; + } else if (totals.assertions.total() > 0 && totals.testCases.allPassed()) { + m_stream << m_colour->guardColour( Colour::ResultSuccess ) + << "All tests passed"; + m_stream << " (" + << pluralise(totals.assertions.passed, "assertion"_sr) << " in " + << pluralise(totals.testCases.passed, "test case"_sr) << ')' + << '\n'; + } else { + + std::vector columns; + columns.push_back(SummaryColumn("", Colour::None) + .addRow(totals.testCases.total()) + .addRow(totals.assertions.total())); + columns.push_back(SummaryColumn("passed", Colour::Success) + .addRow(totals.testCases.passed) + .addRow(totals.assertions.passed)); + columns.push_back(SummaryColumn("failed", Colour::ResultError) + .addRow(totals.testCases.failed) + .addRow(totals.assertions.failed)); + columns.push_back(SummaryColumn("failed as expected", Colour::ResultExpectedFailure) + .addRow(totals.testCases.failedButOk) + .addRow(totals.assertions.failedButOk)); + + printSummaryRow("test cases"_sr, columns, 0); + printSummaryRow("assertions"_sr, columns, 1); + } +} +void ConsoleMPIReporter::printSummaryRow(StringRef label, std::vector const& cols, std::size_t row) { + for (auto col : cols) { + std::string const& value = col.rows[row]; + if (col.label.empty()) { + m_stream << label << ": "; + if ( value != "0" ) { + m_stream << value; + } else { + m_stream << m_colour->guardColour( Colour::Warning ) + << "- none -"; + } + } else if (value != "0") { + m_stream << m_colour->guardColour( Colour::LightGrey ) << " | " + << m_colour->guardColour( col.colour ) << value << ' ' + << col.label; + } + } + m_stream << '\n'; +} + +void ConsoleMPIReporter::printTotalsDivider(Totals const& totals) { + if (totals.testCases.total() > 0) { + std::size_t failedRatio = makeRatio(totals.testCases.failed, totals.testCases.total()); + std::size_t failedButOkRatio = makeRatio(totals.testCases.failedButOk, totals.testCases.total()); + std::size_t passedRatio = makeRatio(totals.testCases.passed, totals.testCases.total()); + while (failedRatio + failedButOkRatio + passedRatio < CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio)++; + while (failedRatio + failedButOkRatio + passedRatio > CATCH_CONFIG_CONSOLE_WIDTH - 1) + findMax(failedRatio, failedButOkRatio, passedRatio)--; + + m_stream << m_colour->guardColour( Colour::Error ) + << std::string( failedRatio, '=' ) + << m_colour->guardColour( Colour::ResultExpectedFailure ) + << std::string( failedButOkRatio, '=' ); + if ( totals.testCases.allPassed() ) { + m_stream << m_colour->guardColour( Colour::ResultSuccess ) + << std::string( passedRatio, '=' ); + } else { + m_stream << m_colour->guardColour( Colour::Success ) + << std::string( passedRatio, '=' ); + } + } else { + m_stream << m_colour->guardColour( Colour::Warning ) + << std::string( CATCH_CONFIG_CONSOLE_WIDTH - 1, '=' ); + } + m_stream << '\n'; +} +void ConsoleMPIReporter::printSummaryDivider() { + m_stream << lineOfChars('-') << '\n'; +} + +} // end namespace Catch + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif \ No newline at end of file diff --git a/tests/unit/mpi_console_reporter.hpp b/tests/unit/mpi_console_reporter.hpp new file mode 100644 index 00000000..e9b55ac9 --- /dev/null +++ b/tests/unit/mpi_console_reporter.hpp @@ -0,0 +1,73 @@ +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 +#ifndef DYAD_CATCH_REPORTER_MPI_CONSOLE_HPP_INCLUDED +#define DYAD_CATCH_REPORTER_MPI_CONSOLE_HPP_INCLUDED + +#include +#include +#include + +namespace Catch { + // Fwd decls + struct SummaryColumn; + class TablePrinter; + + class ConsoleMPIReporter final : public StreamingReporterBase { + Detail::unique_ptr m_tablePrinter; + + public: + ConsoleMPIReporter(ReporterConfig&& config); + ~ConsoleMPIReporter() override; + static std::string getDescription(); + + void noMatchingTestCases( StringRef unmatchedSpec ) override; + void reportInvalidTestSpec( StringRef arg ) override; + + void assertionStarting(AssertionInfo const&) override; + + void assertionEnded(AssertionStats const& _assertionStats) override; + + void sectionStarting(SectionInfo const& _sectionInfo) override; + void sectionEnded(SectionStats const& _sectionStats) override; + + void benchmarkPreparing( StringRef name ) override; + void benchmarkStarting(BenchmarkInfo const& info) override; + void benchmarkEnded(BenchmarkStats<> const& stats) override; + void benchmarkFailed( StringRef error ) override; + + void testCaseEnded(TestCaseStats const& _testCaseStats) override; + void testRunEnded(TestRunStats const& _testRunStats) override; + void testRunStarting(TestRunInfo const& _testRunInfo) override; + + private: + void lazyPrint(); + + void lazyPrintWithoutClosingBenchmarkTable(); + void lazyPrintRunInfo(); + void printTestCaseAndSectionHeader(); + + void printClosedHeader(std::string const& _name); + void printOpenHeader(std::string const& _name); + + // if string has a : in first line will set indent to follow it on + // subsequent lines + void printHeaderString(std::string const& _string, std::size_t indent = 0); + + + void printTotals(Totals const& totals); + void printSummaryRow(StringRef label, std::vector const& cols, std::size_t row); + + void printTotalsDivider(Totals const& totals); + void printSummaryDivider(); + + bool m_headerPrinted = false; + bool m_testRunInfoPrinted = false; + }; + +} // end namespace Catch +CATCH_REGISTER_REPORTER("mpi_console", Catch::ConsoleMPIReporter) +#endif // DYAD_CATCH_REPORTER_MPI_CONSOLE_HPP_INCLUDED \ No newline at end of file From e8b78e543b58269bb6dd376db969626b27fc7012 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 21 Mar 2024 09:27:32 -0700 Subject: [PATCH 28/47] Updates DataSpaces test --- tests/CMakeLists.txt | 12 +++++-- tests/dspaces_perf/data_plane/data_plane.cpp | 34 +++++++++++--------- tests/dspaces_perf/unit_test.cpp | 4 +-- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index da212291..2661b5df 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,2 +1,10 @@ -#add_subdirectory(dspaces_perf) -add_subdirectory(unit) +option(ENABLE_DSPACES_TEST "Enable DataSpaces perf test" OFF) +option(ENABLE_UNIT_TEST "Enable DYAD unit tests" ON) + +if (ENABLE_DSPACES_TEST) + add_subdirectory(dspaces_perf) +endif () + +if (ENABLE_UNIT_TEST) + add_subdirectory(unit) +endif () diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp index 42c4d0e9..2213aec3 100644 --- a/tests/dspaces_perf/data_plane/data_plane.cpp +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -7,20 +7,22 @@ FILE* redirect_stdout(const char* filename) { - size_t dir_len = strlen(args.dspaces_timing_dir); - bool ends_with_separator = (args.dspaces_timing_dir[dir_len-1] == '/'); + size_t dir_len = strlen(args.dspaces_timing_dir.c_str()); + bool ends_with_separator = (args.dspaces_timing_dir.c_str()[dir_len-1] == '/'); size_t filename_len = dir_len + strlen(filename) + 1; if (!ends_with_separator) { filename_len += 1; } - char* full_filename = malloc(filename_len * sizeof(char)); + char* full_filename = (char*) malloc(filename_len * sizeof(char)); memset(full_filename, 0, filename_len*sizeof(char)); - strcpy(full_filename, args.dspaces_timing_dir); + strcpy(full_filename, args.dspaces_timing_dir.c_str()); if (!ends_with_separator) { strcat(full_filename, "/"); } strcat(full_filename, filename); - return freopen(full_filename, "a", stdout); + FILE* fp = freopen(full_filename, "a", stdout); + free(full_filename); + return fp; } int restore_stdout(FILE* freopen_fp) @@ -110,7 +112,7 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); @@ -123,7 +125,7 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); } } - rc = dspaces_fini(); + rc = dspaces_fini(client); REQUIRE (rc == dspaces_SUCCESS); REQUIRE (posttest() == 0); } @@ -136,7 +138,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, false, false) == 0); + REQUIRE (create_files_per_server_process(&client, false, false) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; @@ -156,7 +158,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f // Using aget instead of get because dyad_get_data also allocates the buffer // Unlike the previous test, we set the timeout to -1 so it will do any blocking that it might want to do // TODO: confirm that the timeout is actually needed to guarantee this type of behavior - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); @@ -169,7 +171,7 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); } } - rc = dspaces_fini(); + rc = dspaces_fini(client); REQUIRE (rc == dspaces_SUCCESS); REQUIRE (posttest() == 0); } @@ -183,7 +185,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, true, true) == 0); + REQUIRE (create_files_per_server_process(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; @@ -200,7 +202,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); @@ -213,7 +215,7 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); } } - rc = dspaces_fini(); + rc = dspaces_fini(client); REQUIRE (rc == dspaces_SUCCESS); REQUIRE (posttest() == 0); } @@ -227,7 +229,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f dspaces_client_t client = dspaces_CLIENT_NULL; int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_broker(&client, true, true) == 0); + REQUIRE (create_files_per_server_process(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; char filename[4096]; @@ -244,7 +246,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); @@ -257,7 +259,7 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); } } - rc = dspaces_fini(); + rc = dspaces_fini(client); REQUIRE (rc == dspaces_SUCCESS); REQUIRE (posttest() == 0); } diff --git a/tests/dspaces_perf/unit_test.cpp b/tests/dspaces_perf/unit_test.cpp index 6970de66..a76ce83f 100644 --- a/tests/dspaces_perf/unit_test.cpp +++ b/tests/dspaces_perf/unit_test.cpp @@ -14,7 +14,7 @@ struct Info { size_t num_server_procs; }; struct Arguments { - const char* dspaces_timing_dir; + std::string dspaces_timing_dir; // MPI Configurations size_t process_per_node = 1; // DataSpaces Configuration @@ -80,4 +80,4 @@ int posttest() { } #include "data_plane/data_plane.cpp" // Temporarily disable mdm tests -// #include "mdm/mdm.cpp" \ No newline at end of file +// #include "mdm/mdm.cpp" From 0508d24043668c4056676fb46fa464a1f8ff3d77 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Thu, 21 Mar 2024 11:06:25 -0700 Subject: [PATCH 29/47] Adds scripts for running the DataSpaces tests for the SC paper --- scripts/dspaces/corona.sh | 14 +++++++++++ scripts/dspaces/dspaces_start.sh | 20 +++++++++++++++ scripts/dspaces/dspaces_stop.sh | 3 +++ scripts/dspaces/run.sh | 24 ++++++++++++++++++ scripts/dspaces/run_all.sh | 41 +++++++++++++++++++++++++++++++ scripts/dspaces/setup-env.sh | 25 +++++++++++++++++++ tests/dspaces_perf/CMakeLists.txt | 2 +- 7 files changed, 128 insertions(+), 1 deletion(-) create mode 100755 scripts/dspaces/corona.sh create mode 100644 scripts/dspaces/dspaces_start.sh create mode 100644 scripts/dspaces/dspaces_stop.sh create mode 100755 scripts/dspaces/run.sh create mode 100644 scripts/dspaces/run_all.sh create mode 100644 scripts/dspaces/setup-env.sh diff --git a/scripts/dspaces/corona.sh b/scripts/dspaces/corona.sh new file mode 100755 index 00000000..97db0a9a --- /dev/null +++ b/scripts/dspaces/corona.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +test_case=$1 +num_nodes=$2 +ppn=$3 + +num_iters=16 +num_files=16 +request_size=65536 +hg_conn_str="ucx+rc" + +source ./setup-env.sh + +flux alloc -q $QUEUE -t $TIME -N $NUM_NODES --exclusive ./run.sh $test_case $num_nodes $ppn $num_iters $num_files $request_size $hg_conn_str diff --git a/scripts/dspaces/dspaces_start.sh b/scripts/dspaces/dspaces_start.sh new file mode 100644 index 00000000..036a5ef1 --- /dev/null +++ b/scripts/dspaces/dspaces_start.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +echo "## Config file for DataSpaces server +ndim = 1 +dims = $1 +max_versions = $2 +num_apps = 1" > dataspaces.conf + +# Use provided number of nodes instead of auto-obtained number +# dspaces_num_nodes=$(flux resource info | grep -oP "\d+ Nodes" | grep -oP "^\d+") + +flux submit -N $3 --tasks-per-node $4 dspaces_server $5 + +# Wait for DataSpaces configuration file to be created. +# If we don't do this, the DataSpaces clients will either crash or hang +sleep 1s +while [ ! -f conf.ds ]; do + sleep 1s +done +sleep 3s diff --git a/scripts/dspaces/dspaces_stop.sh b/scripts/dspaces/dspaces_stop.sh new file mode 100644 index 00000000..1d7e0d60 --- /dev/null +++ b/scripts/dspaces/dspaces_stop.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +flux run --ntasks=1 terminator diff --git a/scripts/dspaces/run.sh b/scripts/dspaces/run.sh new file mode 100755 index 00000000..0f5fd2e5 --- /dev/null +++ b/scripts/dspaces/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash +test_case=$1 +NUM_NODES=$2 +PPN=$3 +NUM_ITERS=$4 +NUM_FILES=$5 +REQUEST_SIZE=$6 +HG_CONNECTION_STR="$7" + +source ./setup-env.sh + +EXEC_DIR="${GITHUB_WORKSPACE}/build/bin" + +echo Starting DataSpaces +# Setup DYAD +./dspaces_start.sh $(( REQUEST_SIZE * NUM_ITERS )) ${NUM_FILES} ${NUM_NODES} ${SERVER_PPN} ${HG_CONNECTION_STR} + +echo "Running Test $WORKLOAD" +flux run -N ${NUM_NODES} --tasks-per-node=${PPN} ${EXEC_DIR}/unit_test --filename dp_${NUM_NODES}_${PPN} \ + --ppn ${PPN} --iteration ${NUM_ITERS} --number_of_files ${NUM_FILES} \ + --server_ppn ${SERVER_PPN} --request_size ${REQUEST_SIZE} --timing_dir ${TIMING_DIR} ${test_case} + +echo Stopping DataSpaces +./dspaces_stop.sh diff --git a/scripts/dspaces/run_all.sh b/scripts/dspaces/run_all.sh new file mode 100644 index 00000000..a5348228 --- /dev/null +++ b/scripts/dspaces/run_all.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Do 2 and 1 node runs for Remote and Local tests with varying ppn +ppns=( 1 2 4 8 16 32 64 ) + +for ppn in ${ppns[@]}; do + # Run 2 node test for RemoteDataBandwidth + ./corona.sh RemoteDataBandwidth 2 $ppn + # Run 2 node test for RemoteDataAggBandwidth + ./corona.sh RemoteDataAggBandwidth 2 $ppn + # Run 1 node test for LocalProcessDataBandwidth + ./corona.sh LocalProcessDataBandwidth 1 $ppn + # Run 1 node test for LocalNodeDataBandwidth + ./corona.sh LocalNodeDataBandwidth 1 $ppn +done + +# Do local tests with varying number of nodes and ppn +num_nodes=( 2 4 8 16 32 64 ) +ppns=( 16 32 64 ) + +for nn in ${num_nodes[@]}; do + for ppn in ${ppns[@]}; do + # Run 1 node test for LocalProcessDataBandwidth + ./corona.sh LocalProcessDataBandwidth $nn $ppn + # Run 1 node test for LocalNodeDataBandwidth + ./corona.sh LocalNodeDataBandwidth $nn $ppn + done +done + +# Do remote tests with varying number of nodes and ppn +num_nodes=( 4 8 16 32 64 ) +ppns=( 16 32 64 ) + +for nn in ${num_nodes[@]}; do + for ppn in ${ppns[@]}; do + # Run 1 node test for RemoteDataBandwidth + ./corona.sh RemoteDataBandwidth $nn $ppn + # Run 1 node test for RemoteDataAggDataBandwidth + ./corona.sh RemoteDataAggBandwidth $nn $ppn + done +done diff --git a/scripts/dspaces/setup-env.sh b/scripts/dspaces/setup-env.sh new file mode 100644 index 00000000..da67d9eb --- /dev/null +++ b/scripts/dspaces/setup-env.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +module load gcc/10.3.1 +module load python/3.9.12 +module load openmpi/4.1.2 +# Configurations +export QUEUE=pdebug +export TIME=$((60)) +export SERVER_PPN=1 + +export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad +export SPACK_DIR=/usr/workspace/haridev/spack +export SPACK_ENV=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env +export SPACK_VIEW=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env/.spack-env/view + +# Activate Environments +. ${SPACK_DIR}/share/spack/setup-env.sh +spack env activate -p ${SPACK_ENV} + +# Derived PATHS +export PATH=${PATH}:${SPACK_VIEW}/bin:${SPACK_VIEW}/sbin +export LD_LIBRARY_PATH=/usr/lib64:${SPACK_VIEW}/lib:${SPACK_VIEW}/lib64:${LD_LIBRARY_PATH} + +unset LUA_PATH +unset LUA_CPATH diff --git a/tests/dspaces_perf/CMakeLists.txt b/tests/dspaces_perf/CMakeLists.txt index 137da424..75d37d7c 100644 --- a/tests/dspaces_perf/CMakeLists.txt +++ b/tests/dspaces_perf/CMakeLists.txt @@ -26,4 +26,4 @@ target_link_libraries(unit_test ${TEST_LIBS}) # TODO add the "add_test" directives by going into the subdirectories # add_subdirectory(script) # add_subdirectory(data_plane) -# add_subdirectory(mdm) \ No newline at end of file +# add_subdirectory(mdm) From 4628e8b63289b019361784dd159f6f69e21caee9 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sat, 23 Mar 2024 01:11:53 -0700 Subject: [PATCH 30/47] Added scripts for debugging mpi. --- scripts/Testing/Temporary/CTestCostData.txt | 1 + scripts/mpi_attach.py | 98 +++++++++++++++++++++ scripts/setup-env.sh | 3 + tests/unit/data_plane/CMakeLists.txt | 2 +- tests/unit/mdm/CMakeLists.txt | 2 +- tests/unit/unit_test.cpp | 57 ++++++++++-- 6 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 scripts/Testing/Temporary/CTestCostData.txt create mode 100644 scripts/mpi_attach.py diff --git a/scripts/Testing/Temporary/CTestCostData.txt b/scripts/Testing/Temporary/CTestCostData.txt new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/scripts/Testing/Temporary/CTestCostData.txt @@ -0,0 +1 @@ +--- diff --git a/scripts/mpi_attach.py b/scripts/mpi_attach.py new file mode 100644 index 00000000..c95e9160 --- /dev/null +++ b/scripts/mpi_attach.py @@ -0,0 +1,98 @@ +import argparse +import os +import paramiko +import logging +import shutil +import json + +def parse_args(): + parser = argparse.ArgumentParser( + prog='mpi_attach', + description='Attach to a mpi program') + parser.add_argument('-c', '--conf_dir', help="pass conf_dir else it is infered using VSC_DEBUG_CONF_DIR") + parser.add_argument('-p', '--project_dir', help="pass project_dir") + parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('-d', '--debug', action='store_true') + args = parser.parse_args() + if args.conf_dir == None: + args.conf_dir = os.getenv("VSC_DEBUG_CONF_DIR", ".") + loglevel = logging.WARNING + if args.verbose: + loglevel = logging.INFO + elif args.debug: + loglevel = logging.DEBUG + logging.basicConfig(level=loglevel, + handlers=[ + logging.StreamHandler() + ], + format='[%(levelname)s] [%(asctime)s] %(message)s [%(pathname)s:%(lineno)d]', + datefmt='%H:%M:%S' + ) + logging.info(f"args: {args}") + return args + +def main(): + args = parse_args() + conf_file = f"{args.conf_dir}/debug.conf" + file = open(conf_file, 'r') + lines = file.readlines() + file.close() + + vals = [{}]*len(lines) + logging.info(f"vals has {len(vals)} values") + for line in lines: + exec, rank, hostname, port, pid = line.split(":") + exec = exec.strip() + rank = int(rank.strip()) + hostname = hostname.strip() + port = int(port.strip()) + pid = int(pid.strip()) + vals[rank] = {"hostname":hostname, "port":port, "pid":pid, "exec":exec} + + # Create a launch_json files + launch_file = f"{args.project_dir}/.vscode/launch.json" + with open(launch_file, "r") as jsonFile: + launch_data = json.load(jsonFile) + + # clean previous configurations + confs = launch_data["configurations"] + final_confs = [] + for conf in confs: + if "mpi_gdb" not in conf["name"]: + final_confs.append(conf) + + for rank, val in enumerate(vals): + exec = val["exec"] + port = val["port"] + hostname = val["hostname"] + final_confs.append({ + "type": "gdb", + "request": "attach", + "name": f"mpi_gdb for rank {rank}", + "executable": f"{exec}", + "target": f"{hostname}:{port}", + "remote": True, + "cwd": "${workspaceRoot}", + "gdbpath": "gdb" + }) + launch_data["configurations"]=final_confs + with open(launch_file, "w") as jsonFile: + json.dump(launch_data, jsonFile, indent=2) + + gdbserver_exe = shutil.which("gdbserver") + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + for rank, val in enumerate(vals): + hostname, port, pid = val["hostname"], val["port"], val["pid"] + logging.info(f"rank:{rank} hostname:{hostname} port:{port} pid:{pid}") + ssh.connect(hostname) + cmd = f"{gdbserver_exe} {hostname}:{port} --attach {pid} > {os.getcwd()}/gdbserver-{hostname}-{pid}.log 2>&1 &" + logging.info(f"cmd:{cmd}") + transport = ssh.get_transport() + channel = transport.open_session() + channel.exec_command(cmd) + ssh.close() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index acb05520..f31ce609 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -2,6 +2,9 @@ module load python/3.9.12 module load openmpi/4.1.2 +module load gcc/10.3.1 +export VSC_DEBUG_PROJECT_DIR=/usr/workspace/haridev/dyad +export VSC_DEBUG_CONF_DIR=${VSC_DEBUG_PROJECT_DIR}/build test_case=$1 NUM_NODES=$2 PPN=$3 diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 3450d497..7c8b1e68 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --debug --reporter mpi_console RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) diff --git a/tests/unit/mdm/CMakeLists.txt b/tests/unit/mdm/CMakeLists.txt index 3e25fe90..1d450667 100644 --- a/tests/unit/mdm/CMakeLists.txt +++ b/tests/unit/mdm/CMakeLists.txt @@ -29,7 +29,7 @@ function(add_mdm_test node ppn files ts ops) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_PATH_PRODUCER=$ENV{DYAD_DMD_DIR}) endfunction() -set(ppns 64) +set(ppns 2) set(nodes 1 2 4 8 16 32 64) foreach (node ${nodes}) foreach (ppn ${ppns}) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 9086ad2e..3a947c2d 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -2,7 +2,11 @@ #include #include #include +#include +#include +#include +#include #include namespace fs = std::experimental::filesystem; @@ -18,6 +22,7 @@ struct Info { flux_t* flux_handle; uint32_t broker_idx; uint32_t broker_size; + bool debug_init; }; struct Arguments { // MPI Configurations @@ -47,12 +52,8 @@ int init(int* argc, char*** argv) { MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); info.flux_handle = flux_open (NULL, 0); - if (args.debug && info.rank == 0) { - printf("%d ready for attach\n", info.comm_size); - fflush(stdout); - getchar(); - } - MPI_Barrier(MPI_COMM_WORLD); + info.debug_init = false; + MPI_Barrier(MPI_COMM_WORLD); return 0; } int finalize() { @@ -76,11 +77,51 @@ cl::Parser define_options() { "number_of_files")["-n"]["--number_of_files"]("Number of Files") | cl::Opt(args.brokers_per_node, "brokers_per_node")["-b"]["--brokers_per_node"]("Number of Brokers per node") | - cl::Opt(args.debug, - "debug")["-d"]["--debug"]("debug"); + cl::Opt(args.debug)["-d"]["--debug"]("debug"); } int pretest() { + if (!info.debug_init && args.debug) { + const int HOSTNAME_SIZE = 256; + char hostname[HOSTNAME_SIZE]; + gethostname(hostname, HOSTNAME_SIZE); + int pid = getpid(); + char* start_port_str = getenv("VSC_DEBUG_START_PORT"); + int start_port = 10000; + if (start_port_str != nullptr) { + start_port = atoi(start_port_str); + } + const char* conf_dir = getenv("VSC_DEBUG_CONF_DIR"); + if (conf_dir == nullptr) { + conf_dir = "."; + } + char conf_file[4096]; + sprintf(conf_file, "%s/debug.conf", conf_dir); + + char exe[1024]; + int ret = readlink("/proc/self/exe",exe,sizeof(exe)-1); + REQUIRE(ret !=-1); + exe[ret] = 0; + FILE* fh; + if (info.rank == 0) { + fh = fopen(conf_file, "w+"); + REQUIRE(fh != nullptr); + fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); + fclose(fh); + } + MPI_Barrier(MPI_COMM_WORLD); + if (info.rank != 0) { + fh = fopen(conf_file, "a+"); + REQUIRE(fh != nullptr); + fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); + fclose(fh); + } + if (info.rank == 0) { + printf("%d ready for attach\n", info.comm_size); + sleep(120); + } + info.debug_init = true; + } info.num_nodes = info.comm_size / args.process_per_node; info.num_brokers = info.num_nodes * args.brokers_per_node; flux_get_rank (info.flux_handle, &info.broker_idx); From fb7bf623dafaa0f0caa4948ea8e6e940bbb8f3d1 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sat, 23 Mar 2024 14:41:22 -0700 Subject: [PATCH 31/47] add compound test --- scripts/mpi_attach.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/mpi_attach.py b/scripts/mpi_attach.py index c95e9160..d6dfd473 100644 --- a/scripts/mpi_attach.py +++ b/scripts/mpi_attach.py @@ -61,20 +61,41 @@ def main(): if "mpi_gdb" not in conf["name"]: final_confs.append(conf) + compound_names = [] for rank, val in enumerate(vals): exec = val["exec"] port = val["port"] hostname = val["hostname"] + test_name = f"mpi_gdb for rank {rank}" final_confs.append({ "type": "gdb", "request": "attach", - "name": f"mpi_gdb for rank {rank}", + "name": test_name, "executable": f"{exec}", "target": f"{hostname}:{port}", "remote": True, "cwd": "${workspaceRoot}", "gdbpath": "gdb" }) + compound_names.append(test_name) + final_compounds = [] + compounds = [] + if "compounds" in launch_data: + compounds = launch_data["compounds"] + final_compounds = [] + for compound in compounds: + if "mpi_gdb" not in compound["name"]: + final_compounds.append(compound) + + final_compounds.append({ + "name": "mpi_gdb compound", + "configurations": compound_names, + "preLaunchTask": "${defaultBuildTask}", + "stopAll": True + }) + launch_data["compounds"] = final_compounds + + launch_data["configurations"]=final_confs with open(launch_file, "w") as jsonFile: json.dump(launch_data, jsonFile, indent=2) From 362464a81dab04d9d56fff6ebcb054261a5d7e64 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sun, 24 Mar 2024 12:13:26 -0700 Subject: [PATCH 32/47] fixed tests --- scripts/setup-env.sh | 2 +- tests/unit/data_plane/CMakeLists.txt | 2 +- tests/unit/unit_test.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/setup-env.sh b/scripts/setup-env.sh index f31ce609..ebd30850 100644 --- a/scripts/setup-env.sh +++ b/scripts/setup-env.sh @@ -4,7 +4,7 @@ module load python/3.9.12 module load openmpi/4.1.2 module load gcc/10.3.1 export VSC_DEBUG_PROJECT_DIR=/usr/workspace/haridev/dyad -export VSC_DEBUG_CONF_DIR=${VSC_DEBUG_PROJECT_DIR}/build +export VSC_DEBUG_CONF_DIR=${VSC_DEBUG_PROJECT_DIR}/.vscode/ test_case=$1 NUM_NODES=$2 PPN=$3 diff --git a/tests/unit/data_plane/CMakeLists.txt b/tests/unit/data_plane/CMakeLists.txt index 7c8b1e68..3450d497 100644 --- a/tests/unit/data_plane/CMakeLists.txt +++ b/tests/unit/data_plane/CMakeLists.txt @@ -7,7 +7,7 @@ set(ppns 1 2 4 8 16 32 64) function(add_dp_remote_test node ppn files ts ops) # Remote Streaming RPC over RDMA set(test_name unit_remote_data_${node}_${ppn}) - add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --debug --reporter mpi_console RemoteDataBandwidth) + add_test(${test_name} flux run -N ${node} --tasks-per-node ${ppn} ${CMAKE_BINARY_DIR}/bin/unit_test --filename dp_${node}_${ppn} --ppn ${ppn} --pfs $ENV{DYAD_PFS_DIR} --dmd $ENV{DYAD_DMD_DIR} --iteration ${ops} --number_of_files ${files} --request_size ${ts} --reporter mpi_console RemoteDataBandwidth) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_KVS_NAMESPACE=${DYAD_KEYSPACE}) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_MODULE_SO=${CMAKE_BINARY_DIR}/${DYAD_LIBDIR}/dyad.so) set_property(TEST ${test_name} APPEND PROPERTY ENVIRONMENT DYAD_LOG_DIR=${DYAD_LOG_DIR}) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 3a947c2d..a1a8593b 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -106,6 +106,7 @@ int pretest() { if (info.rank == 0) { fh = fopen(conf_file, "w+"); REQUIRE(fh != nullptr); + fprintf(fh, "%d\n", info.comm_size); fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); fclose(fh); } From 12352d282bfb8f702762f170a77bbf53fe22901a Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sun, 24 Mar 2024 13:09:31 -0700 Subject: [PATCH 33/47] fixed printing --- tests/unit/unit_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index a1a8593b..2caf4de6 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -117,8 +117,10 @@ int pretest() { fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); fclose(fh); } + MPI_Barrier(MPI_COMM_WORLD); if (info.rank == 0) { printf("%d ready for attach\n", info.comm_size); + fflush(stdout); sleep(120); } info.debug_init = true; From aae77fd54ab383c8f0d1716d2419320b000dab47 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sun, 24 Mar 2024 15:05:19 -0700 Subject: [PATCH 34/47] fixed writing to log --- tests/unit/unit_test.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 2caf4de6..1fd1c905 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -102,21 +102,27 @@ int pretest() { int ret = readlink("/proc/self/exe",exe,sizeof(exe)-1); REQUIRE(ret !=-1); exe[ret] = 0; - FILE* fh; if (info.rank == 0) { - fh = fopen(conf_file, "w+"); - REQUIRE(fh != nullptr); - fprintf(fh, "%d\n", info.comm_size); - fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); - fclose(fh); + remove(conf_file); } MPI_Barrier(MPI_COMM_WORLD); - if (info.rank != 0) { - fh = fopen(conf_file, "a+"); - REQUIRE(fh != nullptr); - fprintf(fh, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); - fclose(fh); - } + MPI_File mpi_fh; + int status_orig = MPI_File_open(MPI_COMM_WORLD, conf_file, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpi_fh); + REQUIRE(status_orig == MPI_SUCCESS); + const int buf_len = 16*1024; + char buffer[buf_len]; + int size; + if (info.rank == 0) { + size = sprintf(buffer, "%d\n%s:%d:%s:%d:%d\n", info.comm_size, exe, info.rank, hostname, start_port+info.rank, pid); + } else { + size = sprintf(buffer, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); + } + MPI_Status status; + MPI_File_write_ordered(mpi_fh, buffer, size, MPI_CHAR, &status); + int written_bytes; + MPI_Get_count(&status, MPI_CHAR, &written_bytes); + REQUIRE(written_bytes == size); + MPI_File_close(&mpi_fh); MPI_Barrier(MPI_COMM_WORLD); if (info.rank == 0) { printf("%d ready for attach\n", info.comm_size); From 9a485ff1b75d36e9ffda7bd9118a0837933b99dc Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Sun, 24 Mar 2024 22:13:25 -0700 Subject: [PATCH 35/47] Fix Clang Formatting --- .clang-format | 3 +- tests/unit/data_plane/data_plane.cpp | 355 ++++++++++++++------------- tests/unit/mdm/mdm.cpp | 250 +++++++++---------- tests/unit/unit_test.cpp | 234 +++++++++--------- 4 files changed, 424 insertions(+), 418 deletions(-) diff --git a/.clang-format b/.clang-format index 57aa8853..b18dc481 100644 --- a/.clang-format +++ b/.clang-format @@ -22,4 +22,5 @@ SpaceBeforeParens: Always TabWidth: '4' UseTab: Never AlwaysBreakAfterReturnType: None -AlwaysBreakAfterDefinitionReturnType: None \ No newline at end of file +AlwaysBreakAfterDefinitionReturnType: None +ContinuationIndentWidth: Always \ No newline at end of file diff --git a/tests/unit/data_plane/data_plane.cpp b/tests/unit/data_plane/data_plane.cpp index 02ec2dc9..b69b6173 100644 --- a/tests/unit/data_plane/data_plane.cpp +++ b/tests/unit/data_plane/data_plane.cpp @@ -1,205 +1,216 @@ -#include #include #include #include +#include + #include int create_files_per_broker() { - char filename[4096], first_file[4096]; - bool is_first = true; - size_t file_size = args.request_size*args.iteration; - size_t node_idx = info.rank / args.process_per_node; - bool first_rank_per_node = info.rank % args.process_per_node == 0; - if (first_rank_per_node) { - fs::create_directories (args.dyad_managed_dir); - for (size_t broker_idx = 0; broker_idx < args.brokers_per_node; ++broker_idx) { - for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { - size_t global_broker_idx = node_idx * args.brokers_per_node + broker_idx; - if (is_first) { - sprintf (first_file, "%s/%s_%zu_%zu.bat", - args.dyad_managed_dir.c_str (), args.filename.c_str (), - global_broker_idx, file_idx); - std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " - + std::to_string (file_size) + "; } > " + first_file + " "; - int status = system (cmd.c_str ()); - (void) status; - is_first = false; - } else { - sprintf (filename, "%s/%s_%zu_%zu.bat", - args.dyad_managed_dir.c_str (), args.filename.c_str (), - global_broker_idx, file_idx); - std::string cmd = "cp " + std::string (first_file) + " " + filename; - int status = system (cmd.c_str ()); - (void) status; - } - } + char filename[4096], first_file[4096]; + bool is_first = true; + size_t file_size = args.request_size * args.iteration; + size_t node_idx = info.rank / args.process_per_node; + bool first_rank_per_node = info.rank % args.process_per_node == 0; + if (first_rank_per_node) { + fs::create_directories(args.dyad_managed_dir); + for (size_t broker_idx = 0; broker_idx < args.brokers_per_node; + ++broker_idx) { + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + size_t global_broker_idx = + node_idx * args.brokers_per_node + broker_idx; + if (is_first) { + sprintf(first_file, "%s/%s_%zu_%zu.bat", + args.dyad_managed_dir.c_str(), args.filename.c_str(), + global_broker_idx, file_idx); + std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + + std::to_string(file_size) + "; } > " + first_file + + " "; + int status = system(cmd.c_str()); + (void)status; + is_first = false; + } else { + sprintf(filename, "%s/%s_%zu_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), global_broker_idx, file_idx); + std::string cmd = "cp " + std::string(first_file) + " " + filename; + int status = system(cmd.c_str()); + (void)status; } + } } - MPI_Barrier (MPI_COMM_WORLD); - return 0; + } + MPI_Barrier(MPI_COMM_WORLD); + return 0; } - -TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" +// clang-format off +TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - REQUIRE (clean_directories() == 0); - REQUIRE (create_files_per_broker() == 0); - dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - auto ctx = dyad_ctx_get(); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096]; - uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; - dyad_metadata_t mdata; - mdata.owner_rank = neighour_broker_idx; - size_t data_len = args.request_size*args.iteration; - char* file_data = NULL; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (filename, "%s_%u_%zu.bat", args.filename.c_str (), - neighour_broker_idx, file_idx); - mdata.fpath = filename; - data_time.resumeTime(); - auto rc = dyad_get_data (ctx, &mdata, &file_data, &data_len); - data_time.pauseTime(); - REQUIRE (rc >= 0); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // clang-format on + REQUIRE(pretest() == 0); + REQUIRE(clean_directories() == 0); + REQUIRE(create_files_per_broker() == 0); + dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + auto ctx = dyad_ctx_get(); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; + dyad_metadata_t mdata; + mdata.owner_rank = neighour_broker_idx; + size_t data_len = args.request_size * args.iteration; + char* file_data = NULL; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(filename, "%s_%u_%zu.bat", args.filename.c_str(), + neighour_broker_idx, file_idx); + mdata.fpath = filename; + data_time.resumeTime(); + auto rc = dyad_get_data(ctx, &mdata, &file_data, &data_len); + data_time.pauseTime(); + REQUIRE(rc >= 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + data_len * args.number_of_files, total_data / info.comm_size, + data_len * args.number_of_files * info.comm_size * info.comm_size / + total_data / 1024 / 1024.0); } - auto rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (clean_directories() == 0); - REQUIRE (posttest() == 0); + } + auto rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(clean_directories() == 0); + REQUIRE(posttest() == 0); } - +// clang-format off TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - REQUIRE (clean_directories() == 0); - REQUIRE (create_files_per_broker() == 0); - dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - auto ctx = dyad_ctx_get(); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096], upath[4096]; - uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; - dyad_metadata_t mdata; - mdata.owner_rank = neighour_broker_idx; - size_t data_len = args.request_size*args.iteration; - if (info.rank % args.process_per_node != 0) - usleep (10000); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (upath, "%s_%u_%zu.bat", args.filename.c_str (), - neighour_broker_idx, file_idx); - sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), - neighour_broker_idx, file_idx); - mdata.fpath = upath; - data_time.resumeTime(); - auto rc = dyad_consume_w_metadata(ctx, filename, &mdata); - data_time.pauseTime(); - REQUIRE (rc >= 0); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // clang-format on + REQUIRE(pretest() == 0); + REQUIRE(clean_directories() == 0); + REQUIRE(create_files_per_broker() == 0); + dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + auto ctx = dyad_ctx_get(); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096], upath[4096]; + uint32_t neighour_broker_idx = (info.broker_idx + 1) % info.broker_size; + dyad_metadata_t mdata; + mdata.owner_rank = neighour_broker_idx; + size_t data_len = args.request_size * args.iteration; + if (info.rank % args.process_per_node != 0) usleep(10000); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(upath, "%s_%u_%zu.bat", args.filename.c_str(), + neighour_broker_idx, file_idx); + sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), neighour_broker_idx, file_idx); + mdata.fpath = upath; + data_time.resumeTime(); + auto rc = dyad_consume_w_metadata(ctx, filename, &mdata); + data_time.pauseTime(); + REQUIRE(rc >= 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + data_len * args.number_of_files, total_data / info.comm_size, + data_len * args.number_of_files * info.comm_size * info.comm_size / + total_data / 1024 / 1024.0); } - auto rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (clean_directories() == 0); - REQUIRE (posttest() == 0); + } + auto rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(clean_directories() == 0); + REQUIRE(posttest() == 0); } +// clang-format off TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - REQUIRE (clean_directories() == 0); - REQUIRE (create_files_per_broker() == 0); - dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096]; - size_t data_len = args.request_size*args.iteration; - char* file_data =(char*)malloc(data_len); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), - info.broker_idx, file_idx); - data_time.resumeTime(); - int fd = open(filename, O_RDONLY); - data_time.pauseTime(); - REQUIRE (fd != -1); - data_time.resumeTime(); - int bytes = read (fd,file_data, data_len); - data_time.pauseTime(); - REQUIRE ((size_t)bytes == data_len); - data_time.resumeTime(); - int status = close(fd); - data_time.pauseTime(); - REQUIRE (status == 0); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // clang-format on + REQUIRE(pretest() == 0); + REQUIRE(clean_directories() == 0); + REQUIRE(create_files_per_broker() == 0); + dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size * args.iteration; + char* file_data = (char*)malloc(data_len); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), info.broker_idx, file_idx); + data_time.resumeTime(); + int fd = open(filename, O_RDONLY); + data_time.pauseTime(); + REQUIRE(fd != -1); + data_time.resumeTime(); + int bytes = read(fd, file_data, data_len); + data_time.pauseTime(); + REQUIRE((size_t)bytes == data_len); + data_time.resumeTime(); + int status = close(fd); + data_time.pauseTime(); + REQUIRE(status == 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + data_len * args.number_of_files, total_data / info.comm_size, + data_len * args.number_of_files * info.comm_size * info.comm_size / + total_data / 1024 / 1024.0); } - auto rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (clean_directories() == 0); - REQUIRE (posttest() == 0); + } + auto rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(clean_directories() == 0); + REQUIRE(posttest() == 0); } - +// clang-format off TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - REQUIRE (clean_directories() == 0); - REQUIRE (create_files_per_broker() == 0); - dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - SECTION("Test Max Bandwidth") { - Timer data_time; - char filename[4096]; - size_t data_len = args.request_size*args.iteration; - char* file_data =(char*)malloc(data_len); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str (), args.filename.c_str (), - info.broker_idx, file_idx); - data_time.resumeTime(); - int fd = open(filename, O_RDONLY); - data_time.pauseTime(); - REQUIRE (fd != -1); - data_time.resumeTime(); - int bytes = read (fd,file_data, data_len); - data_time.pauseTime(); - REQUIRE ((size_t)bytes == data_len); - data_time.resumeTime(); - int status = close(fd); - data_time.pauseTime(); - REQUIRE (status == 0); - } - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // clang-format on + REQUIRE(pretest() == 0); + REQUIRE(clean_directories() == 0); + REQUIRE(create_files_per_broker() == 0); + dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + SECTION("Test Max Bandwidth") { + Timer data_time; + char filename[4096]; + size_t data_len = args.request_size * args.iteration; + char* file_data = (char*)malloc(data_len); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), info.broker_idx, file_idx); + data_time.resumeTime(); + int fd = open(filename, O_RDONLY); + data_time.pauseTime(); + REQUIRE(fd != -1); + data_time.resumeTime(); + int bytes = read(fd, file_data, data_len); + data_time.pauseTime(); + REQUIRE((size_t)bytes == data_len); + data_time.resumeTime(); + int status = close(fd); + data_time.pauseTime(); + REQUIRE(status == 0); + } + AGGREGATE_TIME(data); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + data_len * args.number_of_files, total_data / info.comm_size, + data_len * args.number_of_files * info.comm_size * info.comm_size / + total_data / 1024 / 1024.0); } - auto rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (clean_directories() == 0); - REQUIRE (posttest() == 0); + } + auto rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(clean_directories() == 0); + REQUIRE(posttest() == 0); } \ No newline at end of file diff --git a/tests/unit/mdm/mdm.cpp b/tests/unit/mdm/mdm.cpp index aa7b07ee..584c0c3a 100644 --- a/tests/unit/mdm/mdm.cpp +++ b/tests/unit/mdm/mdm.cpp @@ -3,151 +3,139 @@ #include #include +#include + +// clang-format off TEST_CASE("LocalFSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - REQUIRE (clean_directories() == 0); - dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - REQUIRE (rc >= 0); - auto ctx = dyad_ctx_get(); - struct flock exclusive_lock; - SECTION("Throughput") { - char filename[4096]; - Timer kvs_time; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (filename, - "%s/%s_%u_%zu.bat", - args.dyad_managed_dir.c_str (), - args.filename.c_str (), - info.broker_idx, - file_idx); - INFO("The file " << filename << " rank " << info.rank); - kvs_time.resumeTime(); - int lock_fd = open (filename, O_RDWR | O_CREAT, 0666); - kvs_time.pauseTime(); - REQUIRE (lock_fd != -1); + // clang-format on + REQUIRE(pretest() == 0); + REQUIRE(clean_directories() == 0); + dyad_rc_t rc = dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + REQUIRE(rc >= 0); + auto ctx = dyad_ctx_get(); + struct flock exclusive_lock; + SECTION("Throughput") { + char filename[4096]; + Timer kvs_time; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(filename, "%s/%s_%u_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), info.broker_idx, file_idx); + INFO("The file " << filename << " rank " << info.rank); + kvs_time.resumeTime(); + int lock_fd = open(filename, O_RDWR | O_CREAT, 0666); + kvs_time.pauseTime(); + REQUIRE(lock_fd != -1); - kvs_time.resumeTime(); - rc = dyad_excl_flock (ctx, lock_fd, &exclusive_lock); - kvs_time.pauseTime(); - REQUIRE (rc >= 0); + kvs_time.resumeTime(); + rc = dyad_excl_flock(ctx, lock_fd, &exclusive_lock); + kvs_time.pauseTime(); + REQUIRE(rc >= 0); - kvs_time.resumeTime(); - auto file_size = get_file_size (lock_fd); - kvs_time.pauseTime(); - (void)file_size; - kvs_time.resumeTime(); - dyad_release_flock (ctx, lock_fd, &exclusive_lock); - int status = close (lock_fd); - kvs_time.pauseTime(); - REQUIRE (status == 0); - } - AGGREGATE_TIME(kvs); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, args.number_of_files, - total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); - } + kvs_time.resumeTime(); + auto file_size = get_file_size(lock_fd); + kvs_time.pauseTime(); + (void)file_size; + kvs_time.resumeTime(); + dyad_release_flock(ctx, lock_fd, &exclusive_lock); + int status = close(lock_fd); + kvs_time.pauseTime(); + REQUIRE(status == 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + args.number_of_files, total_kvs / info.comm_size, + args.number_of_files * info.comm_size * info.comm_size / + total_kvs / 1000 / 1000); } - rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (clean_directories() == 0); - REQUIRE (posttest() == 0); + } + rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(clean_directories() == 0); + REQUIRE(posttest() == 0); } - +// clang-format off TEST_CASE("LocalKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - REQUIRE (rc >= 0); - auto ctx = dyad_ctx_get(); - SECTION("Throughput") { - Timer kvs_time; - char my_filename[4096], lookup_filename[4096]; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (my_filename, - "%s/%s_%u_%d_%zu.bat", - args.dyad_managed_dir.c_str (), - args.filename.c_str (), - info.broker_idx, - info.rank, - file_idx); - sprintf (lookup_filename, - "%s_%u_%d_%zu.bat", - args.filename.c_str (), - info.broker_idx, - info.rank, - file_idx); - rc = dyad_commit (ctx, my_filename); - REQUIRE (rc >= 0); - dyad_metadata_t* mdata; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; - gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); - kvs_time.resumeTime(); - rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); - kvs_time.pauseTime(); - REQUIRE (rc >= 0); - } - AGGREGATE_TIME(kvs); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, args.number_of_files, - total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); - } + // clang-format on + REQUIRE(pretest() == 0); + dyad_rc_t rc = dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + REQUIRE(rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(my_filename, "%s/%s_%u_%d_%zu.bat", args.dyad_managed_dir.c_str(), + args.filename.c_str(), info.broker_idx, info.rank, file_idx); + sprintf(lookup_filename, "%s_%u_%d_%zu.bat", args.filename.c_str(), + info.broker_idx, info.rank, file_idx); + rc = dyad_commit(ctx, my_filename); + REQUIRE(rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + gen_path_key(lookup_filename, topic, topic_len, ctx->key_depth, + ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read(ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE(rc >= 0); } - rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (posttest() == 0); + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + args.number_of_files, total_kvs / info.comm_size, + args.number_of_files * info.comm_size * info.comm_size / + total_kvs / 1000 / 1000); + } + } + rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(posttest() == 0); } - +// clang-format off TEST_CASE("RemoteKVSLookup", "[number_of_lookups= " + std::to_string(args.number_of_files) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dyad_rc_t rc = dyad_init_env (DYAD_COMM_RECV, info.flux_handle); - REQUIRE (rc >= 0); - auto ctx = dyad_ctx_get(); - SECTION("Throughput") { - Timer kvs_time; - char my_filename[4096], lookup_filename[4096]; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - sprintf (my_filename, - "%s/%s_%u_%d_%d_%zu.bat", - args.dyad_managed_dir.c_str (), - args.filename.c_str (), - info.broker_idx, - info.rank, - info.comm_size, - file_idx); - sprintf (lookup_filename, - "%s_%u_%d_%d_%zu.bat", - args.filename.c_str (), - info.broker_idx, - info.rank, - info.comm_size, - file_idx); - rc = dyad_commit (ctx, my_filename); - REQUIRE (rc >= 0); - dyad_metadata_t* mdata; - const size_t topic_len = PATH_MAX; - char topic[PATH_MAX+1] = {'\0'}; - gen_path_key (lookup_filename, topic, topic_len, ctx->key_depth, ctx->key_bins); - kvs_time.resumeTime(); - rc = dyad_kvs_read (ctx, topic, lookup_filename, false, &mdata); - kvs_time.pauseTime(); - REQUIRE (rc >= 0); - } - AGGREGATE_TIME(kvs); - if (info.rank == 0) { - printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, args.number_of_files, - total_kvs/info.comm_size, args.number_of_files*info.comm_size*info.comm_size/total_kvs/1000/1000); - } + // clang-format on + REQUIRE(pretest() == 0); + dyad_rc_t rc = dyad_init_env(DYAD_COMM_RECV, info.flux_handle); + REQUIRE(rc >= 0); + auto ctx = dyad_ctx_get(); + SECTION("Throughput") { + Timer kvs_time; + char my_filename[4096], lookup_filename[4096]; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + sprintf(my_filename, "%s/%s_%u_%d_%d_%zu.bat", + args.dyad_managed_dir.c_str(), args.filename.c_str(), + info.broker_idx, info.rank, info.comm_size, file_idx); + sprintf(lookup_filename, "%s_%u_%d_%d_%zu.bat", args.filename.c_str(), + info.broker_idx, info.rank, info.comm_size, file_idx); + rc = dyad_commit(ctx, my_filename); + REQUIRE(rc >= 0); + dyad_metadata_t* mdata; + const size_t topic_len = PATH_MAX; + char topic[PATH_MAX + 1] = {'\0'}; + gen_path_key(lookup_filename, topic, topic_len, ctx->key_depth, + ctx->key_bins); + kvs_time.resumeTime(); + rc = dyad_kvs_read(ctx, topic, lookup_filename, false, &mdata); + kvs_time.pauseTime(); + REQUIRE(rc >= 0); + } + AGGREGATE_TIME(kvs); + if (info.rank == 0) { + printf("[DYAD_TEST],%10d,%10lu,%10.6f,%10.6f\n", info.comm_size, + args.number_of_files, total_kvs / info.comm_size, + args.number_of_files * info.comm_size * info.comm_size / + total_kvs / 1000 / 1000); } - rc = dyad_finalize(); - REQUIRE (rc >= 0); - REQUIRE (posttest() == 0); + } + rc = dyad_finalize(); + REQUIRE(rc >= 0); + REQUIRE(posttest() == 0); } diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 1fd1c905..5b999c37 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -15,28 +15,28 @@ namespace fs = std::experimental::filesystem; */ namespace dyad::test { struct Info { - int rank; - int comm_size; - int num_nodes; - int num_brokers; - flux_t* flux_handle; - uint32_t broker_idx; - uint32_t broker_size; - bool debug_init; + int rank; + int comm_size; + int num_nodes; + int num_brokers; + flux_t* flux_handle; + uint32_t broker_idx; + uint32_t broker_size; + bool debug_init; }; struct Arguments { - // MPI Configurations - size_t process_per_node = 1; - size_t brokers_per_node = 1; - // DYAD Configuration - fs::path dyad_managed_dir = "~/dyad/dmd"; - // Test configuration - fs::path pfs = "~/dyad/pfs"; - std::string filename = "test.dat"; - size_t number_of_files = 1; - size_t request_size = 65536; - size_t iteration = 8; - bool debug = false; + // MPI Configurations + size_t process_per_node = 1; + size_t brokers_per_node = 1; + // DYAD Configuration + fs::path dyad_managed_dir = "~/dyad/dmd"; + // Test configuration + fs::path pfs = "~/dyad/pfs"; + std::string filename = "test.dat"; + size_t number_of_files = 1; + size_t request_size = 65536; + size_t iteration = 8; + bool debug = false; }; } // namespace dyad::test @@ -47,114 +47,120 @@ dyad::test::Info info; */ int init(int* argc, char*** argv) { - // fprintf(stdout, "Initializing MPI\n"); - MPI_Init(argc, argv); - MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); - MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); - info.flux_handle = flux_open (NULL, 0); - info.debug_init = false; - MPI_Barrier(MPI_COMM_WORLD); - return 0; + // fprintf(stdout, "Initializing MPI\n"); + MPI_Init(argc, argv); + MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); + MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); + info.flux_handle = flux_open(NULL, 0); + info.debug_init = false; + MPI_Barrier(MPI_COMM_WORLD); + return 0; } int finalize() { - MPI_Finalize(); - return 0; + MPI_Finalize(); + return 0; } cl::Parser define_options() { - return cl::Opt(args.filename, "filename")["-f"]["--filename"]( - "Filename to be use for I/O.") | - cl::Opt(args.pfs, "pfs")["-d"]["--pfs"]( - "Directory used for performing I/O (default pfs)") | - cl::Opt(args.dyad_managed_dir, "dmd")["-d"]["--dmd"]( - "Directory used for DYAD Managed Directory") | - cl::Opt(args.process_per_node, - "process_per_node")["-p"]["--ppn"]("Processes per node") | - cl::Opt(args.request_size, "request_size")["-r"]["--request_size"]( - "Transfer size used for performing I/O") | - cl::Opt(args.iteration, - "iteration")["-i"]["--iteration"]("Number of Iterations") | - cl::Opt(args.number_of_files, - "number_of_files")["-n"]["--number_of_files"]("Number of Files") | - cl::Opt(args.brokers_per_node, - "brokers_per_node")["-b"]["--brokers_per_node"]("Number of Brokers per node") | - cl::Opt(args.debug)["-d"]["--debug"]("debug"); + return cl::Opt(args.filename, "filename")["-f"]["--filename"]( + "Filename to be use for I/O.") | + cl::Opt(args.pfs, "pfs")["-d"]["--pfs"]( + "Directory used for performing I/O (default pfs)") | + cl::Opt(args.dyad_managed_dir, "dmd")["-d"]["--dmd"]( + "Directory used for DYAD Managed Directory") | + cl::Opt(args.process_per_node, + "process_per_node")["-p"]["--ppn"]("Processes per node") | + cl::Opt(args.request_size, "request_size")["-r"]["--request_size"]( + "Transfer size used for performing I/O") | + cl::Opt(args.iteration, + "iteration")["-i"]["--iteration"]("Number of Iterations") | + cl::Opt( + args.number_of_files, + "number_of_files")["-n"]["--number_of_files"]("Number of Files") | + cl::Opt(args.brokers_per_node, + "brokers_per_node")["-b"]["--brokers_per_node"]( + "Number of Brokers per node") | + cl::Opt(args.debug)["-d"]["--debug"]("debug"); } int pretest() { - if (!info.debug_init && args.debug) { - const int HOSTNAME_SIZE = 256; - char hostname[HOSTNAME_SIZE]; - gethostname(hostname, HOSTNAME_SIZE); - int pid = getpid(); - char* start_port_str = getenv("VSC_DEBUG_START_PORT"); - int start_port = 10000; - if (start_port_str != nullptr) { - start_port = atoi(start_port_str); - } - const char* conf_dir = getenv("VSC_DEBUG_CONF_DIR"); - if (conf_dir == nullptr) { - conf_dir = "."; - } - char conf_file[4096]; - sprintf(conf_file, "%s/debug.conf", conf_dir); + if (!info.debug_init && args.debug) { + const int HOSTNAME_SIZE = 256; + char hostname[HOSTNAME_SIZE]; + gethostname(hostname, HOSTNAME_SIZE); + int pid = getpid(); + char* start_port_str = getenv("VSC_DEBUG_START_PORT"); + int start_port = 10000; + if (start_port_str != nullptr) { + start_port = atoi(start_port_str); + } + const char* conf_dir = getenv("VSC_DEBUG_CONF_DIR"); + if (conf_dir == nullptr) { + conf_dir = "."; + } + char conf_file[4096]; + sprintf(conf_file, "%s/debug.conf", conf_dir); - char exe[1024]; - int ret = readlink("/proc/self/exe",exe,sizeof(exe)-1); - REQUIRE(ret !=-1); - exe[ret] = 0; - if (info.rank == 0) { - remove(conf_file); - } - MPI_Barrier(MPI_COMM_WORLD); - MPI_File mpi_fh; - int status_orig = MPI_File_open(MPI_COMM_WORLD, conf_file, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpi_fh); - REQUIRE(status_orig == MPI_SUCCESS); - const int buf_len = 16*1024; - char buffer[buf_len]; - int size; - if (info.rank == 0) { - size = sprintf(buffer, "%d\n%s:%d:%s:%d:%d\n", info.comm_size, exe, info.rank, hostname, start_port+info.rank, pid); - } else { - size = sprintf(buffer, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); - } - MPI_Status status; - MPI_File_write_ordered(mpi_fh, buffer, size, MPI_CHAR, &status); - int written_bytes; - MPI_Get_count(&status, MPI_CHAR, &written_bytes); - REQUIRE(written_bytes == size); - MPI_File_close(&mpi_fh); - MPI_Barrier(MPI_COMM_WORLD); - if (info.rank == 0) { - printf("%d ready for attach\n", info.comm_size); - fflush(stdout); - sleep(120); - } - info.debug_init = true; + char exe[1024]; + int ret = readlink("/proc/self/exe", exe, sizeof(exe) - 1); + REQUIRE(ret != -1); + exe[ret] = 0; + if (info.rank == 0) { + remove(conf_file); + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_File mpi_fh; + int status_orig = MPI_File_open(MPI_COMM_WORLD, conf_file, + MPI_MODE_WRONLY | MPI_MODE_CREATE, + MPI_INFO_NULL, &mpi_fh); + REQUIRE(status_orig == MPI_SUCCESS); + const int buf_len = 16 * 1024; + char buffer[buf_len]; + int size; + if (info.rank == 0) { + size = sprintf(buffer, "%d\n%s:%d:%s:%d:%d\n", info.comm_size, exe, + info.rank, hostname, start_port + info.rank, pid); + } else { + size = sprintf(buffer, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, + start_port + info.rank, pid); } - info.num_nodes = info.comm_size / args.process_per_node; - info.num_brokers = info.num_nodes * args.brokers_per_node; - flux_get_rank (info.flux_handle, &info.broker_idx); - flux_get_size (info.flux_handle, &info.broker_size); + MPI_Status status; + MPI_File_write_ordered(mpi_fh, buffer, size, MPI_CHAR, &status); + int written_bytes; + MPI_Get_count(&status, MPI_CHAR, &written_bytes); + REQUIRE(written_bytes == size); + MPI_File_close(&mpi_fh); MPI_Barrier(MPI_COMM_WORLD); - return 0; + if (info.rank == 0) { + printf("%d ready for attach\n", info.comm_size); + fflush(stdout); + sleep(120); + } + info.debug_init = true; + } + info.num_nodes = info.comm_size / args.process_per_node; + info.num_brokers = info.num_nodes * args.brokers_per_node; + flux_get_rank(info.flux_handle, &info.broker_idx); + flux_get_size(info.flux_handle, &info.broker_size); + MPI_Barrier(MPI_COMM_WORLD); + return 0; } int posttest() { - MPI_Barrier(MPI_COMM_WORLD); - return 0; + MPI_Barrier(MPI_COMM_WORLD); + return 0; } int clean_directories() { - if (info.rank % args.process_per_node == 0) { - auto file_pt = args.pfs.string() + "/" + args.filename; - std::string cmd = "rm -rf " + file_pt + "*"; - int status = system (cmd.c_str ()); - (void) status; - file_pt = args.dyad_managed_dir.string() + "/" + args.filename; - cmd = "rm -rf " + file_pt + "*"; - status = system (cmd.c_str ()); - (void) status; - } - MPI_Barrier(MPI_COMM_WORLD); - return 0; + if (info.rank % args.process_per_node == 0) { + auto file_pt = args.pfs.string() + "/" + args.filename; + std::string cmd = "rm -rf " + file_pt + "*"; + int status = system(cmd.c_str()); + (void)status; + file_pt = args.dyad_managed_dir.string() + "/" + args.filename; + cmd = "rm -rf " + file_pt + "*"; + status = system(cmd.c_str()); + (void)status; + } + MPI_Barrier(MPI_COMM_WORLD); + return 0; } #include "data_plane/data_plane.cpp" #include "mdm/mdm.cpp" From 77bc3eb958568dab95a58668459504849ca4787c Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Mon, 25 Mar 2024 15:40:24 -0700 Subject: [PATCH 36/47] Current state of DataSpaces test --- tests/dspaces_perf/data_plane/data_plane.cpp | 200 ++++++++++++------- tests/dspaces_perf/unit_test.cpp | 62 +++++- 2 files changed, 179 insertions(+), 83 deletions(-) diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp index 2213aec3..e14898cd 100644 --- a/tests/dspaces_perf/data_plane/data_plane.cpp +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -5,6 +5,9 @@ #include +#define NS_TO_SECS(ns_var) ((double) ns_var / 1000000000.0) +#define AGG_TIME(time_var, agg_time_var, dtype) MPI_Reduce(&time_var, &agg_time_var, 1, dtype, MPI_SUM, 0, MPI_COMM_WORLD) + FILE* redirect_stdout(const char* filename) { size_t dir_len = strlen(args.dspaces_timing_dir.c_str()); @@ -87,17 +90,38 @@ int create_files_per_server_process(dspaces_client_t* client, bool is_local, boo return rc; } +void gen_perf_print(size_t data_len, double total_mdata_time, double total_data_time) +{ + double agg_mdata_time = 0; + double agg_data_time = 0; + AGG_TIME(total_mdata_time, agg_mdata_time, MPI_DOUBLE); + AGG_TIME(total_data_time, agg_data_time, MPI_DOUBLE); + if (info.rank == 0) { + double final_mdata_time = agg_mdata_time / info.comm_size; + double final_data_time = agg_data_time / info.comm_size; + printf("[DSPACES_TEST],%10d,%10lu,%10lu,%10.6f,%10.6f,%10.6f,%10.6f\n", + info.comm_size, // Comm Size + data_len*args.number_of_files, // Total I/O per process + args.number_of_files, // Number of mdata ops per process + final_mdata_time, // Metadata Time + final_data_time, // Data Time + data_len*args.number_of_files*info.comm_size/final_mdata_time/1024/1024.0, // Metadata Bandwidth + args.number_of_files*info.comm_size/final_data_time/1024/1024.0 // Data Bandwidth + ); + } +} + TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, false, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_server_process(&client, false, true) == 0); char filename[4096]; gen_var_name(filename, false, true, false, true); size_t data_len = args.request_size*args.iteration; @@ -105,42 +129,48 @@ TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_file int ndim = 1; uint64_t lb = 0; uint64_t ub = data_len - 1; - FILE* fp = redirect_stdout("remote_data_bandwidth.csv"); - REQUIRE(fp != NULL); - printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); + char csv_filename[4096]; + // sprintf(csv_filename, "remote_data_bandwidth_%d.csv", info.rank); + // FILE* fp = fopen(csv_filename, "w+"); + // FILE* fp = redirect_stdout(csv_filename); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } - restore_stdout(fp); - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + rc = dspaces_fini(client); + REQUIRE (rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE (posttest() == 0); } - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); } TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, false, false) == 0); SECTION("Test Max Bandwidth") { Timer data_time; + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_server_process(&client, false, false) == 0); char filename[4096]; gen_var_name(filename, false, false, false, true); char* file_data = NULL; @@ -148,32 +178,38 @@ TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_f int ndim = 1; uint64_t lb = 0; uint64_t ub = data_len - 1; - FILE* fp = redirect_stdout("remote_data_agg_bandwidth.csv"); - REQUIRE(fp != NULL); - printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); + char csv_filename[4096]; + // sprintf(csv_filename, "remote_data_agg_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; if (info.rank % args.process_per_node != 0) usleep (10000); for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Unlike the previous test, we set the timeout to -1 so it will do any blocking that it might want to do // TODO: confirm that the timeout is actually needed to guarantee this type of behavior - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } - restore_stdout(fp); - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + rc = dspaces_fini(client); + REQUIRE (rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE (posttest() == 0); } - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); } @@ -181,13 +217,13 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_server_process(&client, true, true) == 0); char filename[4096]; gen_var_name(filename, true, false, false, false); size_t data_len = args.request_size*args.iteration; @@ -195,29 +231,35 @@ TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_o uint64_t lb = 0; uint64_t ub = data_len - 1; char* file_data = NULL; - FILE* fp = redirect_stdout("local_process_data_bandwidth.csv"); - REQUIRE(fp != NULL); - printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); + char csv_filename[4096]; + // sprintf(csv_filename, "local_process_data_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } - restore_stdout(fp); - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + rc = dspaces_fini(client); + REQUIRE (rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE (posttest() == 0); } - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); } @@ -225,13 +267,13 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" "[parallel_req= " + std::to_string(info.comm_size) +"]" "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, true, true) == 0); SECTION("Test Max Bandwidth") { Timer data_time; + REQUIRE (pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE (rc == dspaces_SUCCESS); + REQUIRE (create_files_per_server_process(&client, true, true) == 0); char filename[4096]; gen_var_name(filename, true, false, true, false); size_t data_len = args.request_size*args.iteration; @@ -239,27 +281,33 @@ TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_f uint64_t lb = 0; uint64_t ub = data_len - 1; char* file_data = NULL; - FILE* fp = redirect_stdout("local_node_data_bandwidth.csv"); - REQUIRE(fp != NULL); - printf("rank,var_name,version,mdata_time_ns,data_time_ns\n"); + char csv_filename[4096]; + // sprintf(csv_filename, "local_node_data_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; data_time.resumeTime(); // Using aget instead of get because dyad_get_data also allocates the buffer // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1); + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); data_time.pauseTime(); REQUIRE (rc == dspaces_SUCCESS); free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } - restore_stdout(fp); - AGGREGATE_TIME(data); - if (info.rank == 0) { - printf("[DSPACES_TEST],%10d,%10lu,%10.6f,%10.6f\n", - info.comm_size, data_len*args.number_of_files, - total_data/info.comm_size, data_len*args.number_of_files*info.comm_size*info.comm_size/total_data/1024/1024.0); - } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + rc = dspaces_fini(client); + REQUIRE (rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE (posttest() == 0); } - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (posttest() == 0); } diff --git a/tests/dspaces_perf/unit_test.cpp b/tests/dspaces_perf/unit_test.cpp index a76ce83f..f67b4f87 100644 --- a/tests/dspaces_perf/unit_test.cpp +++ b/tests/dspaces_perf/unit_test.cpp @@ -12,6 +12,7 @@ struct Info { int comm_size; int num_nodes; size_t num_server_procs; + bool debug_init; }; struct Arguments { std::string dspaces_timing_dir; @@ -39,11 +40,7 @@ int init(int* argc, char*** argv) { MPI_Init(argc, argv); MPI_Comm_rank(MPI_COMM_WORLD, &info.rank); MPI_Comm_size(MPI_COMM_WORLD, &info.comm_size); - if (args.debug && info.rank == 0) { - printf("%d ready for attach\n", info.comm_size); - fflush(stdout); - getchar(); - } + info.debug_init = false; MPI_Barrier(MPI_COMM_WORLD); return 0; } @@ -66,15 +63,66 @@ cl::Parser define_options() { "server_ppn")["-s"]["--server_ppn"]("Number of DataSpaces server processes per node") | cl::Opt(args.dspaces_timing_dir, "dspaces_timing_dir")["-t"]["--timing_dir"]("Directory to write DataSpaces internal timings") | - cl::Opt(args.debug, - "debug")["-d"]["--debug"]("debug"); + cl::Opt(args.debug)["-d"]["--debug"]("debug"); } int pretest() { + if (!info.debug_init && args.debug) { + const int HOSTNAME_SIZE = 256; + char hostname[HOSTNAME_SIZE]; + gethostname(hostname, HOSTNAME_SIZE); + int pid = getpid(); + char* start_port_str = getenv("VSC_DEBUG_START_PORT"); + int start_port = 10000; + if (start_port_str != nullptr) { + start_port = atoi(start_port_str); + } + const char* conf_dir = getenv("VSC_DEBUG_CONF_DIR"); + if (conf_dir == nullptr) { + conf_dir = "."; + } + char conf_file[4096]; + sprintf(conf_file, "%s/debug.conf", conf_dir); + + char exe[1024]; + int ret = readlink("/proc/self/exe",exe,sizeof(exe)-1); + REQUIRE(ret !=-1); + exe[ret] = 0; + if (info.rank == 0) { + remove(conf_file); + } + MPI_Barrier(MPI_COMM_WORLD); + MPI_File mpi_fh; + int status_orig = MPI_File_open(MPI_COMM_WORLD, conf_file, MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_INFO_NULL, &mpi_fh); + REQUIRE(status_orig == MPI_SUCCESS); + const int buf_len = 16*1024; + char buffer[buf_len]; + int size; + if (info.rank == 0) { + size = sprintf(buffer, "%d\n%s:%d:%s:%d:%d\n", info.comm_size, exe, info.rank, hostname, start_port+info.rank, pid); + } else { + size = sprintf(buffer, "%s:%d:%s:%d:%d\n", exe, info.rank, hostname, start_port+info.rank, pid); + } + MPI_Status status; + MPI_File_write_ordered(mpi_fh, buffer, size, MPI_CHAR, &status); + int written_bytes; + MPI_Get_count(&status, MPI_CHAR, &written_bytes); + REQUIRE(written_bytes == size); + MPI_File_close(&mpi_fh); + MPI_Barrier(MPI_COMM_WORLD); + if (info.rank == 0) { + printf("%d ready for attach\n", info.comm_size); + fflush(stdout); + sleep(60); + } + info.debug_init = true; + } info.num_nodes = info.comm_size / args.process_per_node; info.num_server_procs = info.num_nodes * args.server_ppn; + MPI_Barrier(MPI_COMM_WORLD); return 0; } + int posttest() { return 0; } From 885209b410817d24935881f755d66fdf3018def8 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Tue, 26 Mar 2024 19:03:54 -0700 Subject: [PATCH 37/47] Adds data preloader and PyTorch data loader for DataSpaces --- .../configs/workload/dspaces_mummi.yaml | 30 +++ .../configs/workload/dspaces_mummi_small.yaml | 30 +++ .../dlio_benchmark/dspaces-setup-env.sh | 54 ++++++ .../dspaces_h5_torch_data_loader.py | 181 ++++++++++++++++++ .../dlio_benchmark/dspaces_preloader.py | 43 +++++ .../dlio_benchmark/dspaces_run_dlio.sh | 25 +++ 6 files changed, 363 insertions(+) create mode 100644 tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml create mode 100644 tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml create mode 100644 tests/integration/dlio_benchmark/dspaces-setup-env.sh create mode 100644 tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py create mode 100644 tests/integration/dlio_benchmark/dspaces_preloader.py create mode 100755 tests/integration/dlio_benchmark/dspaces_run_dlio.sh diff --git a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml new file mode 100644 index 00000000..c6ccfbad --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml @@ -0,0 +1,30 @@ +model: mummi + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + data_folder: data/mummi/ + format: hdf5 + num_files_train: 600 + num_samples_per_file: 8000 + record_length: 69528 + enable_chunking: True + chunk_size: 17799168 + +reader: + data_loader: pytorch + batch_size: 256 + read_threads: 6 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: dspaces_h5_torch_data_loader.DspacesH5TorchDataLoader + data_loader_sampler: index + +train: + epochs: 10 + computation_time: .133 diff --git a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml new file mode 100644 index 00000000..d7106b0b --- /dev/null +++ b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml @@ -0,0 +1,30 @@ +model: mummi + +framework: pytorch + +workflow: + generate_data: False + train: True + +dataset: + data_folder: data/mummi/ + format: hdf5 + num_files_train: 1 + num_samples_per_file: 100 + record_length: 69528 + enable_chunking: True + chunk_size: 17799168 + +reader: + data_loader: pytorch + batch_size: 1 + read_threads: 2 + file_shuffle: seed + sample_shuffle: seed + multiprocessing_context: spawn + data_loader_classname: dspaces_h5_torch_data_loader.DspacesH5TorchDataLoader + data_loader_sampler: index + +train: + epochs: 10 + computation_time: .133 diff --git a/tests/integration/dlio_benchmark/dspaces-setup-env.sh b/tests/integration/dlio_benchmark/dspaces-setup-env.sh new file mode 100644 index 00000000..4fb2108c --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces-setup-env.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# TODO make sure python mod matches Spack environment if there is python in there +module load python/3.9.12 +module load openmpi/4.1.2 + +# Configurations +export DLIO_WORKLOAD=dspaces_mummi #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi +export NUM_NODES=64 # 1 for small config, 64 for full config +export PPN=8 # 1 for small config, 64 for full config +export QUEUE=pbatch +export TIME=$((60)) +export BROKERS_PER_NODE=1 +export GENERATE_DATA="0" + +export DYAD_INSTALL_PREFIX=/usr/workspace/haridev/dyad/env/spack/.spack-env/view +export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad +export SPACK_DIR=/usr/workspace/haridev/spack +export SPACK_ENV=/usr/workspace/haridev/dyad/env/spack +export PYTHON_ENV=/usr/workspace/haridev/dyad/env/python +export DLIO_DATA_DIR=/p/lustre1/iopp/dyad/dlio_benchmark/dyad_mummi # dyad_resnet50 dyad_unet3d_basic +#export DLIO_DATA_DIR=/p/lustre2/haridev/dyad/dlio_benchmark/dyad_unet3d_basic # dyad_resnet50 + +# DLIO Profiler Configurations +export DLIO_PROFILER_ENABLE=0 +export DLIO_PROFILER_INC_METADATA=1 +export DLIO_PROFILER_DATA_DIR=${DLIO_DATA_DIR}:${DYAD_PATH} +export DLIO_PROFILER_LOG_FILE=/usr/workspace/haridev/dyad/tests/integration/dlio_benchmark/profiler/dyad +export DLIO_PROFILER_LOG_LEVEL=ERROR +#export GOTCHA_DEBUG=3 + +export DLIO_PROFILER_BIND_SIGNALS=0 +export MV2_BCAST_HWLOC_TOPOLOGY=0 +export HDF5_USE_FILE_LOCKING=0 + +#mkdir -p ${DYAD_PATH} +mkdir -p ${DLIO_PROFILER_LOG_FILE} +# Activate Environments +. ${SPACK_DIR}/share/spack/setup-env.sh +spack env activate -p ${SPACK_ENV} +source ${PYTHON_ENV}/bin/activate + +# Derived Configurations +export DYAD_DLIO_RUN_LOG=dyad_${DLIO_WORKLOAD}_${NUM_NODES}_${PPN}_${BROKERS_PER_NODE}.log +export CONFIG_ARG="--config-dir=${GITHUB_WORKSPACE}/tests/integration/dlio_benchmark/configs" +#export CONFIG_ARG="" + +# Derived PATHS +export PATH=${PATH}:${DYAD_INSTALL_PREFIX}/bin:${DYAD_INSTALL_PREFIX}/sbin +export LD_LIBRARY_PATH=/usr/lib64:${DYAD_INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH} +export PYTHONPATH=${GITHUB_WORKSPACE}/tests/integration/dlio_benchmark:$PYTHONPATH + +unset LUA_PATH +unset LUA_CPATH diff --git a/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py b/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py new file mode 100644 index 00000000..02fbe2e1 --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py @@ -0,0 +1,181 @@ +""" + Copyright (c) 2022, UChicago Argonne, LLC + All Rights Reserved + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +from time import time +import logging +import math +import pickle +import torch +from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler + +from dlio_benchmark.common.constants import MODULE_DATA_LOADER +from dlio_benchmark.common.enumerations import Shuffle, DatasetType, DataLoaderType +from dlio_benchmark.data_loader.base_data_loader import BaseDataLoader +from dlio_benchmark.reader.reader_factory import ReaderFactory +from dlio_benchmark.utils.utility import utcnow, DLIOMPI +from dlio_benchmark.utils.config import ConfigArguments +from dlio_profiler.logger import fn_interceptor as Profile + +import dspaces +import numpy as np +import flux +import os +import h5py +import fcntl +dlp = Profile(MODULE_DATA_LOADER) + + +class DspacesH5TorchDataset(Dataset): + """ + Currently, we only support loading one sample per file + TODO: support multiple samples per file + """ + @dlp.log_init + def __init__(self, format_type, dataset_type, epoch, num_samples, num_workers, batch_size): + self.format_type = format_type + self.dataset_type = dataset_type + self.epoch_number = epoch + self.num_samples = num_samples + self.reader = None + self.num_images_read = 0 + self.batch_size = batch_size + args = ConfigArguments.get_instance() + self.img_dim = args.dimension + self.serial_args = pickle.dumps(args) + self.dlp_logger = None + if num_workers == 0: + self.worker_init(-1) + + @dlp.log + def worker_init(self, worker_id): + pickle.loads(self.serial_args) + self._args = ConfigArguments.get_instance() + self._args.configure_dlio_logging(is_child=True) + self.dlp_logger = self._args.configure_dlio_profiler(is_child=True, use_pid=True) + logging.debug(f"{utcnow()} worker initialized {worker_id} with format {self.format_type}") + self.reader = ReaderFactory.get_reader(type=self.format_type, + dataset_type=self.dataset_type, + thread_index=worker_id, + epoch_number=self.epoch_number) + proc_rank = os.getpid() + self.ds_client = dspaces.dspaces(rank=proc_rank) + + def __del__(self): + if self.dlp_logger: + self.dlp_logger.finalize() + # Manually invoke finalizer for DataSpaces to ensure it is shutdown properly + if self.ds_client: + del self.ds_client + self.ds_client = None + + @dlp.log + def __len__(self): + return self.num_samples + + @dlp.log + def __getitem__(self, image_idx): + logging.debug(f"{utcnow()} Rank {DLIOMPI.get_instance().rank()} reading {image_idx} image") + self.num_images_read += 1 + step = int(math.ceil(self.num_images_read / self.batch_size)) + filename, sample_index = self._args.global_index_map[image_idx] + lb = tuple([sample_index, 0, 0]) + ub = tuple([sample_index, self.img_dim-1, self.img_dim-1]) + dlp.update(args={"fname":filename}) + dlp.update(args={"image_idx":image_idx}) + dlp.update(args={"version":0}) + dlp.update(args={"lb":lb}) + dlp.update(args={"ub":ub}) + data = self.ds_client.get( + filename, # variable name + 0, # variable version + lb, # lower bound in global dims + ub, # upper bound in global dims + np.uint8, # NumPy datatype of elements + -1 # timeout + ) + dlp.update(step=step) + dlp.update(image_size=data.nbytes) + return data + +class DspacesH5TorchDataLoader(BaseDataLoader): + @dlp.log_init + def __init__(self, format_type, dataset_type, epoch_number): + super().__init__(format_type, dataset_type, epoch_number, DataLoaderType.PYTORCH) + + @dlp.log + def read(self): + do_shuffle = True if self._args.sample_shuffle != Shuffle.OFF else False + dataset = DspacesH5TorchDataset(self.format_type, self.dataset_type, self.epoch_number, self.num_samples, self._args.read_threads, self.batch_size) + if do_shuffle: + sampler = RandomSampler(dataset) + else: + sampler = SequentialSampler(dataset) + if self._args.read_threads >= 1: + prefetch_factor = math.ceil(self._args.prefetch_size / self._args.read_threads) + else: + prefetch_factor = self._args.prefetch_size + if prefetch_factor > 0: + if self._args.my_rank == 0: + logging.debug( + f"{utcnow()} Prefetch size is {self._args.prefetch_size}; prefetch factor of {prefetch_factor} will be set to Torch DataLoader.") + else: + prefetch_factor = 2 + if self._args.my_rank == 0: + logging.debug( + f"{utcnow()} Prefetch size is 0; a default prefetch factor of 2 will be set to Torch DataLoader.") + logging.debug(f"{utcnow()} Setup dataloader with {self._args.read_threads} workers {torch.__version__}") + if self._args.read_threads==0: + kwargs={} + else: + kwargs={'multiprocessing_context':self._args.multiprocessing_context, + 'prefetch_factor': prefetch_factor} + if torch.__version__ != '1.3.1': + kwargs['persistent_workers'] = True + if torch.__version__ == '1.3.1': + if 'prefetch_factor' in kwargs: + del kwargs['prefetch_factor'] + self._dataset = DataLoader(dataset, + batch_size=self.batch_size, + sampler=sampler, + num_workers=self._args.read_threads, + pin_memory=True, + drop_last=True, + worker_init_fn=dataset.worker_init, + **kwargs) + else: + self._dataset = DataLoader(dataset, + batch_size=self.batch_size, + sampler=sampler, + num_workers=self._args.read_threads, + pin_memory=True, + drop_last=True, + worker_init_fn=dataset.worker_init, + **kwargs) # 2 is the default value + logging.debug(f"{utcnow()} Rank {self._args.my_rank} will read {len(self._dataset) * self.batch_size} files") + + # self._dataset.sampler.set_epoch(epoch_number) + + @dlp.log + def next(self): + super().next() + total = self._args.training_steps if self.dataset_type is DatasetType.TRAIN else self._args.eval_steps + logging.debug(f"{utcnow()} Rank {self._args.my_rank} should read {total} batches") + for batch in self._dataset: + yield batch + + @dlp.log + def finalize(self): + pass diff --git a/tests/integration/dlio_benchmark/dspaces_preloader.py b/tests/integration/dlio_benchmark/dspaces_preloader.py new file mode 100644 index 00000000..67c0b285 --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces_preloader.py @@ -0,0 +1,43 @@ +from mpi4py import MPI +from pathlib import Path +import h5py +import dspaces + +import argparse + +VALID_HDF5_EXTS = [".hdf5", ".h5"] + + +def collect_file_names(dirname): + return list(sorted([f for f in dirname.iterdir() if f.is_file() and f.suffix() in VALID_HDF5_EXTS])) + + +def get_local_files(all_files, rank, comm_size): + return [all_files[i] for i in range(rank, len(all_files), comm_size)] + + +def store_samples_for_file(ds_client, local_file): + with h5py.File(str(local_file), "r") as f: + data = f["records"][:] + offset = tuple([0] * len(data.shape)) + ds_client.put(data, str(local_file), 0, offset) + + +def main(): + parser = argparse.ArgumentParser("Preload data for MuMMI training") + parser.add_argument("data_dir", type=Path, + help="Path to the directory containing HDF5 files") + args = parser.parse_args() + data_dir = args.data_dir.expanduser().resolve() + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + comm_size = comm.Get_size() + ds_client = dspaces.dspaces(comm=comm) + all_files = collect_file_names(data_dir) + local_files = get_local_files(all_files, rank, comm_size) + for lf in local_files: + store_samples_for_file(ds_client, lf) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh new file mode 100755 index 00000000..43a98389 --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source ./dspaces-setup-env.sh + +# TODO startup dspaces_server + +if [[ "${GENERATE_DATA}" == "1" ]]; then +# Generate Data for Workload +echo Generating DLIO Dataset +flux submit -o cpu-affinity=off -N $((NUM_NODES*BROKERS_PER_NODE)) --tasks-per-node=$((PPN/BROKERS_PER_NODE)) dlio_benchmark ${CONFIG_ARG} workload=${DLIO_WORKLOAD} ++workload.dataset.data_folder=${DLIO_DATA_DIR} ++workload.workflow.generate_data=True ++workload.workflow.train=False +GEN_PID=$(flux job last) +flux job attach ${GEN_PID} +echo "Run without Gen data to do training" +exit +fi + +# Run Training +echo Running DLIO Training for ${DLIO_WORKLOAD} +flux submit -N $((NUM_NODES*BROKERS_PER_NODE)) -o cpu-affinity=on --tasks-per-node=$((PPN/BROKERS_PER_NODE)) dlio_benchmark ${CONFIG_ARG} workload=${DLIO_WORKLOAD} ++workload.dataset.data_folder=${DLIO_DATA_DIR} ++workload.workflow.generate_data=False ++workload.workflow.train=True +RUN_PID=$(flux job last) +flux job attach ${RUN_PID} > ${DYAD_DLIO_RUN_LOG} 2>&1 +#cat ${DYAD_DLIO_RUN_LOG} +echo "Finished Executing check ${DYAD_DLIO_RUN_LOG} for output" + + +# TODO run terminate From 7edc5a6863c07e08fec2c8a7273345b8cea4d53f Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 27 Mar 2024 09:55:17 -0700 Subject: [PATCH 38/47] Updates the setup-env and run scripts for DLIO and DataSpaces --- .../dlio_benchmark/dspaces-setup-env.sh | 25 ++++++++------- .../dlio_benchmark/dspaces_run_dlio.sh | 31 ++++++++++++++++--- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/tests/integration/dlio_benchmark/dspaces-setup-env.sh b/tests/integration/dlio_benchmark/dspaces-setup-env.sh index 4fb2108c..96d0ec15 100644 --- a/tests/integration/dlio_benchmark/dspaces-setup-env.sh +++ b/tests/integration/dlio_benchmark/dspaces-setup-env.sh @@ -7,25 +7,26 @@ module load openmpi/4.1.2 # Configurations export DLIO_WORKLOAD=dspaces_mummi #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi export NUM_NODES=64 # 1 for small config, 64 for full config -export PPN=8 # 1 for small config, 64 for full config +export PPN=8 # 1 for small config, 8 for full config export QUEUE=pbatch export TIME=$((60)) export BROKERS_PER_NODE=1 export GENERATE_DATA="0" - -export DYAD_INSTALL_PREFIX=/usr/workspace/haridev/dyad/env/spack/.spack-env/view -export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad -export SPACK_DIR=/usr/workspace/haridev/spack -export SPACK_ENV=/usr/workspace/haridev/dyad/env/spack -export PYTHON_ENV=/usr/workspace/haridev/dyad/env/python -export DLIO_DATA_DIR=/p/lustre1/iopp/dyad/dlio_benchmark/dyad_mummi # dyad_resnet50 dyad_unet3d_basic +export DSPACES_HG_STRING="ofi+verbs" + +export GITHUB_WORKSPACE=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad +export SPACK_DIR=/g/g90/lumsden1/ws/spack +export SPACK_ENV=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env +export SPACK_VIEW=$SPACK_ENV/.spack-env/view +export PYTHON_ENV=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/mummi_venv +export DLIO_DATA_DIR=/p/lustre1/lumsden1/dyad_mummi # dyad_resnet50 dyad_unet3d_basic #export DLIO_DATA_DIR=/p/lustre2/haridev/dyad/dlio_benchmark/dyad_unet3d_basic # dyad_resnet50 # DLIO Profiler Configurations export DLIO_PROFILER_ENABLE=0 export DLIO_PROFILER_INC_METADATA=1 export DLIO_PROFILER_DATA_DIR=${DLIO_DATA_DIR}:${DYAD_PATH} -export DLIO_PROFILER_LOG_FILE=/usr/workspace/haridev/dyad/tests/integration/dlio_benchmark/profiler/dyad +export DLIO_PROFILER_LOG_FILE=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad/tests/integration/dlio_benchmark/profiler/dspaces export DLIO_PROFILER_LOG_LEVEL=ERROR #export GOTCHA_DEBUG=3 @@ -41,13 +42,13 @@ spack env activate -p ${SPACK_ENV} source ${PYTHON_ENV}/bin/activate # Derived Configurations -export DYAD_DLIO_RUN_LOG=dyad_${DLIO_WORKLOAD}_${NUM_NODES}_${PPN}_${BROKERS_PER_NODE}.log +export DSPACES_DLIO_RUN_LOG=dyad_${DLIO_WORKLOAD}_${NUM_NODES}_${PPN}_${BROKERS_PER_NODE}.log export CONFIG_ARG="--config-dir=${GITHUB_WORKSPACE}/tests/integration/dlio_benchmark/configs" #export CONFIG_ARG="" # Derived PATHS -export PATH=${PATH}:${DYAD_INSTALL_PREFIX}/bin:${DYAD_INSTALL_PREFIX}/sbin -export LD_LIBRARY_PATH=/usr/lib64:${DYAD_INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH} +export PATH=${PATH}:${SPACK_VIEW}/bin:${SPACK_VIEW}/sbin +export LD_LIBRARY_PATH=/usr/lib64:${SPACK_VIEW}/lib:${SPACK_VIEW}/lib64:${LD_LIBRARY_PATH} export PYTHONPATH=${GITHUB_WORKSPACE}/tests/integration/dlio_benchmark:$PYTHONPATH unset LUA_PATH diff --git a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh index 43a98389..01c4df04 100755 --- a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh +++ b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh @@ -1,7 +1,23 @@ #!/bin/bash source ./dspaces-setup-env.sh -# TODO startup dspaces_server +# Startup dspaces_server +echo "## Config file for DataSpaces server +ndim = 3 +dims = x +max_versions = 1 +num_apps = 1" > dataspaces.conf + +flux submit -N $NUM_NODES --cores=$(( NUM_NODES*16 )) \ + --tasks-per-node=$BROKERS_PER_NODE dspaces_server $DSPACES_HG_STRING + +# Wait for DataSpaces's server to create conf.ds +sleep 1s +while [ ! -f conf.ds ]; do + sleep 1s +done +# Give the server enough time to write the contents +sleep 3s if [[ "${GENERATE_DATA}" == "1" ]]; then # Generate Data for Workload @@ -13,13 +29,20 @@ echo "Run without Gen data to do training" exit fi +# Preload data into DataSpaces +flux run -N $NUM_NODES --tasks-per-node=1 python3 ./dspaces_preloader.py $DLIO_DATA_DIR/train + # Run Training echo Running DLIO Training for ${DLIO_WORKLOAD} -flux submit -N $((NUM_NODES*BROKERS_PER_NODE)) -o cpu-affinity=on --tasks-per-node=$((PPN/BROKERS_PER_NODE)) dlio_benchmark ${CONFIG_ARG} workload=${DLIO_WORKLOAD} ++workload.dataset.data_folder=${DLIO_DATA_DIR} ++workload.workflow.generate_data=False ++workload.workflow.train=True +flux submit -N $NUM_NODES --cores=$(( NUM_NODES*32 )) -o cpu-affinity=on --tasks-per-node=$PPN \ + dlio_benchmark ${CONFIG_ARG} workload=${DLIO_WORKLOAD} \ + ++workload.dataset.data_folder=${DLIO_DATA_DIR} \ + ++workload.workflow.generate_data=False ++workload.workflow.train=True RUN_PID=$(flux job last) -flux job attach ${RUN_PID} > ${DYAD_DLIO_RUN_LOG} 2>&1 +flux job attach ${RUN_PID} > ${DSPACES_DLIO_RUN_LOG} 2>&1 #cat ${DYAD_DLIO_RUN_LOG} -echo "Finished Executing check ${DYAD_DLIO_RUN_LOG} for output" +echo "Finished Executing check ${DSPACES_DLIO_RUN_LOG} for output" +flux run --ntasks=1 terminator # TODO run terminate From 5faf27c9c6e33235cf0d221aeed4f5fea829eb64 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 27 Mar 2024 09:56:10 -0700 Subject: [PATCH 39/47] Adds a utility Python application to make the Python bindings of DataSpaces visible to a venv --- .../dlio_benchmark/add_dspaces_to_env.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/integration/dlio_benchmark/add_dspaces_to_env.py diff --git a/tests/integration/dlio_benchmark/add_dspaces_to_env.py b/tests/integration/dlio_benchmark/add_dspaces_to_env.py new file mode 100644 index 00000000..980927d3 --- /dev/null +++ b/tests/integration/dlio_benchmark/add_dspaces_to_env.py @@ -0,0 +1,39 @@ +import sysconfig +import site +from pathlib import Path +import argparse +import re + + +def get_sitepackages_for_dspaces(): + env_sitepackages = sysconfig.get_path("platlib") + match_obj = re.match(r"^.*(?Plib/.*)$", env_sitepackages) + if match_obj is None: + raise FileNotFoundError("Could not locate site-packages for venv") + return match_obj.group("site") + + +def main(): + parser = argparse.ArgumentParser("Adds DataSpaces's Python bindings to venv") + parser.add_argument("dspaces_install_prefix", type=Path, + help="Path to the DataSpaces install") + parser.add_argument("--dspaces_sitepackages_dir", "-d", type=str, + default=get_sitepackages_for_dspaces(), + help="Override default path from DataSpaces install prefix to Python bindings") + parser.add_argument("--venv_sitepackages_dir", "-v", type=Path, + default=Path(site.getsitepackages()[0]).expanduser().resolve(), + help="Override path to venv's site-packages directory") + parser.add_argument("--pth_filename", "-p", type=str, + default="dspaces.pth", + help="Override the default name of the pth file that will be creating") + args = parser.parse_args() + dspaces_install_prefix = args.dspaces_install_prefix.expanduser().resolve() + dspaces_sitepackages_dir = dspaces_install_prefix / args.dspaces_sitepackages_dir + pth_file_contents = str(dspaces_sitepackages_dir) + "\n" + full_pth_filename = args.venv_sitepackages_dir / args.pth_filename + with open(str(full_pth_filename), "w") as f: + f.write(pth_file_contents) + + +if __name__ == "__main__": + main() From 6d9a3e57087ffde6ce88e228c94554eea4c3109b Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 27 Mar 2024 10:03:35 -0700 Subject: [PATCH 40/47] Adds cleanup of dataspaces.conf and conf.ds --- tests/integration/dlio_benchmark/dspaces-setup-env.sh | 2 ++ tests/integration/dlio_benchmark/dspaces_run_dlio.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/dlio_benchmark/dspaces-setup-env.sh b/tests/integration/dlio_benchmark/dspaces-setup-env.sh index 96d0ec15..fa8badf1 100644 --- a/tests/integration/dlio_benchmark/dspaces-setup-env.sh +++ b/tests/integration/dlio_benchmark/dspaces-setup-env.sh @@ -14,6 +14,8 @@ export BROKERS_PER_NODE=1 export GENERATE_DATA="0" export DSPACES_HG_STRING="ofi+verbs" +export MAX_DIM_LENGTH_FOR_FILES=8000 + export GITHUB_WORKSPACE=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad export SPACK_DIR=/g/g90/lumsden1/ws/spack export SPACK_ENV=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env diff --git a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh index 01c4df04..051e0e4c 100755 --- a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh +++ b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh @@ -4,7 +4,7 @@ source ./dspaces-setup-env.sh # Startup dspaces_server echo "## Config file for DataSpaces server ndim = 3 -dims = x +dims = $MAX_DIM_LENGTH_FOR_FILES max_versions = 1 num_apps = 1" > dataspaces.conf @@ -45,4 +45,4 @@ echo "Finished Executing check ${DSPACES_DLIO_RUN_LOG} for output" flux run --ntasks=1 terminator -# TODO run terminate +rm dataspaces.conf conf.ds From 93d65162b1d714092425e6eff886b18198e81990 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 27 Mar 2024 10:11:20 -0700 Subject: [PATCH 41/47] Adds a script to run DLIO w/ DataSpaces through flux batch --- tests/integration/dlio_benchmark/dspaces-setup-env.sh | 6 +++--- tests/integration/dlio_benchmark/dspaces_batch.sh | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 tests/integration/dlio_benchmark/dspaces_batch.sh diff --git a/tests/integration/dlio_benchmark/dspaces-setup-env.sh b/tests/integration/dlio_benchmark/dspaces-setup-env.sh index fa8badf1..3787a483 100644 --- a/tests/integration/dlio_benchmark/dspaces-setup-env.sh +++ b/tests/integration/dlio_benchmark/dspaces-setup-env.sh @@ -5,9 +5,9 @@ module load python/3.9.12 module load openmpi/4.1.2 # Configurations -export DLIO_WORKLOAD=dspaces_mummi #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi -export NUM_NODES=64 # 1 for small config, 64 for full config -export PPN=8 # 1 for small config, 8 for full config +export DLIO_WORKLOAD=dspaces_mummi_small #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi +export NUM_NODES=1 # 1 for small config, 64 for full config +export PPN=1 # 1 for small config, 8 for full config export QUEUE=pbatch export TIME=$((60)) export BROKERS_PER_NODE=1 diff --git a/tests/integration/dlio_benchmark/dspaces_batch.sh b/tests/integration/dlio_benchmark/dspaces_batch.sh new file mode 100644 index 00000000..43344af5 --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces_batch.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source ./dspaces-setup-env.sh + +flux batch -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + ./dspaces_run_dlio.sh \ No newline at end of file From 7524c08b8ee79c5b9feed074e609eb266a3d2b5b Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 27 Mar 2024 23:04:10 -0700 Subject: [PATCH 42/47] Bug fix through Unit Test 1. Found that we were doing the check too soon 2. Also for strings > 256 we were failing. --- src/dyad/core/dyad_core.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/dyad/core/dyad_core.c b/src/dyad/core/dyad_core.c index d3d22b3b..44d2f953 100644 --- a/src/dyad/core/dyad_core.c +++ b/src/dyad/core/dyad_core.c @@ -44,13 +44,17 @@ DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, uint32_t hash[4] = {0u}; // Output for the hash size_t cx = 0ul; int n = 0; + if (str == NULL || path_key == NULL || len == 0ul) { + DYAD_C_FUNCTION_END(); + return -1; + } size_t str_len = strlen (str); - const char* str_long = str; - - if (str == NULL || path_key == NULL || len == 0ul || str_len == 0ul) { + if (str_len == 0ul) { DYAD_C_FUNCTION_END(); return -1; } + const char* str_long = str; + path_key[0] = '\0'; // Just append the string so that it can be as large as 128 bytes. @@ -75,7 +79,8 @@ DYAD_DLL_EXPORTED int gen_path_key (const char* restrict str, } } n = snprintf (path_key + cx, len - cx, "%s", str); - if (cx + n >= len || n < 0) { + // FIXME: cx + n >= len fails for str_len > 256 + if (n < 0) { DYAD_C_FUNCTION_END(); return -1; } From 2f163ce70922060c80dd797b8bb98ad27de0c4e9 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Wed, 27 Mar 2024 23:05:00 -0700 Subject: [PATCH 43/47] added unit tests for dyad_core gen_path_key Added unit tests for gen_path_key function. --- tests/unit/CMakeLists.txt | 3 +- tests/unit/dyad_core/CMakeLists.txt | 1 + tests/unit/dyad_core/core_functions.cpp | 71 +++++++++++++++++++++++++ tests/unit/unit_test.cpp | 1 + 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 tests/unit/dyad_core/CMakeLists.txt create mode 100644 tests/unit/dyad_core/core_functions.cpp diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 4095c330..fc2654e1 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -32,4 +32,5 @@ add_dependencies(unit_test dyad) add_subdirectory(script) add_subdirectory(data_plane) -add_subdirectory(mdm) \ No newline at end of file +add_subdirectory(mdm) +add_subdirectory(dyad_core) \ No newline at end of file diff --git a/tests/unit/dyad_core/CMakeLists.txt b/tests/unit/dyad_core/CMakeLists.txt new file mode 100644 index 00000000..1be584bf --- /dev/null +++ b/tests/unit/dyad_core/CMakeLists.txt @@ -0,0 +1 @@ +add_test(unit_dyad_core ${CMAKE_BINARY_DIR}/bin/unit_test --reporter mpi_console gen_path_key) \ No newline at end of file diff --git a/tests/unit/dyad_core/core_functions.cpp b/tests/unit/dyad_core/core_functions.cpp new file mode 100644 index 00000000..11125435 --- /dev/null +++ b/tests/unit/dyad_core/core_functions.cpp @@ -0,0 +1,71 @@ + +#include +/** + * Test cases + */ +TEST_CASE("gen_path_key", + "[module=dyad_core]" + "[method=gen_path_key]") { + SECTION("should generate path key") { + const char* str = "test_string"; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == 0); + REQUIRE(strcmp(path_key, "") != 0); + } + SECTION("should_return_minus_one_when_input_string_is_null") { + const char* str = nullptr; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == -1); + } + SECTION("should_handle_input_string_of_length_less_than_128_bytes") { + const char* str = "short_string"; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == 0); + REQUIRE(strcmp(path_key, "") != 0); + } + SECTION("should_handle_input_string_of_length_more_than_128_bytes") { + const char* str = + "this_is_a_very_long_string_that_is_more_than_128_bytes_long_this_is_a_" + "very_long_string_that_is_more_than_128_bytes_long_this_is_a_very_long_" + "string_that_is_more_than_128_bytes_long_this_is_a_very_long_string_" + "that_is_more_than_128_bytes_long_this_is_a_very_long_string_that_is_" + "more_than_128_bytes_long_this_is_a_very_long_string_that_is_more_than_" + "128_bytes_long_this_is_a_very_long_string_that_is_more_than_128_bytes_" + "long_this_is_a_very_long_string_that_is_more_than_128_bytes_long_this_" + "is_a_very_long_string_that_is_more_than_128_bytes_long_this_is_a_very_" + "long_string_that_is_more_than_128_bytes_long"; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == 0); + REQUIRE(strcmp(path_key, "") != 0); + } + SECTION("should_generate_path_key_with_depth_and_width_specified") { + const char* str = "test_string"; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == 0); + REQUIRE(strcmp(path_key, "") != 0); + } + SECTION("should_generate_path_key_with_depth_and_width_set_to_0") { + const char* str = "test_string"; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 0, 0); + REQUIRE(result == 0); + REQUIRE(strcmp(path_key, "") != 0); + } + SECTION("should_return_minus_1_when_input_string_is_NULL") { + const char* str = NULL; + char path_key[256] = {'\0'}; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == -1); + } + SECTION("should_return_minus_1_when_path_key_is_NULL") { + const char* str = "test_string"; + char* path_key = NULL; + int result = gen_path_key(str, path_key, sizeof(path_key), 3, 5); + REQUIRE(result == -1); + } +} \ No newline at end of file diff --git a/tests/unit/unit_test.cpp b/tests/unit/unit_test.cpp index 5b999c37..cb617c46 100644 --- a/tests/unit/unit_test.cpp +++ b/tests/unit/unit_test.cpp @@ -163,4 +163,5 @@ int clean_directories() { return 0; } #include "data_plane/data_plane.cpp" +#include "dyad_core/core_functions.cpp" #include "mdm/mdm.cpp" From 1dae96e7c53c13a9964eae75d5b2b6c0b0250bb1 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 3 Apr 2024 12:55:56 -0700 Subject: [PATCH 44/47] Updates scripts for DataSpaces testing --- scripts/dspaces/aggregate.py | 63 +++++++++++++++++++++++++++ scripts/dspaces/collect.py | 75 ++++++++++++++++++++++++++++++++ scripts/dspaces/corona.sh | 23 +++++++++- scripts/dspaces/dspaces_start.sh | 2 +- scripts/dspaces/dspaces_stop.sh | 2 + scripts/dspaces/print_all.sh | 66 ++++++++++++++++++++++++++++ scripts/dspaces/run.sh | 25 ++++++++--- scripts/dspaces/run_all.sh | 52 ++++++++++++++-------- scripts/dspaces/setup-env.sh | 12 ++--- scripts/dspaces/validate.py | 53 ++++++++++++++++++++++ 10 files changed, 341 insertions(+), 32 deletions(-) create mode 100644 scripts/dspaces/aggregate.py create mode 100644 scripts/dspaces/collect.py mode change 100644 => 100755 scripts/dspaces/dspaces_start.sh mode change 100644 => 100755 scripts/dspaces/dspaces_stop.sh create mode 100755 scripts/dspaces/print_all.sh mode change 100644 => 100755 scripts/dspaces/run_all.sh mode change 100644 => 100755 scripts/dspaces/setup-env.sh create mode 100644 scripts/dspaces/validate.py diff --git a/scripts/dspaces/aggregate.py b/scripts/dspaces/aggregate.py new file mode 100644 index 00000000..ad65f8ab --- /dev/null +++ b/scripts/dspaces/aggregate.py @@ -0,0 +1,63 @@ +from pathlib import Path +import re +import argparse +import pandas as pd + + +def process_single_run_csvs(dir_path): + dirname = dir_path.name + match_obj = re.match(r"(?P[a-zA-Z]+)_(?P[0-9]+)_(?P[0-9]+)", dirname) + if match_obj is None: + raise RuntimeError("Cannot parse directory name") + num_nodes = int(match_obj.group("num_nodes")) + ppn = int(match_obj.group("ppn")) + csv_files = list(dir_path.glob("*.csv")) + df = pd.concat(map(pd.read_csv, csv_files), ignore_index=True) + num_ops = len(df) + df = df.drop(columns=["var_name", "version"]) + df = df.groupby("rank").agg("sum") + return { + "test_name": match_obj.group("test_name"), + "num_nodes": num_nodes, + "ppn": ppn, + "num_mdata_ops": num_ops, + "data_size": df["data_size"].sum(), + "mdata_time_ns": df["mdata_time_ns"].max(), + "data_time_ns": df["data_time_ns"].max(), + } + + +def build_result_dataframe(testdir): + top_level_rundir_name = testdir.parent.name + test_dir_name = testdir.name + output_df_name = "{}_{}.csv".format(top_level_rundir_name, test_dir_name) + print("Building", output_df_name) + df_rows = [] + for subdir in testdir.iterdir(): + if subdir.is_dir(): + print("Getting data for", str(subdir)) + df_row = process_single_run_csvs(subdir) + df_rows.append(df_row) + output_df = pd.DataFrame(data=df_rows) + return output_df_name, output_df + + +def main(): + parser = argparse.ArgumentParser("Aggregate data for test") + parser.add_argument("testdir", type=Path, + help="Path to the test directory to collect data for") + parser.add_argument("--dump_dir", "-d", type=Path, + help="Directory to dump the resulting CSV into") + args = parser.parse_args() + csv_name, df = build_result_dataframe(args.testdir.expanduser().resolve()) + dump_dir = args.dump_dir.expanduser().resolve() + if not dump_dir.is_dir(): + print("Creating non-existant dump directory {}".format(str(dump_dir))) + dump_dir.mkdir(parents=True) + full_csv_name = dump_dir / csv_name + df.to_csv(str(full_csv_name)) + print("Wrote data to {}".format(str(full_csv_name))) + + +if __name__ == "__main__": + main() diff --git a/scripts/dspaces/collect.py b/scripts/dspaces/collect.py new file mode 100644 index 00000000..b42c5f3d --- /dev/null +++ b/scripts/dspaces/collect.py @@ -0,0 +1,75 @@ +from pathlib import Path +import re +import argparse +import json + + +def validate_log(out_file): + with open(str(out_file), "r") as f: + for line in f: + if line.startswith("[DSPACES_TEST]"): + return line + return None + + +def validate_dir(path): + dirname = path.name + match_obj = re.match(r"(?P[a-zA-Z]+)_(?P[0-9]+)_(?P[0-9]+)", dirname) + if match_obj is None: + raise RuntimeError("Cannot parse directory name") + test_name = match_obj.group("test_name") + num_nodes = int(match_obj.group("num_nodes")) + ppn = int(match_obj.group("ppn")) + # num_tasks = num_nodes * ppn + out_file = path / "run.out" + if not out_file.is_file(): + raise RuntimeError("Could not find run.out for {}".format(path)) + perf_line = validate_log(out_file) + if perf_line is None: + raise RuntimeError("Run for {} failed because we don't have perf numbers".format(path)) + return { + "test_name": test_name, + "num_nodes": num_nodes, + "ppn": ppn, + "perf": perf_line, + } + + +def validate_rundir(td): + print("Validating tests in {}:".format(td.name)) + subdirs = [sd for sd in td.iterdir() if sd.is_dir()] + perf_entries = [] + for sd in subdirs: + print(" * Validating {}:".format(sd.name), end=" ") + try: + new_perf = validate_dir(sd) + perf_entries.append(new_perf) + print("GOOD") + except RuntimeError as e: + print("BAD") + raise e + return perf_entries + + +def main(): + parser = argparse.ArgumentParser("Validate runs") + parser.add_argument("testdir", type=Path, + help="Top-level directory representing the results of a single iteration of the testing") + parser.add_argument("--dump_file", "-d", type=Path, default=None, + help="Path to JSON file where we want to dump performance results") + args = parser.parse_args() + perf_entries = validate_rundir(args.testdir.expanduser().resolve()) + if args.dump_file is not None: + dump_file = args.dump_file.expanduser().resolve() + if not dump_file.name.endswith(".json"): + raise ValueError("Invalid file suffix for JSON file") + if not dump_file.parent.is_dir(): + dump_file.parent.mkdir(parents=True) + with open(str(dump_file), "w") as f: + json.dump(perf_entries, f, indent=4, sort_keys=True) + else: + print(json.dumps(perf_entries, sort_keys=True, indent=4)) + + +if __name__ == "__main__": + main() diff --git a/scripts/dspaces/corona.sh b/scripts/dspaces/corona.sh index 97db0a9a..6c5158ec 100755 --- a/scripts/dspaces/corona.sh +++ b/scripts/dspaces/corona.sh @@ -3,12 +3,31 @@ test_case=$1 num_nodes=$2 ppn=$3 +timing_root_dir=$4 +use_alloc=$5 num_iters=16 num_files=16 request_size=65536 -hg_conn_str="ucx+rc" +hg_conn_str="ofi+verbs" + +extra_flux_flags="--setattr=system.bank=ice4hpc" source ./setup-env.sh -flux alloc -q $QUEUE -t $TIME -N $NUM_NODES --exclusive ./run.sh $test_case $num_nodes $ppn $num_iters $num_files $request_size $hg_conn_str +timing_dir=$timing_root_dir/${test_case}_${num_nodes}_${ppn} + +if [ -d $timing_dir ]; then + echo "Dump directory $timing_dir already exists" + exit 1 +fi + +mkdir -p $timing_dir + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +if $use_alloc; then + flux alloc -q $QUEUE -t $TIME -N $num_nodes --exclusive $extra_flux_flags ./run.sh $test_case $num_nodes $ppn $num_iters $num_files $request_size $hg_conn_str $timing_dir $SCRIPT_DIR +else + flux batch -q $QUEUE -t $TIME -N $num_nodes --output=$timing_dir/run.out --error=$timing_dir/run.err --exclusive $extra_flux_flags ./run.sh $test_case $num_nodes $ppn $num_iters $num_files $request_size $hg_conn_str $timing_dir $SCRIPT_DIR +fi diff --git a/scripts/dspaces/dspaces_start.sh b/scripts/dspaces/dspaces_start.sh old mode 100644 new mode 100755 index 036a5ef1..ce69e73a --- a/scripts/dspaces/dspaces_start.sh +++ b/scripts/dspaces/dspaces_start.sh @@ -9,7 +9,7 @@ num_apps = 1" > dataspaces.conf # Use provided number of nodes instead of auto-obtained number # dspaces_num_nodes=$(flux resource info | grep -oP "\d+ Nodes" | grep -oP "^\d+") -flux submit -N $3 --tasks-per-node $4 dspaces_server $5 +flux submit -N $3 --cores=$(( $3*1 )) --tasks-per-node=$4 dspaces_server $5 # Wait for DataSpaces configuration file to be created. # If we don't do this, the DataSpaces clients will either crash or hang diff --git a/scripts/dspaces/dspaces_stop.sh b/scripts/dspaces/dspaces_stop.sh old mode 100644 new mode 100755 index 1d7e0d60..eaa18642 --- a/scripts/dspaces/dspaces_stop.sh +++ b/scripts/dspaces/dspaces_stop.sh @@ -1,3 +1,5 @@ #!/bin/bash flux run --ntasks=1 terminator + +# rm conf.ds dataspaces.conf diff --git a/scripts/dspaces/print_all.sh b/scripts/dspaces/print_all.sh new file mode 100755 index 00000000..a5ff350a --- /dev/null +++ b/scripts/dspaces/print_all.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +timing_root_dir=$1 + +echo "#!/bin/bash" + +ppns=( 1 2 4 8 16 32 64 ) + +echo "# Remote ppn scaling test" +for ppn in ${ppns[@]}; do + # Run 2 node test for RemoteDataBandwidth + echo "./corona.sh RemoteDataBandwidth 2 $ppn $timing_root_dir/2_node_remote_data false" + # Run 2 node test for RemoteDataAggBandwidth + echo "./corona.sh RemoteDataAggBandwidth 2 $ppn $timing_root_dir/2_node_remote_data false" +done +echo "" + +num_nodes=( 4 8 16 32 64 ) +ppns=( 16 ) + +echo "# Num nodes remote scaling" +for nn in ${num_nodes[@]}; do + for ppn in ${ppns[@]}; do + # Run 1 node test for LocalProcessDataBandwidth + echo "./corona.sh RemoteDataBandwidth $nn $ppn $timing_root_dir/multi_node_remote_data false" + # Run 1 node test for LocalNodeDataBandwidth + echo "./corona.sh RemoteDataAggBandwidth $nn $ppn $timing_root_dir/multi_node_remote_data false" + done +done +echo "" + +num_nodes=( 1 2 4 8 16 32 64 ) +echo "# Metadata Perf" +for nn in ${num_nodes[@]}; do + # Process local mdata + echo "./corona.sh LocalProcessDataBandwidth $nn 16 $timing_root_dir/proc_local_mdata false" + # Service mdata + echo "./corona.sh LocalNodeDataBandwidth $nn 16 $timing_root_dir/service_mdata false" + # Global mdata + echo "./corona.sh RemoteDataBandwidth $nn 16 $timing_root_dir/global_mdata false" +done +echo "" + +ppns=( 1 2 4 8 16 32 64 ) + +echo "# Local process ppn scaling" +for ppn in ${ppns[@]}; do + # Run 2 node test for RemoteDataBandwidth + echo "./corona.sh LocalProcessDataBandwidth 1 $ppn $timing_root_dir/1_node_local_data false" + # Run 2 node test for RemoteDataAggBandwidth + echo "./corona.sh LocalNodeDataBandwidth 1 $ppn $timing_root_dir/1_node_local_data false" +done +echo "" + +num_nodes=( 2 4 8 16 32 64 ) +ppns=( 16 ) + +echo "# Local node scaling" +for nn in ${num_nodes[@]}; do + for ppn in ${ppns[@]}; do + # Run 2 node test for RemoteDataBandwidth + echo "./corona.sh LocalProcessDataBandwidth $nn $ppn $timing_root_dir/multi_node_local_data false" + # Run 2 node test for RemoteDataAggBandwidth + echo "./corona.sh LocalNodeDataBandwidth $nn $ppn $timing_root_dir/multi_node_local_data false" + done +done diff --git a/scripts/dspaces/run.sh b/scripts/dspaces/run.sh index 0f5fd2e5..088070cb 100755 --- a/scripts/dspaces/run.sh +++ b/scripts/dspaces/run.sh @@ -5,20 +5,35 @@ PPN=$3 NUM_ITERS=$4 NUM_FILES=$5 REQUEST_SIZE=$6 -HG_CONNECTION_STR="$7" +HG_CONNECTION_STR=$7 +TIMING_DIR=$8 +SCRIPT_DIR=$9 -source ./setup-env.sh +ulimit -c unlimited + +test_cores=$(( NUM_NODES*32 )) + +curr_dir=$(pwd) + +cd $TIMING_DIR + +source $SCRIPT_DIR/setup-env.sh EXEC_DIR="${GITHUB_WORKSPACE}/build/bin" +# export DSPACES_DEBUG=1 + echo Starting DataSpaces # Setup DYAD -./dspaces_start.sh $(( REQUEST_SIZE * NUM_ITERS )) ${NUM_FILES} ${NUM_NODES} ${SERVER_PPN} ${HG_CONNECTION_STR} +$SCRIPT_DIR/dspaces_start.sh $(( REQUEST_SIZE * NUM_ITERS )) ${NUM_FILES} ${NUM_NODES} ${SERVER_PPN} ${HG_CONNECTION_STR} echo "Running Test $WORKLOAD" -flux run -N ${NUM_NODES} --tasks-per-node=${PPN} ${EXEC_DIR}/unit_test --filename dp_${NUM_NODES}_${PPN} \ +flux run -N ${NUM_NODES} --cores=$test_cores --tasks-per-node=${PPN} \ + ${EXEC_DIR}/unit_test --filename dp_${NUM_NODES}_${PPN} \ --ppn ${PPN} --iteration ${NUM_ITERS} --number_of_files ${NUM_FILES} \ --server_ppn ${SERVER_PPN} --request_size ${REQUEST_SIZE} --timing_dir ${TIMING_DIR} ${test_case} echo Stopping DataSpaces -./dspaces_stop.sh +$SCRIPT_DIR/dspaces_stop.sh + +cd $curr_dir diff --git a/scripts/dspaces/run_all.sh b/scripts/dspaces/run_all.sh old mode 100644 new mode 100755 index a5348228..b58885b5 --- a/scripts/dspaces/run_all.sh +++ b/scripts/dspaces/run_all.sh @@ -1,41 +1,55 @@ #!/bin/bash -# Do 2 and 1 node runs for Remote and Local tests with varying ppn +timing_root_dir=$1 + ppns=( 1 2 4 8 16 32 64 ) for ppn in ${ppns[@]}; do # Run 2 node test for RemoteDataBandwidth - ./corona.sh RemoteDataBandwidth 2 $ppn + ./corona.sh RemoteDataBandwidth 2 $ppn $timing_root_dir/2_node_remote_data false # Run 2 node test for RemoteDataAggBandwidth - ./corona.sh RemoteDataAggBandwidth 2 $ppn - # Run 1 node test for LocalProcessDataBandwidth - ./corona.sh LocalProcessDataBandwidth 1 $ppn - # Run 1 node test for LocalNodeDataBandwidth - ./corona.sh LocalNodeDataBandwidth 1 $ppn + ./corona.sh RemoteDataAggBandwidth 2 $ppn $timing_root_dir/2_node_remote_data false done -# Do local tests with varying number of nodes and ppn -num_nodes=( 2 4 8 16 32 64 ) -ppns=( 16 32 64 ) +num_nodes=( 4 8 16 32 64 ) +ppns=( 16 ) for nn in ${num_nodes[@]}; do for ppn in ${ppns[@]}; do # Run 1 node test for LocalProcessDataBandwidth - ./corona.sh LocalProcessDataBandwidth $nn $ppn + ./corona.sh RemoteDataBandwidth $nn $ppn $timing_root_dir/multi_node_remote_data false # Run 1 node test for LocalNodeDataBandwidth - ./corona.sh LocalNodeDataBandwidth $nn $ppn + ./corona.sh RemoteDataAggBandwidth $nn $ppn $timing_root_dir/multi_node_remote_data false done done -# Do remote tests with varying number of nodes and ppn -num_nodes=( 4 8 16 32 64 ) -ppns=( 16 32 64 ) +num_nodes=( 1 2 4 8 16 32 64 ) +for nn in ${num_nodes[@]}; do + # Process local mdata + ./corona.sh LocalProcessDataBandwidth $nn 64 $timing_root_dir/proc_local_mdata false + # Service mdata + ./corona.sh LocalNodeDataBandwidth $nn 64 $timing_root_dir/service_mdata false + # Global mdata + ./corona.sh RemoteDataBandwidth $nn 64 $timing_root_dir/global_mdata false +done + +ppns=( 1 2 4 8 16 32 64 ) + +for ppn in ${ppns[@]}; do + # Run 2 node test for RemoteDataBandwidth + ./corona.sh LocalProcessDataBandwidth 1 $ppn $timing_root_dir/1_node_local_data false + # Run 2 node test for RemoteDataAggBandwidth + ./corona.sh LocalNodeDataBandwidth 1 $ppn $timing_root_dir/1_node_local_data false +done + +num_nodes=( 2 4 8 16 32 64 ) +ppns=( 64 ) for nn in ${num_nodes[@]}; do for ppn in ${ppns[@]}; do - # Run 1 node test for RemoteDataBandwidth - ./corona.sh RemoteDataBandwidth $nn $ppn - # Run 1 node test for RemoteDataAggDataBandwidth - ./corona.sh RemoteDataAggBandwidth $nn $ppn + # Run 2 node test for RemoteDataBandwidth + ./corona.sh LocalProcessDataBandwidth $nn $ppn $timing_root_dir/multi_node_local_data false + # Run 2 node test for RemoteDataAggBandwidth + ./corona.sh LocalNodeDataBandwidth $nn $ppn $timing_root_dir/multi_node_local_data false done done diff --git a/scripts/dspaces/setup-env.sh b/scripts/dspaces/setup-env.sh old mode 100644 new mode 100755 index da67d9eb..6bcf276b --- a/scripts/dspaces/setup-env.sh +++ b/scripts/dspaces/setup-env.sh @@ -1,20 +1,22 @@ #!/bin/bash +export VSC_DEBUG_CONF_DIR=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad/.vscode + module load gcc/10.3.1 module load python/3.9.12 module load openmpi/4.1.2 # Configurations -export QUEUE=pdebug -export TIME=$((60)) +export QUEUE=pbatch +export TIME="15m" export SERVER_PPN=1 -export GITHUB_WORKSPACE=/usr/workspace/haridev/dyad -export SPACK_DIR=/usr/workspace/haridev/spack +export GITHUB_WORKSPACE=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad +export SPACK_DIR=/g/g90/lumsden1/ws/spack export SPACK_ENV=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env export SPACK_VIEW=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/baseline_env/.spack-env/view # Activate Environments -. ${SPACK_DIR}/share/spack/setup-env.sh +source ${SPACK_DIR}/share/spack/setup-env.sh spack env activate -p ${SPACK_ENV} # Derived PATHS diff --git a/scripts/dspaces/validate.py b/scripts/dspaces/validate.py new file mode 100644 index 00000000..dca5ccf1 --- /dev/null +++ b/scripts/dspaces/validate.py @@ -0,0 +1,53 @@ +from pathlib import Path +import re +import argparse + + +NUM_FILES = 16 + + +def validate_csv(csv_file): + with open(str(csv_file), "r") as f: + num_lines = len(f.readlines()) + if num_lines != NUM_FILES + 1: + raise RuntimeError("CSV file {} only contains {} lines, expected {}".format(csv_file, num_lines, NUM_FILES+1)) + + +def validate_dir(path): + dirname = path.name + match_obj = re.match(r"(?P[a-zA-Z]+)_(?P[0-9]+)_(?P[0-9]+)", dirname) + if match_obj is None: + raise RuntimeError("Cannot parse directory name") + num_nodes = int(match_obj.group("num_nodes")) + ppn = int(match_obj.group("ppn")) + num_tasks = num_nodes * ppn + csv_files = list(path.glob("*.csv")) + if len(csv_files) != num_tasks: + raise RuntimeError("Only found {} CSV files, but expected {} for {}".format(len(csv_files), num_tasks, str(path))) + for f in csv_files: + validate_csv(f) + + +def validate_rundir(td): + print("Validating tests in {}:".format(td.name)) + subdirs = [sd for sd in td.iterdir() if sd.is_dir()] + for sd in subdirs: + print(" * Validating {}:".format(sd.name), end=" ") + try: + validate_dir(sd) + print("GOOD") + except RuntimeError as e: + print("BAD") + raise e + + +def main(): + parser = argparse.ArgumentParser("Validate runs") + parser.add_argument("testdir", type=Path, + help="Top-level directory representing the results of a single iteration of the testing") + args = parser.parse_args() + validate_rundir(args.testdir.expanduser().resolve()) + + +if __name__ == "__main__": + main() From 4cef77b9d4a2c3dacbe94de907bbcd24b714664f Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 3 Apr 2024 12:57:32 -0700 Subject: [PATCH 45/47] Final version of DataSpaces baseline code --- tests/dspaces_perf/data_plane/data_plane.cpp | 642 +++++++++++-------- 1 file changed, 359 insertions(+), 283 deletions(-) diff --git a/tests/dspaces_perf/data_plane/data_plane.cpp b/tests/dspaces_perf/data_plane/data_plane.cpp index e14898cd..35fb43ca 100644 --- a/tests/dspaces_perf/data_plane/data_plane.cpp +++ b/tests/dspaces_perf/data_plane/data_plane.cpp @@ -1,313 +1,389 @@ -#include #include +#include #include #include #include -#define NS_TO_SECS(ns_var) ((double) ns_var / 1000000000.0) -#define AGG_TIME(time_var, agg_time_var, dtype) MPI_Reduce(&time_var, &agg_time_var, 1, dtype, MPI_SUM, 0, MPI_COMM_WORLD) +#define NS_TO_SECS(ns_var) ((double)ns_var / 1000000000.0) +#define AGG_TIME(time_var, agg_time_var, dtype) \ + MPI_Reduce(&time_var, &agg_time_var, 1, dtype, MPI_SUM, 0, MPI_COMM_WORLD) -FILE* redirect_stdout(const char* filename) -{ - size_t dir_len = strlen(args.dspaces_timing_dir.c_str()); - bool ends_with_separator = (args.dspaces_timing_dir.c_str()[dir_len-1] == '/'); - size_t filename_len = dir_len + strlen(filename) + 1; - if (!ends_with_separator) { - filename_len += 1; - } - char* full_filename = (char*) malloc(filename_len * sizeof(char)); - memset(full_filename, 0, filename_len*sizeof(char)); - strcpy(full_filename, args.dspaces_timing_dir.c_str()); - if (!ends_with_separator) { - strcat(full_filename, "/"); - } - strcat(full_filename, filename); - FILE* fp = freopen(full_filename, "a", stdout); - free(full_filename); - return fp; +FILE *redirect_stdout(const char *filename) { + size_t dir_len = strlen(args.dspaces_timing_dir.c_str()); + bool ends_with_separator = + (args.dspaces_timing_dir.c_str()[dir_len - 1] == '/'); + size_t filename_len = dir_len + strlen(filename) + 1; + if (!ends_with_separator) { + filename_len += 1; + } + char *full_filename = (char *)malloc(filename_len * sizeof(char)); + memset(full_filename, 0, filename_len * sizeof(char)); + strcpy(full_filename, args.dspaces_timing_dir.c_str()); + if (!ends_with_separator) { + strcat(full_filename, "/"); + } + strcat(full_filename, filename); + FILE *fp = freopen(full_filename, "a", stdout); + free(full_filename); + return fp; } -int restore_stdout(FILE* freopen_fp) -{ - return fclose(freopen_fp); -} +int restore_stdout(FILE *freopen_fp) { return fclose(freopen_fp); } -void gen_var_name(char* filename, bool is_local, bool add_rank_if_remote, bool next_local_rank, bool next_node) { - size_t node_idx = info.rank / args.process_per_node; - size_t local_rank = info.rank % args.process_per_node; - if (next_local_rank) { - local_rank = (local_rank + 1) % args.process_per_node; - } - if (next_node) { - node_idx = (node_idx + 1) % info.num_nodes; - } - size_t server_proc_local_idx = info.rank % args.server_ppn; - size_t global_server_proc_idx = node_idx * args.server_ppn + server_proc_local_idx; - if (is_local) { - size_t global_rank_from_local = node_idx * args.process_per_node + local_rank; - sprintf(filename, "%s_%zu.bat", args.filename.c_str(), global_rank_from_local); +void gen_var_name(char *filename, bool is_local, bool add_rank_if_remote, + bool next_local_rank, bool next_node) { + size_t node_idx = info.rank / args.process_per_node; + size_t local_rank = info.rank % args.process_per_node; + if (next_local_rank) { + local_rank = (local_rank + 1) % args.process_per_node; + } + if (next_node) { + node_idx = (node_idx + 1) % info.num_nodes; + } + size_t server_proc_local_idx = info.rank % args.server_ppn; + size_t global_server_proc_idx = + node_idx * args.server_ppn + server_proc_local_idx; + if (is_local) { + size_t global_rank_from_local = + node_idx * args.process_per_node + local_rank; + sprintf(filename, "%s_%zu.bat", args.filename.c_str(), + global_rank_from_local); + } else { + if (add_rank_if_remote) { + sprintf(filename, "%s_%zu_%zu.bat", args.filename.c_str(), + global_server_proc_idx, local_rank); } else { - if (add_rank_if_remote) { - sprintf (filename, "%s_%zu_%zu.bat", args.filename.c_str(), global_server_proc_idx, local_rank); - } else { - sprintf (filename, "%s_%zu.bat", args.filename.c_str(), global_server_proc_idx); - } + sprintf(filename, "%s_%zu.bat", args.filename.c_str(), + global_server_proc_idx); } + } } -int create_files_per_server_process(dspaces_client_t* client, bool is_local, bool add_rank) { - int rc = 0; - char filename[4096]; - size_t file_size = args.request_size*args.iteration; - gen_var_name (filename, is_local, add_rank, false, false); - // Clients are connected round-robin to server processes on the same node. - // To work out which processes should write "files", we first get a node-local rank for - // each client process by modulo dividing the global rank (info.rank) with the processes per node - // (args.process_per_node). Then, we floor divide that node-local rank by args.server_ppn. This - // floor division should only equal 0 for the first "args.server_ppn" ranks to connect to local - // server processes. Since DataSpaces connects to local processes round-robin, these ranks for which the - // floor division equals 0 are the "first" processes to connect to each local server processes. - bool first_rank_per_server_proc = (info.rank % args.process_per_node) / args.server_ppn == 0; - if (is_local || add_rank || first_rank_per_server_proc) { - std::string rand_data = GenRandom(file_size); - uint64_t lb = 0; - uint64_t ub = rand_data.size()-1; - uint64_t buf_size = ub + 1; - dspaces_define_gdim(*client, filename, 1, &buf_size); - for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { - if (is_local) { - rc = dspaces_put_local(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); - } else { - rc = dspaces_put(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, rand_data.data()); - } - if (rc != 0) { - // TODO log error? - } - } +int create_files_per_server_process(dspaces_client_t *client, bool is_local, + bool add_rank) { + int rc = 0; + char filename[4096]; + size_t file_size = args.request_size * args.iteration; + gen_var_name(filename, is_local, add_rank, false, false); + // Clients are connected round-robin to server processes on the same node. + // To work out which processes should write "files", we first get a node-local + // rank for each client process by modulo dividing the global rank (info.rank) + // with the processes per node (args.process_per_node). Then, we floor divide + // that node-local rank by args.server_ppn. This floor division should only + // equal 0 for the first "args.server_ppn" ranks to connect to local server + // processes. Since DataSpaces connects to local processes round-robin, these + // ranks for which the floor division equals 0 are the "first" processes to + // connect to each local server processes. + bool first_rank_per_server_proc = + (info.rank % args.process_per_node) / args.server_ppn == 0; + int color = (int)(is_local || add_rank || first_rank_per_server_proc); + MPI_Comm split_comm; + int split_rank; + MPI_Comm_split(MPI_COMM_WORLD, color, info.rank, &split_comm); + MPI_Comm_rank(split_comm, &split_rank); + if (is_local || add_rank || first_rank_per_server_proc) { + std::string rand_data = GenRandom(file_size); + uint64_t lb = 0; + uint64_t ub = rand_data.size() - 1; + uint64_t buf_size = ub + 1; + dspaces_define_gdim(*client, filename, 1, &buf_size); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + if (is_local) { + rc = dspaces_put_local(*client, filename, file_idx, sizeof(char), 1, + &lb, &ub, rand_data.data()); + } else { + rc = dspaces_put(*client, filename, file_idx, sizeof(char), 1, &lb, &ub, + rand_data.data()); + } + if (rc != 0) { + // TODO log error? + } + MPI_Barrier(split_comm); + if (split_rank == 0) { + printf("Finished putting file %lu\n", file_idx); + fflush(stdout); + } } - MPI_Barrier (MPI_COMM_WORLD); - return rc; + } + MPI_Barrier(MPI_COMM_WORLD); + if (info.rank == 0) { + printf("Finished putting data into server\n"); + fflush(stdout); + } + MPI_Barrier(MPI_COMM_WORLD); + return rc; } -void gen_perf_print(size_t data_len, double total_mdata_time, double total_data_time) -{ - double agg_mdata_time = 0; - double agg_data_time = 0; - AGG_TIME(total_mdata_time, agg_mdata_time, MPI_DOUBLE); - AGG_TIME(total_data_time, agg_data_time, MPI_DOUBLE); - if (info.rank == 0) { - double final_mdata_time = agg_mdata_time / info.comm_size; - double final_data_time = agg_data_time / info.comm_size; - printf("[DSPACES_TEST],%10d,%10lu,%10lu,%10.6f,%10.6f,%10.6f,%10.6f\n", - info.comm_size, // Comm Size - data_len*args.number_of_files, // Total I/O per process - args.number_of_files, // Number of mdata ops per process - final_mdata_time, // Metadata Time - final_data_time, // Data Time - data_len*args.number_of_files*info.comm_size/final_mdata_time/1024/1024.0, // Metadata Bandwidth - args.number_of_files*info.comm_size/final_data_time/1024/1024.0 // Data Bandwidth - ); - } +void gen_perf_print(size_t data_len, double total_mdata_time, + double total_data_time) { + double agg_mdata_time = 0; + double agg_data_time = 0; + AGG_TIME(total_mdata_time, agg_mdata_time, MPI_DOUBLE); + AGG_TIME(total_data_time, agg_data_time, MPI_DOUBLE); + if (info.rank == 0) { + double final_mdata_time = agg_mdata_time / info.comm_size; + double final_data_time = agg_data_time / info.comm_size; + printf("[DSPACES_TEST],%10d,%10lu,%10lu,%10.6f,%10.6f,%10.6f,%20.6f\n", + info.comm_size, // Comm Size + data_len * args.number_of_files, // Total I/O per process + args.number_of_files, // Number of mdata ops per process + final_data_time, // Data Time + final_mdata_time, // Metadata Time + data_len * args.number_of_files * info.comm_size / final_data_time / + 1024.0 / 1024.0, // Data Bandwidth + // TODO change division to be by 1000 instead of 1024 + args.number_of_files * info.comm_size / + final_mdata_time // Metadata Bandwidth + ); + fflush(stdout); + } } -TEST_CASE("RemoteDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - SECTION("Test Max Bandwidth") { - Timer data_time; - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, false, true) == 0); - char filename[4096]; - gen_var_name(filename, false, true, false, true); - size_t data_len = args.request_size*args.iteration; - char* file_data = NULL; - int ndim = 1; - uint64_t lb = 0; - uint64_t ub = data_len - 1; - char csv_filename[4096]; - // sprintf(csv_filename, "remote_data_bandwidth_%d.csv", info.rank); - // FILE* fp = fopen(csv_filename, "w+"); - // FILE* fp = redirect_stdout(csv_filename); - // REQUIRE(fp != NULL); - // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); - double total_mdata_time = 0; - double total_data_time = 0; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - long long int mdata_time_ns = 0; - long long int data_time_ns = 0; - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - free(file_data); - total_mdata_time += NS_TO_SECS(mdata_time_ns); - total_data_time += NS_TO_SECS(data_time_ns); - } - // restore_stdout(fp); - // AGGREGATE_TIME(data); - gen_perf_print(data_len, total_mdata_time, total_data_time); - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - // fclose(fp); - REQUIRE (posttest() == 0); +TEST_CASE("RemoteDataBandwidth", + "[files= " + std::to_string(args.number_of_files) + + "]" + "[file_size= " + + std::to_string(args.request_size * args.iteration) + + "]" + "[parallel_req= " + + std::to_string(info.comm_size) + + "]" + "[num_nodes= " + + std::to_string(info.comm_size / args.process_per_node) + "]") { + SECTION("Test Max Bandwidth") { + Timer data_time; + REQUIRE(pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE(rc == dspaces_SUCCESS); + REQUIRE(create_files_per_server_process(&client, false, true) == 0); + char filename[4096]; + gen_var_name(filename, false, true, false, true); + size_t data_len = args.request_size * args.iteration; + char *file_data = NULL; + int ndim = 1; + uint64_t lb = 0; + uint64_t ub = data_len - 1; + char csv_filename[4096]; + // sprintf(csv_filename, "remote_data_bandwidth_%d.csv", info.rank); + // FILE* fp = fopen(csv_filename, "w+"); + // FILE* fp = redirect_stdout(csv_filename); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the + // buffer Also, setting timeout to 0 to prevent blocking for data + // availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, + (void **)&file_data, -1, &mdata_time_ns, &data_time_ns); + data_time.pauseTime(); + REQUIRE(rc == dspaces_SUCCESS); + free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Abort(MPI_COMM_WORLD, 0); + rc = dspaces_fini(client); + REQUIRE(rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE(posttest() == 0); + } } -TEST_CASE("RemoteDataAggBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - SECTION("Test Max Bandwidth") { - Timer data_time; - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, false, false) == 0); - char filename[4096]; - gen_var_name(filename, false, false, false, true); - char* file_data = NULL; - size_t data_len = args.request_size*args.iteration; - int ndim = 1; - uint64_t lb = 0; - uint64_t ub = data_len - 1; - char csv_filename[4096]; - // sprintf(csv_filename, "remote_data_agg_bandwidth_%d.csv", info.rank); - // FILE* fp = redirect_stdout(csv_filename); - // FILE* fp = fopen(csv_filename, "w+"); - // REQUIRE(fp != NULL); - // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); - double total_mdata_time = 0; - double total_data_time = 0; - if (info.rank % args.process_per_node != 0) - usleep (10000); - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - long long int mdata_time_ns = 0; - long long int data_time_ns = 0; - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Unlike the previous test, we set the timeout to -1 so it will do any blocking that it might want to do - // TODO: confirm that the timeout is actually needed to guarantee this type of behavior - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - free(file_data); - total_mdata_time += NS_TO_SECS(mdata_time_ns); - total_data_time += NS_TO_SECS(data_time_ns); - } - // restore_stdout(fp); - // AGGREGATE_TIME(data); - gen_perf_print(data_len, total_mdata_time, total_data_time); - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - // fclose(fp); - REQUIRE (posttest() == 0); +TEST_CASE("RemoteDataAggBandwidth", + "[files= " + std::to_string(args.number_of_files) + + "]" + "[file_size= " + + std::to_string(args.request_size * args.iteration) + + "]" + "[parallel_req= " + + std::to_string(info.comm_size) + + "]" + "[num_nodes= " + + std::to_string(info.comm_size / args.process_per_node) + "]") { + SECTION("Test Max Bandwidth") { + Timer data_time; + REQUIRE(pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE(rc == dspaces_SUCCESS); + REQUIRE(create_files_per_server_process(&client, false, false) == 0); + char filename[4096]; + gen_var_name(filename, false, false, false, true); + char *file_data = NULL; + size_t data_len = args.request_size * args.iteration; + int ndim = 1; + uint64_t lb = 0; + uint64_t ub = data_len - 1; + char csv_filename[4096]; + // sprintf(csv_filename, "remote_data_agg_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; + if (info.rank % args.process_per_node != 0) + usleep(10000); + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the + // buffer Unlike the previous test, we set the timeout to -1 so it will do + // any blocking that it might want to do + // TODO: confirm that the timeout is actually needed to guarantee this + // type of behavior + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, + (void **)&file_data, -1, &mdata_time_ns, &data_time_ns); + data_time.pauseTime(); + REQUIRE(rc == dspaces_SUCCESS); + free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + // if (info.rank == 0) { + // printf("[MANUAL_TIMING] total_data = %10.6f\n", total_data); + // fflush(stdout); + // } + gen_perf_print(data_len, total_mdata_time, total_data_time); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Abort(MPI_COMM_WORLD, 0); + rc = dspaces_fini(client); + REQUIRE(rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE(posttest() == 0); + } } - -TEST_CASE("LocalProcessDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - SECTION("Test Max Bandwidth") { - Timer data_time; - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, true, true) == 0); - char filename[4096]; - gen_var_name(filename, true, false, false, false); - size_t data_len = args.request_size*args.iteration; - int ndim = 1; - uint64_t lb = 0; - uint64_t ub = data_len - 1; - char* file_data = NULL; - char csv_filename[4096]; - // sprintf(csv_filename, "local_process_data_bandwidth_%d.csv", info.rank); - // FILE* fp = redirect_stdout(csv_filename); - // FILE* fp = fopen(csv_filename, "w+"); - // REQUIRE(fp != NULL); - // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); - double total_mdata_time = 0; - double total_data_time = 0; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - long long int mdata_time_ns = 0; - long long int data_time_ns = 0; - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - free(file_data); - total_mdata_time += NS_TO_SECS(mdata_time_ns); - total_data_time += NS_TO_SECS(data_time_ns); - } - // restore_stdout(fp); - // AGGREGATE_TIME(data); - gen_perf_print(data_len, total_mdata_time, total_data_time); - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - // fclose(fp); - REQUIRE (posttest() == 0); +TEST_CASE("LocalProcessDataBandwidth", + "[files= " + std::to_string(args.number_of_files) + + "]" + "[file_size= " + + std::to_string(args.request_size * args.iteration) + + "]" + "[parallel_req= " + + std::to_string(info.comm_size) + + "]" + "[num_nodes= " + + std::to_string(info.comm_size / args.process_per_node) + "]") { + SECTION("Test Max Bandwidth") { + Timer data_time; + REQUIRE(pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE(rc == dspaces_SUCCESS); + REQUIRE(create_files_per_server_process(&client, true, true) == 0); + char filename[4096]; + gen_var_name(filename, true, false, false, false); + size_t data_len = args.request_size * args.iteration; + int ndim = 1; + uint64_t lb = 0; + uint64_t ub = data_len - 1; + char *file_data = NULL; + char csv_filename[4096]; + // sprintf(csv_filename, "local_process_data_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the + // buffer Also, setting timeout to 0 to prevent blocking for data + // availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, + (void **)&file_data, -1, &mdata_time_ns, &data_time_ns); + data_time.pauseTime(); + REQUIRE(rc == dspaces_SUCCESS); + free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Abort(MPI_COMM_WORLD, 0); + rc = dspaces_fini(client); + REQUIRE(rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE(posttest() == 0); + } } - -TEST_CASE("LocalNodeDataBandwidth", "[files= " + std::to_string(args.number_of_files) +"]" - "[file_size= " + std::to_string(args.request_size*args.iteration) +"]" - "[parallel_req= " + std::to_string(info.comm_size) +"]" - "[num_nodes= " + std::to_string(info.comm_size / args.process_per_node) +"]") { - SECTION("Test Max Bandwidth") { - Timer data_time; - REQUIRE (pretest() == 0); - dspaces_client_t client = dspaces_CLIENT_NULL; - int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); - REQUIRE (rc == dspaces_SUCCESS); - REQUIRE (create_files_per_server_process(&client, true, true) == 0); - char filename[4096]; - gen_var_name(filename, true, false, true, false); - size_t data_len = args.request_size*args.iteration; - int ndim = 1; - uint64_t lb = 0; - uint64_t ub = data_len - 1; - char* file_data = NULL; - char csv_filename[4096]; - // sprintf(csv_filename, "local_node_data_bandwidth_%d.csv", info.rank); - // FILE* fp = redirect_stdout(csv_filename); - // FILE* fp = fopen(csv_filename, "w+"); - // REQUIRE(fp != NULL); - // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); - double total_mdata_time = 0; - double total_data_time = 0; - for (size_t file_idx=0; file_idx < args.number_of_files; ++file_idx) { - long long int mdata_time_ns = 0; - long long int data_time_ns = 0; - data_time.resumeTime(); - // Using aget instead of get because dyad_get_data also allocates the buffer - // Also, setting timeout to 0 to prevent blocking for data availability since the data should always be available. - rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, (void**) &file_data, -1, &mdata_time_ns, &data_time_ns); - data_time.pauseTime(); - REQUIRE (rc == dspaces_SUCCESS); - free(file_data); - total_mdata_time += NS_TO_SECS(mdata_time_ns); - total_data_time += NS_TO_SECS(data_time_ns); - } - // restore_stdout(fp); - // AGGREGATE_TIME(data); - gen_perf_print(data_len, total_mdata_time, total_data_time); - rc = dspaces_fini(client); - REQUIRE (rc == dspaces_SUCCESS); - // fclose(fp); - REQUIRE (posttest() == 0); +TEST_CASE("LocalNodeDataBandwidth", + "[files= " + std::to_string(args.number_of_files) + + "]" + "[file_size= " + + std::to_string(args.request_size * args.iteration) + + "]" + "[parallel_req= " + + std::to_string(info.comm_size) + + "]" + "[num_nodes= " + + std::to_string(info.comm_size / args.process_per_node) + "]") { + SECTION("Test Max Bandwidth") { + Timer data_time; + REQUIRE(pretest() == 0); + dspaces_client_t client = dspaces_CLIENT_NULL; + int rc = dspaces_init_mpi(MPI_COMM_WORLD, &client); + REQUIRE(rc == dspaces_SUCCESS); + REQUIRE(create_files_per_server_process(&client, true, true) == 0); + char filename[4096]; + gen_var_name(filename, true, false, true, false); + size_t data_len = args.request_size * args.iteration; + int ndim = 1; + uint64_t lb = 0; + uint64_t ub = data_len - 1; + char *file_data = NULL; + char csv_filename[4096]; + // sprintf(csv_filename, "local_node_data_bandwidth_%d.csv", info.rank); + // FILE* fp = redirect_stdout(csv_filename); + // FILE* fp = fopen(csv_filename, "w+"); + // REQUIRE(fp != NULL); + // printf("rank,var_name,version,data_size,mdata_time_ns,data_time_ns\n"); + double total_mdata_time = 0; + double total_data_time = 0; + for (size_t file_idx = 0; file_idx < args.number_of_files; ++file_idx) { + long long int mdata_time_ns = 0; + long long int data_time_ns = 0; + data_time.resumeTime(); + // Using aget instead of get because dyad_get_data also allocates the + // buffer Also, setting timeout to 0 to prevent blocking for data + // availability since the data should always be available. + rc = dspaces_aget(client, filename, file_idx, ndim, &lb, &ub, + (void **)&file_data, -1, &mdata_time_ns, &data_time_ns); + data_time.pauseTime(); + REQUIRE(rc == dspaces_SUCCESS); + free(file_data); + total_mdata_time += NS_TO_SECS(mdata_time_ns); + total_data_time += NS_TO_SECS(data_time_ns); } + // restore_stdout(fp); + // AGGREGATE_TIME(data); + gen_perf_print(data_len, total_mdata_time, total_data_time); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Abort(MPI_COMM_WORLD, 0); + rc = dspaces_fini(client); + REQUIRE(rc == dspaces_SUCCESS); + // fclose(fp); + REQUIRE(posttest() == 0); + } } From 91ff208a97e606c4c36a471afd549754b97202e1 Mon Sep 17 00:00:00 2001 From: Ian Lumsden Date: Wed, 3 Apr 2024 12:59:48 -0700 Subject: [PATCH 46/47] Final version of DataSpaces stuff for DLIO --- .../configs/workload/dspaces_mummi.yaml | 2 +- .../configs/workload/dspaces_mummi_small.yaml | 2 +- .../dlio_benchmark/dspaces-setup-env.sh | 13 ++++-- .../dlio_benchmark/dspaces_alloc.sh | 15 ++++++ .../dlio_benchmark/dspaces_batch.sh | 27 +++++++++-- .../dspaces_h5_torch_data_loader.py | 14 ++++-- .../dlio_benchmark/dspaces_preloader.py | 46 +++++++++++++++++-- .../dlio_benchmark/dspaces_run_dlio.sh | 31 +++++++++++-- 8 files changed, 128 insertions(+), 22 deletions(-) create mode 100755 tests/integration/dlio_benchmark/dspaces_alloc.sh mode change 100644 => 100755 tests/integration/dlio_benchmark/dspaces_batch.sh diff --git a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml index c6ccfbad..86b07f8f 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi.yaml @@ -18,7 +18,7 @@ dataset: reader: data_loader: pytorch batch_size: 256 - read_threads: 6 + read_threads: 2 file_shuffle: seed sample_shuffle: seed multiprocessing_context: spawn diff --git a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml index d7106b0b..5fdf2289 100644 --- a/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml +++ b/tests/integration/dlio_benchmark/configs/workload/dspaces_mummi_small.yaml @@ -18,7 +18,7 @@ dataset: reader: data_loader: pytorch batch_size: 1 - read_threads: 2 + read_threads: 1 # 2 file_shuffle: seed sample_shuffle: seed multiprocessing_context: spawn diff --git a/tests/integration/dlio_benchmark/dspaces-setup-env.sh b/tests/integration/dlio_benchmark/dspaces-setup-env.sh index 3787a483..49e61230 100644 --- a/tests/integration/dlio_benchmark/dspaces-setup-env.sh +++ b/tests/integration/dlio_benchmark/dspaces-setup-env.sh @@ -1,20 +1,23 @@ #!/bin/bash # TODO make sure python mod matches Spack environment if there is python in there +module load gcc/10.3.1 module load python/3.9.12 module load openmpi/4.1.2 # Configurations -export DLIO_WORKLOAD=dspaces_mummi_small #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi -export NUM_NODES=1 # 1 for small config, 64 for full config -export PPN=1 # 1 for small config, 8 for full config +export DLIO_WORKLOAD=dspaces_mummi #dyad_mummi #dyad_unet3d_large # unet3d_base dyad_unet3d dyad_unet3d_small resnet50_base dyad_resnet50 unet3d_base_large mummi_base dyad_mummi +export NUM_NODES=32 # 2 for small config, for full config, scale 8, 16, 32, 64 +export PPN=8 # 1 for small config, 8 for full config export QUEUE=pbatch -export TIME=$((60)) +export TIME=$((180)) export BROKERS_PER_NODE=1 export GENERATE_DATA="0" export DSPACES_HG_STRING="ofi+verbs" -export MAX_DIM_LENGTH_FOR_FILES=8000 +export NUM_SAMPLES_PER_FILE=8000 +export NUM_ROWS_PER_SAMPLE=263 +export NUM_COLS_PER_SAMPLE=263 export GITHUB_WORKSPACE=/g/g90/lumsden1/ws/dyad_sc24_paper_dspaces/dyad export SPACK_DIR=/g/g90/lumsden1/ws/spack diff --git a/tests/integration/dlio_benchmark/dspaces_alloc.sh b/tests/integration/dlio_benchmark/dspaces_alloc.sh new file mode 100755 index 00000000..08aaf3f4 --- /dev/null +++ b/tests/integration/dlio_benchmark/dspaces_alloc.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +source ./dspaces-setup-env.sh + +# export DSPACES_DEBUG=1 + +if [ $# -eq 0 ]; then + flux alloc -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + ./dspaces_run_dlio.sh +else + flux alloc -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + ./dspaces_run_dlio.sh $1 +fi diff --git a/tests/integration/dlio_benchmark/dspaces_batch.sh b/tests/integration/dlio_benchmark/dspaces_batch.sh old mode 100644 new mode 100755 index 43344af5..e48bba14 --- a/tests/integration/dlio_benchmark/dspaces_batch.sh +++ b/tests/integration/dlio_benchmark/dspaces_batch.sh @@ -2,6 +2,27 @@ source ./dspaces-setup-env.sh -flux batch -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ - --broker-opts=--setattr=log-filename=./logs/flux.log \ - ./dspaces_run_dlio.sh \ No newline at end of file +# export DSPACES_DEBUG=1 + +if [ $# -eq 0 ]; then + flux batch -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + ./dspaces_run_dlio.sh +else + dump_dir=$1 + if [ ! -d $1 ]; then + mkdir -p $1 + fi + if [ $# -eq 1 ]; then + flux batch -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + --output=$1/run.out --error=$1/run.err \ + ./dspaces_run_dlio.sh $1 + else + export NUM_NODES=$2 + flux batch -N $NUM_NODES -t $TIME -q $QUEUE --exclusive \ + --broker-opts=--setattr=log-filename=./logs/flux.log \ + --output=$1/run.out --error=$1/run.err \ + ./dspaces_run_dlio.sh $1 $2 + fi +fi \ No newline at end of file diff --git a/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py b/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py index 02fbe2e1..950eaa8b 100644 --- a/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py +++ b/tests/integration/dlio_benchmark/dspaces_h5_torch_data_loader.py @@ -31,7 +31,6 @@ import dspaces import numpy as np -import flux import os import h5py import fcntl @@ -71,15 +70,16 @@ def worker_init(self, worker_id): thread_index=worker_id, epoch_number=self.epoch_number) proc_rank = os.getpid() + logging.debug("Intializing dspaces") self.ds_client = dspaces.dspaces(rank=proc_rank) def __del__(self): if self.dlp_logger: self.dlp_logger.finalize() # Manually invoke finalizer for DataSpaces to ensure it is shutdown properly - if self.ds_client: - del self.ds_client - self.ds_client = None + # if self.ds_client: + # del self.ds_client + # self.ds_client = None @dlp.log def __len__(self): @@ -94,10 +94,14 @@ def __getitem__(self, image_idx): lb = tuple([sample_index, 0, 0]) ub = tuple([sample_index, self.img_dim-1, self.img_dim-1]) dlp.update(args={"fname":filename}) + logging.debug(f"Filename is {filename}") dlp.update(args={"image_idx":image_idx}) dlp.update(args={"version":0}) dlp.update(args={"lb":lb}) + logging.debug(f"lb is {lb}") dlp.update(args={"ub":ub}) + logging.debug(f"ub is {ub}") + logging.debug("Starting dspaces aget") data = self.ds_client.get( filename, # variable name 0, # variable version @@ -106,7 +110,9 @@ def __getitem__(self, image_idx): np.uint8, # NumPy datatype of elements -1 # timeout ) + logging.debug("Finished dspaces aget") dlp.update(step=step) + logging.debug(f"data shape is {data.shape}") dlp.update(image_size=data.nbytes) return data diff --git a/tests/integration/dlio_benchmark/dspaces_preloader.py b/tests/integration/dlio_benchmark/dspaces_preloader.py index 67c0b285..d8ec6785 100644 --- a/tests/integration/dlio_benchmark/dspaces_preloader.py +++ b/tests/integration/dlio_benchmark/dspaces_preloader.py @@ -4,12 +4,14 @@ import dspaces import argparse +import logging +import time VALID_HDF5_EXTS = [".hdf5", ".h5"] def collect_file_names(dirname): - return list(sorted([f for f in dirname.iterdir() if f.is_file() and f.suffix() in VALID_HDF5_EXTS])) + return list(sorted([f for f in dirname.iterdir() if f.is_file() and f.suffix in VALID_HDF5_EXTS])) def get_local_files(all_files, rank, comm_size): @@ -18,26 +20,60 @@ def get_local_files(all_files, rank, comm_size): def store_samples_for_file(ds_client, local_file): with h5py.File(str(local_file), "r") as f: - data = f["records"][:] - offset = tuple([0] * len(data.shape)) - ds_client.put(data, str(local_file), 0, offset) + for starting_sample in range(0, f["records"].shape[0], 256): + end = starting_sample+256 + if end > f["records"].shape[0]: + end = f["records"].shape[0] + data = f["records"][starting_sample:end] + offset = (starting_sample, 0, 0) + ds_client.put(data, str(local_file), 0, offset) def main(): parser = argparse.ArgumentParser("Preload data for MuMMI training") parser.add_argument("data_dir", type=Path, help="Path to the directory containing HDF5 files") + parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('-d', '--debug', action='store_true') args = parser.parse_args() + loglevel = logging.WARNING + if args.verbose: + loglevel = logging.INFO + elif args.debug: + loglevel = logging.DEBUG + logging.basicConfig(level=loglevel, + handlers=[ + logging.StreamHandler() + ], + format='[%(levelname)s] [%(asctime)s] %(message)s [%(pathname)s:%(lineno)d]', + datefmt='%H:%M:%S' + ) data_dir = args.data_dir.expanduser().resolve() comm = MPI.COMM_WORLD rank = comm.Get_rank() comm_size = comm.Get_size() + start = time.time() + if rank == 0: + logging.info("Preloading with {} processes".format(comm_size)) + logging.debug("RANK {}: initializing DataSpaces".format(rank)) ds_client = dspaces.dspaces(comm=comm) + logging.debug("RANK {}: collecting filenames".format(rank)) all_files = collect_file_names(data_dir) + logging.debug("RANK {}: obtaining local files".format(rank)) local_files = get_local_files(all_files, rank, comm_size) + logging.debug("RANK {}: putting the following files: {}".format(rank, local_files)) for lf in local_files: store_samples_for_file(ds_client, lf) + end = time.time() + local_time = end - start + total_time = 0.0 + total_time = comm.reduce(local_time,MPI.SUM, root=0) + if rank == 0: + total_time /= comm_size + print("DataSpaces preload time is {} s".format(total_time), flush=True) + comm.Barrier() + comm.Abort(0) # Perform an abort because finalization sometimes hangs on Corona if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh index 051e0e4c..a36d5929 100755 --- a/tests/integration/dlio_benchmark/dspaces_run_dlio.sh +++ b/tests/integration/dlio_benchmark/dspaces_run_dlio.sh @@ -1,23 +1,44 @@ #!/bin/bash source ./dspaces-setup-env.sh +curr_dir=$(pwd) +if [ $# -ge 1 ]; then + cd $1 +fi + +if [ $# -gt 1 ]; then + export NUM_NODES=$2 +fi +echo "Number of nodes is $NUM_NODES" + +ulimit -c unlimited + # Startup dspaces_server +echo "Generating DataSpaces config file" echo "## Config file for DataSpaces server ndim = 3 -dims = $MAX_DIM_LENGTH_FOR_FILES +dims = $NUM_SAMPLES_PER_FILE,$NUM_ROWS_PER_SAMPLE,$NUM_COLS_PER_SAMPLE max_versions = 1 num_apps = 1" > dataspaces.conf +redirect_flag="" +if [ ! -z ${DSPACES_DEBUG+x} ]; then + redirect_flag="--output=server.out --error=server.err" +fi + +echo "Launching DataSpaces server" flux submit -N $NUM_NODES --cores=$(( NUM_NODES*16 )) \ - --tasks-per-node=$BROKERS_PER_NODE dspaces_server $DSPACES_HG_STRING + --tasks-per-node=$BROKERS_PER_NODE $redirect_flag dspaces_server $DSPACES_HG_STRING # Wait for DataSpaces's server to create conf.ds +echo "Waiting on conf.ds to be created" sleep 1s while [ ! -f conf.ds ]; do sleep 1s done # Give the server enough time to write the contents sleep 3s +echo "Server running!" if [[ "${GENERATE_DATA}" == "1" ]]; then # Generate Data for Workload @@ -30,7 +51,8 @@ exit fi # Preload data into DataSpaces -flux run -N $NUM_NODES --tasks-per-node=1 python3 ./dspaces_preloader.py $DLIO_DATA_DIR/train +echo "Preloading samples into DataSpaces" +flux run -N $NUM_NODES --cores=$(( NUM_NODES*32 )) --tasks-per-node=32 python3 $GITHUB_WORKSPACE/tests/integration/dlio_benchmark/dspaces_preloader.py $DLIO_DATA_DIR/train # Run Training echo Running DLIO Training for ${DLIO_WORKLOAD} @@ -46,3 +68,6 @@ echo "Finished Executing check ${DSPACES_DLIO_RUN_LOG} for output" flux run --ntasks=1 terminator rm dataspaces.conf conf.ds +if [ $# -ge 1 ]; then + cd $curr_dir +fi From 3780bed1ed98e079d4a38bfa4419709b80865a05 Mon Sep 17 00:00:00 2001 From: hariharandev1 Date: Tue, 30 Jul 2024 10:09:12 -0700 Subject: [PATCH 47/47] remove error file. --- dyad_mod_0.err | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 dyad_mod_0.err diff --git a/dyad_mod_0.err b/dyad_mod_0.err deleted file mode 100644 index 39f83ede..00000000 --- a/dyad_mod_0.err +++ /dev/null @@ -1,14 +0,0 @@ -DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal -DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: Did not find DTL 'mode' option. Using env DYAD_DTL_MODE=(null)DYAD_DTL_MODE is not set. Defaulting to FLUX_RPC -[DYAD ERROR]: No KVS namespace provided! - -DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal -DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: Did not find DTL 'mode' option. Using env DYAD_DTL_MODE=(null)DYAD_DTL_MODE is not set. Defaulting to FLUX_RPC -[DYAD ERROR]: No KVS namespace provided! - -DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal -DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: DTL 'mode' option set. Setting env DYAD_DTL_MODE=UCX[DYAD ERROR]: No KVS namespace provided! - -DYAD_MOD: positional arguments /l/ssd/haridev/dyad/internal -DYAD_MOD: Loading DYAD Module with Path /l/ssd/haridev/dyad/internalDYAD_MOD: DTL 'mode' option set. Setting env DYAD_DTL_MODE=UCX[DYAD ERROR]: No KVS namespace provided! -