From 06116d1604d8c071ad5fe51df5eabcdab71dff82 Mon Sep 17 00:00:00 2001 From: Evgeny Budilovsky Date: Tue, 24 Sep 2024 17:58:34 +0300 Subject: [PATCH] [Filestore] avoid fsync on every write for local service (#1990) * enforce FORCE_STATIC_LINKING flag for libfuse * NBSNEBIUS-371: avoid fsync on every write for local service --- build/conf/opensource.conf | 2 +- cloud/filestore/libs/client/session.cpp | 1 + .../libs/diagnostics/profile_log_events.cpp | 23 ++++++ .../libs/diagnostics/profile_log_events.h | 1 + cloud/filestore/libs/service/auth_scheme.cpp | 2 + cloud/filestore/libs/service/request.h | 16 ++++ .../filestore/libs/service_kikimr/service.cpp | 42 +++++++++- .../libs/service_kikimr/service_ut.cpp | 53 +++++++++++- cloud/filestore/libs/service_local/fs.h | 3 + .../filestore/libs/service_local/fs_data.cpp | 50 ++++++++++-- .../libs/service_local/service_ut.cpp | 34 ++++++++ cloud/filestore/libs/storage/api/service.h | 2 +- .../libs/storage/service/service_actor.cpp | 2 +- .../libs/storage/service/service_actor.h | 2 +- .../service/service_actor_complete.cpp | 2 +- .../storage/tablet/tablet_actor_request.cpp | 2 +- .../libs/storage/testlib/service_client.h | 2 +- cloud/filestore/libs/vfs_fuse/fs_impl.h | 6 ++ .../filestore/libs/vfs_fuse/fs_impl_data.cpp | 80 +++++++++++++++++-- .../filestore/libs/vfs_fuse/fs_impl_list.cpp | 31 +++++++ cloud/filestore/libs/vfs_fuse/fs_ut.cpp | 54 +++++++++++++ cloud/filestore/libs/vhost/request.h | 26 ++++++ cloud/filestore/public/api/grpc/service.proto | 14 ++++ cloud/filestore/public/api/protos/data.proto | 57 +++++++++++++ .../analytics/libs/event-log/dump_ut.cpp | 25 +++--- .../libs/event-log/request_printer.cpp | 1 + 26 files changed, 495 insertions(+), 38 deletions(-) diff --git a/build/conf/opensource.conf b/build/conf/opensource.conf index 1c79b2124c..0d2516d91e 100644 --- a/build/conf/opensource.conf +++ b/build/conf/opensource.conf @@ -23,10 +23,10 @@ when ($OPENSOURCE == "yes") { SO_OUTPUTS=yes UDF_NO_PROBE=yes USE_ASMLIB=no - USE_DYNAMIC_LIBFUSE=yes USE_MKL=no VALIDATE_DATA=no when ($FORCE_STATIC_LINKING != "yes") { + USE_DYNAMIC_LIBFUSE=yes _USE_AIO=dynamic _USE_ICONV=dynamic _USE_IDN=dynamic diff --git a/cloud/filestore/libs/client/session.cpp b/cloud/filestore/libs/client/session.cpp index ca7ee95155..1ce66a0a7a 100644 --- a/cloud/filestore/libs/client/session.cpp +++ b/cloud/filestore/libs/client/session.cpp @@ -321,6 +321,7 @@ class TSession // FILESTORE_IMPLEMENT_METHOD FILESTORE_DATA_METHODS(FILESTORE_IMPLEMENT_METHOD) +FILESTORE_LOCAL_DATA_METHODS(FILESTORE_IMPLEMENT_METHOD) #undef FILESTORE_IMPLEMENT_METHOD diff --git a/cloud/filestore/libs/diagnostics/profile_log_events.cpp b/cloud/filestore/libs/diagnostics/profile_log_events.cpp index 9784a45a0c..00d8f3261b 100644 --- a/cloud/filestore/libs/diagnostics/profile_log_events.cpp +++ b/cloud/filestore/libs/diagnostics/profile_log_events.cpp @@ -418,6 +418,27 @@ void InitProfileLogRequestInfo( nodeInfo->SetNodeName(request.GetCheckpointId()); } +template <> +void InitProfileLogRequestInfo( + NProto::TProfileLogRequestInfo& profileLogRequest, + const NProto::TFsyncRequest& request) +{ + auto* nodeInfo = profileLogRequest.MutableNodeInfo(); + nodeInfo->SetNodeId(request.GetNodeId()); + nodeInfo->SetHandle(request.GetHandle()); + nodeInfo->SetFlags(request.GetDataSync()); +} + +template <> +void InitProfileLogRequestInfo( + NProto::TProfileLogRequestInfo& profileLogRequest, + const NProto::TFsyncDirRequest& request) +{ + auto* nodeInfo = profileLogRequest.MutableNodeInfo(); + nodeInfo->SetNodeId(request.GetNodeId()); + nodeInfo->SetFlags(request.GetDataSync()); +} + //////////////////////////////////////////////////////////////////////////////// #define IMPLEMENT_DEFAULT_METHOD(name, ns) \ @@ -474,6 +495,8 @@ void InitProfileLogRequestInfo( IMPLEMENT_DEFAULT_METHOD(DescribeData, NProtoPrivate) IMPLEMENT_DEFAULT_METHOD(GenerateBlobIds, NProtoPrivate) IMPLEMENT_DEFAULT_METHOD(AddData, NProtoPrivate) + IMPLEMENT_DEFAULT_METHOD(Fsync, NProto) + IMPLEMENT_DEFAULT_METHOD(FsyncDir, NProto) #undef IMPLEMENT_DEFAULT_METHOD diff --git a/cloud/filestore/libs/diagnostics/profile_log_events.h b/cloud/filestore/libs/diagnostics/profile_log_events.h index c5b7c1e1a2..63f5bb5830 100644 --- a/cloud/filestore/libs/diagnostics/profile_log_events.h +++ b/cloud/filestore/libs/diagnostics/profile_log_events.h @@ -30,6 +30,7 @@ namespace NFuse { #define FILESTORE_FUSE_REQUESTS(xxx, ...) \ xxx(Flush, __VA_ARGS__) \ xxx(Fsync, __VA_ARGS__) \ + xxx(FsyncDir, __VA_ARGS__) \ // FILESTORE_FUSE_REQUESTS #define FILESTORE_MATERIALIZE_REQUEST(name, ...) name, diff --git a/cloud/filestore/libs/service/auth_scheme.cpp b/cloud/filestore/libs/service/auth_scheme.cpp index d2ea01b3c3..53a962cd9a 100644 --- a/cloud/filestore/libs/service/auth_scheme.cpp +++ b/cloud/filestore/libs/service/auth_scheme.cpp @@ -47,6 +47,8 @@ TPermissionList GetRequestPermissions(EFileStoreRequest requestType) case EFileStoreRequest::GenerateBlobIds: case EFileStoreRequest::WriteBlob: case EFileStoreRequest::AddData: + case EFileStoreRequest::Fsync: + case EFileStoreRequest::FsyncDir: return CreatePermissionList({EPermission::Write}); case EFileStoreRequest::AddClusterNode: diff --git a/cloud/filestore/libs/service/request.h b/cloud/filestore/libs/service/request.h index 5c8a235622..7ed2378ada 100644 --- a/cloud/filestore/libs/service/request.h +++ b/cloud/filestore/libs/service/request.h @@ -72,8 +72,14 @@ namespace NCloud::NFileStore { xxx(AllocateData, __VA_ARGS__) \ // FILESTORE_DATA_METHODS +#define FILESTORE_LOCAL_DATA_METHODS(xxx, ...) \ + xxx(Fsync, __VA_ARGS__) \ + xxx(FsyncDir, __VA_ARGS__) \ +// FILESTORE_LOCAL_DATA_METHODS + #define FILESTORE_DATA_SERVICE(xxx, ...) \ FILESTORE_DATA_METHODS(xxx, __VA_ARGS__) \ + FILESTORE_LOCAL_DATA_METHODS(xxx, __VA_ARGS__) \ // FILESTORE_DATA_SERVICE #define FILESTORE_CONTROL_SERVICE(xxx, ...) \ @@ -87,6 +93,14 @@ namespace NCloud::NFileStore { xxx(PingSession, __VA_ARGS__) \ FILESTORE_SERVICE_METHODS(xxx, __VA_ARGS__) \ FILESTORE_DATA_METHODS(xxx, __VA_ARGS__) \ + FILESTORE_LOCAL_DATA_METHODS(xxx, __VA_ARGS__) \ +// FILESTORE_SERVICE + +#define FILESTORE_REMOTE_SERVICE(xxx, ...) \ + xxx(Ping, __VA_ARGS__) \ + xxx(PingSession, __VA_ARGS__) \ + FILESTORE_SERVICE_METHODS(xxx, __VA_ARGS__) \ + FILESTORE_DATA_METHODS(xxx, __VA_ARGS__) \ // FILESTORE_SERVICE #define FILESTORE_ENDPOINT_METHODS(xxx, ...) \ @@ -108,6 +122,7 @@ namespace NCloud::NFileStore { xxx(PingSession, __VA_ARGS__) \ FILESTORE_SERVICE_METHODS(xxx, __VA_ARGS__) \ FILESTORE_DATA_METHODS(xxx, __VA_ARGS__) \ + FILESTORE_LOCAL_DATA_METHODS(xxx, __VA_ARGS__) \ xxx(GetSessionEventsStream, __VA_ARGS__) \ FILESTORE_ENDPOINT_METHODS(xxx, __VA_ARGS__) \ // FILESTORE_REQUESTS @@ -119,6 +134,7 @@ namespace NCloud::NFileStore { xxx(PingSession, __VA_ARGS__) \ FILESTORE_SERVICE_METHODS(xxx, __VA_ARGS__) \ FILESTORE_DATA_METHODS(xxx, __VA_ARGS__) \ + FILESTORE_LOCAL_DATA_METHODS(xxx, __VA_ARGS__) \ FILESTORE_ENDPOINT_METHODS(xxx, __VA_ARGS__) \ // FILESTORE_PROTO_REQUESTS diff --git a/cloud/filestore/libs/service_kikimr/service.cpp b/cloud/filestore/libs/service_kikimr/service.cpp index 446d129ab3..b62aeab6b9 100644 --- a/cloud/filestore/libs/service_kikimr/service.cpp +++ b/cloud/filestore/libs/service_kikimr/service.cpp @@ -34,7 +34,21 @@ namespace { }; \ // FILESTORE_DECLARE_METHOD -FILESTORE_SERVICE(FILESTORE_DECLARE_METHOD) +FILESTORE_REMOTE_SERVICE(FILESTORE_DECLARE_METHOD) + +#undef FILESTORE_DECLARE_METHOD + +#define FILESTORE_DECLARE_METHOD(name, ...) \ + struct T##name##Method \ + { \ + static constexpr auto RequestName = TStringBuf(#name); \ + \ + using TRequest = NProto::T##name##Request; \ + using TResponse = NProto::T##name##Response; \ + }; \ +// FILESTORE_DECLARE_METHOD + +FILESTORE_LOCAL_DATA_METHODS(FILESTORE_DECLARE_METHOD) #undef FILESTORE_DECLARE_METHOD @@ -291,6 +305,32 @@ class TKikimrFileStore final std::move(response))); } + template<> + void ExecuteRequest( + TCallContextPtr callContext, + std::shared_ptr request, + TPromise response) + { + Y_UNUSED(callContext); + Y_UNUSED(request); + Y_UNUSED(TFsyncMethod::RequestName); + + response.SetValue(TFsyncMethod::TResponse()); + } + + template<> + void ExecuteRequest( + TCallContextPtr callContext, + std::shared_ptr request, + TPromise response) + { + Y_UNUSED(callContext); + Y_UNUSED(request); + Y_UNUSED(TFsyncDirMethod::RequestName); + + response.SetValue(TFsyncDirMethod::TResponse()); + } + template void ExecuteStreamRequest( TCallContextPtr callContext, diff --git a/cloud/filestore/libs/service_kikimr/service_ut.cpp b/cloud/filestore/libs/service_kikimr/service_ut.cpp index aa9592cc4e..ab6a513aec 100644 --- a/cloud/filestore/libs/service_kikimr/service_ut.cpp +++ b/cloud/filestore/libs/service_kikimr/service_ut.cpp @@ -53,14 +53,14 @@ struct TTestServiceActor final } \ // FILESTORE_IMPLEMENT_METHOD - FILESTORE_SERVICE(FILESTORE_IMPLEMENT_METHOD, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_IMPLEMENT_METHOD, TEvService) #undef FILESTORE_IMPLEMENT_METHOD STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { - FILESTORE_SERVICE(FILESTORE_HANDLE_REQUEST, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_HANDLE_REQUEST, TEvService) } } }; @@ -69,7 +69,7 @@ struct TTestServiceActor final //////////////////////////////////////////////////////////////////////////////// -Y_UNIT_TEST_SUITE(TIndexStoreTest) +Y_UNIT_TEST_SUITE(TKikimrFileStore) { Y_UNIT_TEST(ShouldHandleRequests) { @@ -101,6 +101,53 @@ Y_UNIT_TEST_SUITE(TIndexStoreTest) service->Stop(); } + Y_UNIT_TEST(ShouldHandleFsyncRequestsOutsideActorSystem) + { + auto serviceActor = std::make_unique(); + + auto actorSystem = MakeIntrusive(); + actorSystem->RegisterTestService(std::move(serviceActor)); + + auto service = CreateKikimrFileStore(actorSystem); + service->Start(); + + { + auto context = MakeIntrusive(); + auto request = std::make_shared(); + + auto future = service->Fsync( + std::move(context), + std::move(request)); + + actorSystem->DispatchEvents(WaitTimeout); + + const auto& response = future.GetValue(WaitTimeout); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + response.GetError().GetCode(), + response.GetError().GetMessage()); + } + + { + auto context = MakeIntrusive(); + auto request = std::make_shared(); + + auto future = service->FsyncDir( + std::move(context), + std::move(request)); + + actorSystem->DispatchEvents(WaitTimeout); + + const auto& response = future.GetValue(WaitTimeout); + UNIT_ASSERT_VALUES_EQUAL_C( + S_OK, + response.GetError().GetCode(), + response.GetError().GetMessage()); + } + + service->Stop(); + } + Y_UNIT_TEST(ShouldGetSessionEventsStream) { TActorId eventHandler; diff --git a/cloud/filestore/libs/service_local/fs.h b/cloud/filestore/libs/service_local/fs.h index 9ac30ff86f..bc03886aed 100644 --- a/cloud/filestore/libs/service_local/fs.h +++ b/cloud/filestore/libs/service_local/fs.h @@ -58,6 +58,9 @@ namespace NCloud::NFileStore { xxx(TestLock, __VA_ARGS__) \ \ xxx(AllocateData, __VA_ARGS__) \ + \ + xxx(Fsync, __VA_ARGS__) \ + xxx(FsyncDir, __VA_ARGS__) \ // FILESTORE_DATA_METHODS_LOCAL_SYNC #define FILESTORE_DATA_METHODS_LOCAL_ASYNC(xxx, ...) \ diff --git a/cloud/filestore/libs/service_local/fs_data.cpp b/cloud/filestore/libs/service_local/fs_data.cpp index e42757cfb3..8351918ef6 100644 --- a/cloud/filestore/libs/service_local/fs_data.cpp +++ b/cloud/filestore/libs/service_local/fs_data.cpp @@ -112,21 +112,14 @@ TFuture TLocalFileSystem::WriteDataAsync( TErrorResponse(ErrorInvalidHandle(request.GetHandle()))); } - const FHANDLE fd = *handle; auto b = std::move(*request.MutableBuffer()); TArrayRef data(b.begin(), b.vend()); auto promise = NewPromise(); FileIOService->AsyncWrite(*handle, request.GetOffset(), data).Subscribe( - [b = std::move(b), promise, fd] (const TFuture& f) mutable { + [b = std::move(b), promise] (const TFuture& f) mutable { NProto::TWriteDataResponse response; try { f.GetValue(); - TFileHandle h(fd); - const bool flushed = h.Flush(); - h.Release(); - if (!flushed) { - throw yexception() << "failed to flush " << fd; - } } catch (...) { *response.MutableError() = MakeError(E_IO, CurrentExceptionMessage()); @@ -161,4 +154,45 @@ NProto::TAllocateDataResponse TLocalFileSystem::AllocateData( return {}; } +NProto::TFsyncResponse TLocalFileSystem::Fsync( + const NProto::TFsyncRequest& request) +{ + STORAGE_TRACE("Fsync " << DumpMessage(request)); + + auto session = GetSession(request); + auto* handle = session->LookupHandle(request.GetHandle()); + if (!handle || !handle->IsOpen()) { + return TErrorResponse(ErrorInvalidHandle(request.GetHandle())); + } + + const bool flushed = + request.GetDataSync() ? handle->FlushData() : handle->Flush(); + if (!flushed) { + return TErrorResponse(E_IO, "flush failed"); + } + + return {}; +} + +NProto::TFsyncDirResponse TLocalFileSystem::FsyncDir( + const NProto::TFsyncDirRequest& request) +{ + STORAGE_TRACE("FsyncDir " << DumpMessage(request)); + + auto session = GetSession(request); + auto node = session->LookupNode(request.GetNodeId()); + if (!node) { + return TErrorResponse(ErrorInvalidTarget(request.GetNodeId())); + } + + auto handle = node->OpenHandle(O_RDONLY|O_DIRECTORY); + const bool flushed = + request.GetDataSync() ? handle.FlushData() : handle.Flush(); + if (!flushed) { + return TErrorResponse(E_IO, "flush failed"); + } + + return {}; +} + } // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/service_local/service_ut.cpp b/cloud/filestore/libs/service_local/service_ut.cpp index 45a1a33b4d..ce6ee4594c 100644 --- a/cloud/filestore/libs/service_local/service_ut.cpp +++ b/cloud/filestore/libs/service_local/service_ut.cpp @@ -636,6 +636,23 @@ struct TTestBootstrap return request; } + auto CreateFsyncRequest(ui64 node, ui64 handle, bool dataSync) + { + auto request = CreateRequest(); + request->SetNodeId(node); + request->SetHandle(handle); + request->SetDataSync(dataSync); + return request; + } + + auto CreateFsyncDirRequest(ui64 node, bool dataSync) + { + auto request = CreateRequest(); + request->SetNodeId(node); + request->SetDataSync(dataSync); + return request; + } + #define FILESTORE_DECLARE_METHOD(name, ns) \ template \ NProto::T##name##Response name(Args&&... args) \ @@ -1580,6 +1597,23 @@ Y_UNIT_TEST_SUITE(LocalFileStore) UNIT_ASSERT_VALUES_EQUAL(names2[0], "d"); } } + + Y_UNIT_TEST(ShouldFsyncFileAndDir) + { + TTestBootstrap bootstrap("fs"); + + auto fileNodeId = CreateFile(bootstrap, RootNodeId, "file1"); + auto dirNodeId = CreateDirectory(bootstrap, RootNodeId, "dir1"); + + auto fileHandle = + bootstrap.CreateHandle(fileNodeId, "", TCreateHandleArgs::RDWR) + .GetHandle(); + + for (auto dataSync: {true, false}) { + bootstrap.Fsync(fileNodeId, fileHandle, dataSync); + bootstrap.FsyncDir(dirNodeId, dataSync); + } + } }; } // namespace NCloud::NFileStore diff --git a/cloud/filestore/libs/storage/api/service.h b/cloud/filestore/libs/storage/api/service.h index ab387a104c..25f50ab60f 100644 --- a/cloud/filestore/libs/storage/api/service.h +++ b/cloud/filestore/libs/storage/api/service.h @@ -272,7 +272,7 @@ struct TEvService static_assert(EvEnd < (int)TFileStoreEvents::SERVICE_END, "EvEnd expected to be < TFileStoreEvents::SERVICE_END"); - FILESTORE_SERVICE(FILESTORE_DECLARE_PROTO_EVENTS, NProto) + FILESTORE_REMOTE_SERVICE(FILESTORE_DECLARE_PROTO_EVENTS, NProto) using TEvRegisterLocalFileStoreRequest = TRequestEvent< TRegisterLocalFileStore, diff --git a/cloud/filestore/libs/storage/service/service_actor.cpp b/cloud/filestore/libs/storage/service/service_actor.cpp index 7651896332..b25c92887d 100644 --- a/cloud/filestore/libs/storage/service/service_actor.cpp +++ b/cloud/filestore/libs/storage/service/service_actor.cpp @@ -125,7 +125,7 @@ bool TStorageServiceActor::HandleRequests(STFUNC_SIG) FILESTORE_HANDLE_REQUEST(name, ns) \ FILESTORE_HANDLE_RESPONSE(name, ns) \ - FILESTORE_SERVICE(FILESTORE_HANDLE_REQUEST_RESPONSE, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_HANDLE_REQUEST_RESPONSE, TEvService) FILESTORE_SERVICE_REQUESTS_PRIVATE(FILESTORE_HANDLE_REQUEST_RESPONSE, TEvServicePrivate) #undef FILESTORE_HANDLE_REQUEST_RESPONSE diff --git a/cloud/filestore/libs/storage/service/service_actor.h b/cloud/filestore/libs/storage/service/service_actor.h index b5e0821618..96d5353d92 100644 --- a/cloud/filestore/libs/storage/service/service_actor.h +++ b/cloud/filestore/libs/storage/service/service_actor.h @@ -100,7 +100,7 @@ class TStorageServiceActor final const ns::TEv##name##Response::TPtr& ev, \ const NActors::TActorContext& ctx); \ - FILESTORE_SERVICE(FILESTORE_DECLARE_REQUEST_RESPONSE, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_DECLARE_REQUEST_RESPONSE, TEvService) FILESTORE_SERVICE_REQUESTS_PRIVATE(FILESTORE_DECLARE_REQUEST_RESPONSE, TEvServicePrivate) #undef FILESTORE_DECLARE_REQUEST_RESPONSE diff --git a/cloud/filestore/libs/storage/service/service_actor_complete.cpp b/cloud/filestore/libs/storage/service/service_actor_complete.cpp index 8f738e1936..0adc16d33e 100644 --- a/cloud/filestore/libs/storage/service/service_actor_complete.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_complete.cpp @@ -71,7 +71,7 @@ void TStorageServiceActor::CompleteRequest( CompleteRequest(ctx, ev); \ } - FILESTORE_SERVICE(FILESTORE_IMPLEMENT_RESPONSE, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_IMPLEMENT_RESPONSE, TEvService) FILESTORE_SERVICE_REQUESTS_PRIVATE(FILESTORE_IMPLEMENT_RESPONSE, TEvServicePrivate) #undef FILESTORE_IMPLEMENT_RESPONSE diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp index ac228f1187..71b3245431 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_request.cpp @@ -117,7 +117,7 @@ template void TIndexTabletActor::CompleteResponse( const NActors::TActorContext& ctx); \ // FILESTORE_IMPL_VALIDATE -FILESTORE_SERVICE(FILESTORE_GENERATE_IMPL, TEvService) +FILESTORE_REMOTE_SERVICE(FILESTORE_GENERATE_IMPL, TEvService) FILESTORE_GENERATE_IMPL(DescribeData, TEvIndexTablet) FILESTORE_GENERATE_IMPL(DescribeSessions, TEvIndexTablet) FILESTORE_GENERATE_IMPL(GenerateBlobIds, TEvIndexTablet) diff --git a/cloud/filestore/libs/storage/testlib/service_client.h b/cloud/filestore/libs/storage/testlib/service_client.h index 05ea2244e4..bbbbd64062 100644 --- a/cloud/filestore/libs/storage/testlib/service_client.h +++ b/cloud/filestore/libs/storage/testlib/service_client.h @@ -608,7 +608,7 @@ class TServiceClient } \ // FILESTORE_DECLARE_METHOD - FILESTORE_SERVICE(FILESTORE_DECLARE_METHOD, TEvService) + FILESTORE_REMOTE_SERVICE(FILESTORE_DECLARE_METHOD, TEvService) #undef FILESTORE_DECLARE_METHOD diff --git a/cloud/filestore/libs/vfs_fuse/fs_impl.h b/cloud/filestore/libs/vfs_fuse/fs_impl.h index d5bdf9621d..89d5056380 100644 --- a/cloud/filestore/libs/vfs_fuse/fs_impl.h +++ b/cloud/filestore/libs/vfs_fuse/fs_impl.h @@ -364,6 +364,12 @@ class TFileSystem final fuse_req_t req, fuse_ino_t ino); + bool ValidateDirectoryHandle( + TCallContext& callContext, + fuse_req_t req, + fuse_ino_t ino, + uint64_t fh); + bool UpdateNodesCache( const NProto::TNodeAttr& attrs, fuse_entry_param& entry); diff --git a/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp b/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp index 9362af0cec..65e9b77430 100644 --- a/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp +++ b/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp @@ -7,6 +7,7 @@ namespace NCloud::NFileStore::NFuse { using namespace NCloud::NFileStore::NVFS; +using namespace NThreading; namespace { @@ -444,7 +445,8 @@ void TFileSystem::FSync( TNodeId{fi ? ino : InvalidNodeId}, THandle{fi ? fi->fh : InvalidHandle}); - auto callback = [=, ptr = weak_from_this(), requestInfo = std::move(requestInfo)] + std::function&)> + callback = [=, ptr = weak_from_this(), requestInfo = std::move(requestInfo)] (const auto& future) mutable { auto self = ptr.lock(); if (!self) { @@ -465,6 +467,34 @@ void TFileSystem::FSync( } }; + if (fi) { + callback = [ptr = weak_from_this(), + callContext, + ino, + datasync, + fh = fi->fh, + callback = std::move(callback)](const auto& future) mutable + { + auto self = ptr.lock(); + if (!self) { + return; + } + + if (HasError(future.GetValue())) { + callback(future); + return; + } + + auto request = StartRequest(ino); + request->SetHandle(fh); + request->SetDataSync(datasync); + self->Session->Fsync(callContext, std::move(request)) + .Apply([](const auto& future) + { return future.GetValue().GetError(); }) + .Subscribe(std::move(callback)); + }; + } + if (fi) { if (datasync) { FSyncQueue.WaitForDataRequests(reqId, TNodeId {ino}, THandle {fi->fh}) @@ -491,21 +521,30 @@ void TFileSystem::FSyncDir( int datasync, fuse_file_info* fi) { + Y_ABORT_UNLESS(fi); + STORAGE_DEBUG("FSyncDir #" << ino << " @" << fi->fh); if (!ValidateNodeId(*callContext, req, ino)) { return; } + if (!ValidateDirectoryHandle(*callContext, req, ino, fi->fh)) { + return; + } + const auto reqId = callContext->RequestId; NProto::TProfileLogRequestInfo requestInfo; - InitProfileLogRequestInfo(requestInfo, EFileStoreFuseRequest::Fsync, Now()); + InitProfileLogRequestInfo( + requestInfo, + EFileStoreFuseRequest::FsyncDir, + Now()); InitNodeInfo( requestInfo, datasync, - TNodeId{fi ? ino : InvalidNodeId}, - THandle{fi ? fi->fh : InvalidHandle}); + TNodeId{ino}, + THandle{fi->fh}); auto callback = [=, ptr = weak_from_this(), requestInfo = std::move(requestInfo)] (const auto& future) mutable { @@ -528,12 +567,37 @@ void TFileSystem::FSyncDir( } }; - if (datasync) { - FSyncQueue.WaitForDataRequests(reqId) + auto waitCallback = + [ptr = weak_from_this(), + callContext, + ino, + datasync, + callback = std::move(callback)](const auto& future) mutable + { + auto self = ptr.lock(); + if (!self) { + return; + } + + if (HasError(future.GetValue())) { + callback(future); + return; + } + + auto request = StartRequest(ino); + request->SetDataSync(datasync); + self->Session->FsyncDir(callContext, std::move(request)) + .Apply([](const auto& future) + { return future.GetValue().GetError(); }) .Subscribe(std::move(callback)); + }; + + if (datasync) { + FSyncQueue.WaitForDataRequests(reqId).Subscribe( + std::move(waitCallback)); } else { - FSyncQueue.WaitForRequests(reqId) - .Subscribe(std::move(callback)); + FSyncQueue.WaitForRequests(reqId).Subscribe( + std::move(waitCallback)); } } diff --git a/cloud/filestore/libs/vfs_fuse/fs_impl_list.cpp b/cloud/filestore/libs/vfs_fuse/fs_impl_list.cpp index 1695af0fea..7e49f0a6ef 100644 --- a/cloud/filestore/libs/vfs_fuse/fs_impl_list.cpp +++ b/cloud/filestore/libs/vfs_fuse/fs_impl_list.cpp @@ -398,4 +398,35 @@ void TFileSystem::ReleaseDir( ReplyError(*callContext, {}, req, 0); } +bool TFileSystem::ValidateDirectoryHandle( + TCallContext& callContext, + fuse_req_t req, + fuse_ino_t ino, + uint64_t fh) +{ + std::shared_ptr handle; + with_lock (CacheLock) { + auto it = DirectoryHandles.find(fh); + if (it == DirectoryHandles.end()) { + ReplyError( + callContext, + ErrorInvalidHandle(fh), + req, + EBADF); + return false; + } + + handle = it->second; + } + + Y_ABORT_UNLESS(handle); + + if (!CheckDirectoryHandle(req, ino, *handle, Log, __func__)) { + ReplyError(callContext, ErrorInvalidHandle(fh), req, EBADF); + return false; + } + + return true; +} + } // namespace NCloud::NFileStore::NFuse diff --git a/cloud/filestore/libs/vfs_fuse/fs_ut.cpp b/cloud/filestore/libs/vfs_fuse/fs_ut.cpp index 91429c9d68..b8126b1696 100644 --- a/cloud/filestore/libs/vfs_fuse/fs_ut.cpp +++ b/cloud/filestore/libs/vfs_fuse/fs_ut.cpp @@ -267,6 +267,21 @@ Y_UNIT_TEST_SUITE(TFileSystemTest) return MakeFuture(result); }; + std::atomic fsyncCalledWithDataSync = 0; + std::atomic fsyncCalledWithoutDataSync = 0; + + bootstrap.Service->FsyncHandler = [&] (auto callContext, auto request) { + UNIT_ASSERT_VALUES_EQUAL(FileSystemId, callContext->FileSystemId); + + if (request->GetDataSync()) { + fsyncCalledWithDataSync++; + } else { + fsyncCalledWithoutDataSync++; + } + + return MakeFuture(NProto::TFsyncResponse()); + }; + bootstrap.Start(); auto handle = bootstrap.Fuse->SendRequest("/file1", RootNodeId); @@ -275,6 +290,18 @@ Y_UNIT_TEST_SUITE(TFileSystemTest) auto write = bootstrap.Fuse->SendRequest( nodeId, handleId, 0, CreateBuffer(4096, 'a')); UNIT_ASSERT_NO_EXCEPTION(write.GetValue(WaitTimeout)); + + auto fsync = bootstrap.Fuse->SendRequest( + nodeId, handleId, false /* no data sync */); + UNIT_ASSERT_NO_EXCEPTION(fsync.GetValue(WaitTimeout)); + UNIT_ASSERT_VALUES_EQUAL(1, fsyncCalledWithoutDataSync.load()); + UNIT_ASSERT_VALUES_EQUAL(0, fsyncCalledWithDataSync.load()); + + fsync = bootstrap.Fuse->SendRequest( + nodeId, handleId, true /* data sync */); + UNIT_ASSERT_NO_EXCEPTION(fsync.GetValue(WaitTimeout)); + UNIT_ASSERT_VALUES_EQUAL(1, fsyncCalledWithoutDataSync.load()); + UNIT_ASSERT_VALUES_EQUAL(1, fsyncCalledWithDataSync.load()); } Y_UNIT_TEST(ShouldPassSessionId) @@ -500,6 +527,21 @@ Y_UNIT_TEST_SUITE(TFileSystemTest) return MakeFuture(result); }; + std::atomic fsyncDirCalledWithDataSync = 0; + std::atomic fsyncDirCalledWithoutDataSync = 0; + + bootstrap.Service->FsyncDirHandler = [&] (auto callContext, auto request) { + UNIT_ASSERT_VALUES_EQUAL(FileSystemId, callContext->FileSystemId); + + if (request->GetDataSync()) { + fsyncDirCalledWithDataSync++; + } else { + fsyncDirCalledWithoutDataSync++; + } + + return MakeFuture(NProto::TFsyncDirResponse()); + }; + bootstrap.Start(); const ui64 nodeId = 123; @@ -519,6 +561,18 @@ Y_UNIT_TEST_SUITE(TFileSystemTest) size = read.GetValue(); UNIT_ASSERT_VALUES_EQUAL(size, 0); + auto fsyncdir = bootstrap.Fuse->SendRequest( + nodeId, handleId, false /* no data sync */); + UNIT_ASSERT_NO_EXCEPTION(fsyncdir.GetValue(WaitTimeout)); + UNIT_ASSERT_EQUAL(1, fsyncDirCalledWithoutDataSync.load()); + UNIT_ASSERT_EQUAL(0, fsyncDirCalledWithDataSync.load()); + + fsyncdir = bootstrap.Fuse->SendRequest( + nodeId, handleId, true /* data sync */); + UNIT_ASSERT_NO_EXCEPTION(fsyncdir.GetValue(WaitTimeout)); + UNIT_ASSERT_EQUAL(1, fsyncDirCalledWithoutDataSync.load()); + UNIT_ASSERT_EQUAL(1, fsyncDirCalledWithDataSync.load()); + auto close = bootstrap.Fuse->SendRequest(nodeId, handleId); UNIT_ASSERT_NO_EXCEPTION(close.GetValue(WaitTimeout)); } diff --git a/cloud/filestore/libs/vhost/request.h b/cloud/filestore/libs/vhost/request.h index 6592d7e52b..d8373e0417 100644 --- a/cloud/filestore/libs/vhost/request.h +++ b/cloud/filestore/libs/vhost/request.h @@ -424,4 +424,30 @@ struct TReleaseRequest } }; +//////////////////////////////////////////////////////////////////////////////// + +struct TFsyncRequest + : public TRequestBase +{ + TFsyncRequest(ui64 nodeId, ui64 fh, bool datasync) + { + In->Header.opcode = FUSE_FSYNC; + In->Header.nodeid = nodeId; + In->Body.fh = fh; + In->Body.fsync_flags = datasync; + } +}; + +struct TFsyncDirRequest + : public TRequestBase +{ + TFsyncDirRequest(ui64 nodeId, ui64 fh, bool datasync) + { + In->Header.opcode = FUSE_FSYNCDIR; + In->Header.nodeid = nodeId; + In->Body.fh = fh; + In->Body.fsync_flags = datasync; + } +}; + } // namespace NCloud::NFileStore::NVhost diff --git a/cloud/filestore/public/api/grpc/service.proto b/cloud/filestore/public/api/grpc/service.proto index 7b7fe2ab5d..924f930834 100644 --- a/cloud/filestore/public/api/grpc/service.proto +++ b/cloud/filestore/public/api/grpc/service.proto @@ -346,6 +346,20 @@ service TFileStoreService }; } + rpc Fsync(TFsyncRequest) returns (TFsyncResponse) { + option (google.api.http) = { + post: "/fsync" + body: "*" + }; + } + + rpc FsyncDir(TFsyncDirRequest) returns (TFsyncDirResponse) { + option (google.api.http) = { + post: "/fsync_dir" + body: "*" + }; + } + // // Locking operations. // diff --git a/cloud/filestore/public/api/protos/data.proto b/cloud/filestore/public/api/protos/data.proto index 2d1d731330..a9d5fc31e7 100644 --- a/cloud/filestore/public/api/protos/data.proto +++ b/cloud/filestore/public/api/protos/data.proto @@ -259,3 +259,60 @@ message TTruncateDataResponse // Optional response headers. TResponseHeaders Headers = 1000; } + +//////////////////////////////////////////////////////////////////////////////// +// Fsync request/response + +message TFsyncRequest +{ + // Optional request headers. + THeaders Headers = 1; + + // FileSystem identifier. + string FileSystemId = 2; + + // Node. + uint64 NodeId = 3; + + // IO handle. + uint64 Handle = 4; + + // If true only the user data should be flushed, not the meta data. + bool DataSync = 5; +} + +message TFsyncResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + + // Optional response headers. + TResponseHeaders Headers = 1000; +} + +//////////////////////////////////////////////////////////////////////////////// +// FsyncDir request/response + +message TFsyncDirRequest +{ + // Optional request headers. + THeaders Headers = 1; + + // FileSystem identifier. + string FileSystemId = 2; + + // Node. + uint64 NodeId = 3; + + // If true only the user data should be flushed, not the meta data. + bool DataSync = 4; +} + +message TFsyncDirResponse +{ + // Optional error, set only if error happened. + NCloud.NProto.TError Error = 1; + + // Optional response headers. + TResponseHeaders Headers = 1000; +} diff --git a/cloud/filestore/tools/analytics/libs/event-log/dump_ut.cpp b/cloud/filestore/tools/analytics/libs/event-log/dump_ut.cpp index 1556b89273..4150891ff5 100644 --- a/cloud/filestore/tools/analytics/libs/event-log/dump_ut.cpp +++ b/cloud/filestore/tools/analytics/libs/event-log/dump_ut.cpp @@ -94,7 +94,7 @@ Y_UNIT_TEST_SUITE(TDumpTest) { const auto requests = GetRequestTypes(); - UNIT_ASSERT_VALUES_EQUAL(70, requests.size()); + UNIT_ASSERT_VALUES_EQUAL(73, requests.size()); ui32 index = 0; #define TEST_REQUEST_TYPE(id, name) \ @@ -151,20 +151,23 @@ Y_UNIT_TEST_SUITE(TDumpTest) TEST_REQUEST_TYPE(43, ReadData); TEST_REQUEST_TYPE(44, WriteData); TEST_REQUEST_TYPE(45, AllocateData); - TEST_REQUEST_TYPE(46, GetSessionEventsStream); - TEST_REQUEST_TYPE(47, StartEndpoint); - TEST_REQUEST_TYPE(48, StopEndpoint); - TEST_REQUEST_TYPE(49, ListEndpoints); - TEST_REQUEST_TYPE(50, KickEndpoint); - TEST_REQUEST_TYPE(51, DescribeData); - TEST_REQUEST_TYPE(52, GenerateBlobIds); - TEST_REQUEST_TYPE(53, AddData); - TEST_REQUEST_TYPE(54, ReadBlob); - TEST_REQUEST_TYPE(55, WriteBlob); + TEST_REQUEST_TYPE(46, Fsync); + TEST_REQUEST_TYPE(47, FsyncDir); + TEST_REQUEST_TYPE(48, GetSessionEventsStream); + TEST_REQUEST_TYPE(49, StartEndpoint); + TEST_REQUEST_TYPE(50, StopEndpoint); + TEST_REQUEST_TYPE(51, ListEndpoints); + TEST_REQUEST_TYPE(52, KickEndpoint); + TEST_REQUEST_TYPE(53, DescribeData); + TEST_REQUEST_TYPE(54, GenerateBlobIds); + TEST_REQUEST_TYPE(55, AddData); + TEST_REQUEST_TYPE(56, ReadBlob); + TEST_REQUEST_TYPE(57, WriteBlob); // Fuse TEST_REQUEST_TYPE(1001, Flush); TEST_REQUEST_TYPE(1002, Fsync); + TEST_REQUEST_TYPE(1003, FsyncDir); // Tablet TEST_REQUEST_TYPE(10001, Flush); diff --git a/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp b/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp index 5cb5ed95b3..1a44e96b9d 100644 --- a/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp +++ b/cloud/filestore/tools/analytics/libs/event-log/request_printer.cpp @@ -567,6 +567,7 @@ IRequestPrinterPtr CreateRequestPrinter(ui32 requestType) switch (static_cast(requestType)) { case NFuse::EFileStoreFuseRequest::Flush: case NFuse::EFileStoreFuseRequest::Fsync: + case NFuse::EFileStoreFuseRequest::FsyncDir: return std::make_shared(); default: break;