diff --git a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go index c9e44d6ab3..ec6cdde311 100644 --- a/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go @@ -156,6 +156,20 @@ func (client *kurtosisBackendLogsDatabaseClient) FilterExistingServiceUuids( return filteredServiceUuidsSet, nil } +func (client *kurtosisBackendLogsDatabaseClient) StartLogFileManagement(ctx context.Context) { + // no log file management needs to be done for this logs db client +} + +func (client *kurtosisBackendLogsDatabaseClient) RemoveEnclaveLogs(enclaveUuid string) error { + // no log file management needs to be done for this logs db client + return nil +} + +func (client *kurtosisBackendLogsDatabaseClient) RemoveAllLogs() error { + // no log file management needs to be done for this logs db client + return nil +} + // ==================================================================================================== // // Private helper functions diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/file_layout.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/file_layout.go new file mode 100644 index 0000000000..40ca4369f6 --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/file_layout.go @@ -0,0 +1,26 @@ +package file_layout + +import ( + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" + "time" +) + +type LogFileLayout interface { + // GetLogFileLayoutFormat returns a string representation the "format" that files are laid out in + // Formats are composed: + // - "/" - representing a nested directory + // - "" - representing where an enclave uuid is inserted + // - "" - representing where a service uuid is inserted + // - time formats specified by strftime https://cplusplus.com/reference/ctime/strftime/ + // - any other ascii text + GetLogFileLayoutFormat() string + + // GetLogFilePath gets the log file path for [serviceUuid] in [enclaveUuid] at [time] + GetLogFilePath(time time.Time, enclaveUuid, serviceUuid string) string + + // GetLogFilePaths retrieves a list of filepaths [filesystem] for [serviceUuid] in [enclaveUuid] + // If [retentionPeriodIntervals] is set to -1, retrieves all filepaths from the currentTime till [retentionPeriod] in order + // If [retentionPeriodIntervals] is positive, retrieves all filepaths within the range [currentTime - retentionPeriod] and [currentTime - (retentionPeriodIntervals) * retentionPeriod] + // Returned filepaths sorted from most recent to least recent + GetLogFilePaths(filesystem volume_filesystem.VolumeFilesystem, retentionPeriod time.Duration, retentionPeriodIntervals int, enclaveUuid, serviceUuid string) ([]string, error) +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout.go new file mode 100644 index 0000000000..c4fcb7fd8f --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout.go @@ -0,0 +1,124 @@ +package file_layout + +import ( + "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" + "golang.org/x/exp/slices" + "math" + "os" + "strconv" + "time" +) + +const ( + oneWeekInHours = 7 * 24 + oneWeekDuration = oneWeekInHours * time.Hour + + // basepath /year/week + PerWeekDirPathStr = "%s%s/%s/" + + // ... enclave uuid/service uuid + PerWeekFilePathFmtStr = PerWeekDirPathStr + "%s/%s%s" +) + +type PerWeekFileLayout struct { + time logs_clock.LogsClock +} + +func NewPerWeekFileLayout(time logs_clock.LogsClock) *PerWeekFileLayout { + return &PerWeekFileLayout{time: time} +} + +func (pwf *PerWeekFileLayout) GetLogFileLayoutFormat() string { + // Right now this format is specifically made for Vector Logs Aggregators format + // This wil be used my Vector LogsAggregator to determine the path to output to + return "/var/log/kurtosis/%%Y/%%V/{{ enclave_uuid }}/{{ service_uuid }}.json" +} + +func (pwf *PerWeekFileLayout) GetLogFilePath(time time.Time, enclaveUuid, serviceUuid string) string { + year, week := time.ISOWeek() + return getLogFilePath(year, week, enclaveUuid, serviceUuid) +} + +func (pwf *PerWeekFileLayout) GetLogFilePaths( + filesystem volume_filesystem.VolumeFilesystem, + retentionPeriod time.Duration, + retentionPeriodIntervals int, + enclaveUuid, serviceUuid string) ([]string, error) { + var paths []string + retentionPeriodInWeeks := DurationToWeeks(retentionPeriod) + + if retentionPeriodIntervals < 0 { + return pwf.getLogFilePathsFromNowTillRetentionPeriod(filesystem, retentionPeriodInWeeks, enclaveUuid, serviceUuid) + } else { + paths = pwf.getLogFilePathsBeyondRetentionPeriod(filesystem, retentionPeriodInWeeks, retentionPeriodIntervals, enclaveUuid, serviceUuid) + } + + return paths, nil +} + +func (pwf *PerWeekFileLayout) getLogFilePathsFromNowTillRetentionPeriod(fs volume_filesystem.VolumeFilesystem, retentionPeriodInWeeks int, enclaveUuid, serviceUuid string) ([]string, error) { + var paths []string + currentTime := pwf.time.Now() + + // scan for first existing log file + firstWeekWithLogs := 0 + for i := 0; i < retentionPeriodInWeeks; i++ { + year, week := currentTime.Add(time.Duration(-i) * oneWeekDuration).ISOWeek() + filePathStr := getLogFilePath(year, week, enclaveUuid, serviceUuid) + if _, err := fs.Stat(filePathStr); err == nil { + paths = append(paths, filePathStr) + firstWeekWithLogs = i + break + } else { + // return if error is not due to nonexistent file path + if !os.IsNotExist(err) { + return paths, err + } + } + } + + // scan for remaining files as far back as they exist before the retention period + for i := firstWeekWithLogs + 1; i < retentionPeriodInWeeks; i++ { + year, week := currentTime.Add(time.Duration(-i) * oneWeekDuration).ISOWeek() + filePathStr := getLogFilePath(year, week, enclaveUuid, serviceUuid) + if _, err := fs.Stat(filePathStr); err != nil { + break + } + paths = append(paths, filePathStr) + } + + // reverse for oldest to most recent + slices.Reverse(paths) + + return paths, nil +} + +func (pwf *PerWeekFileLayout) getLogFilePathsBeyondRetentionPeriod(fs volume_filesystem.VolumeFilesystem, retentionPeriodInWeeks int, retentionPeriodIntervals int, enclaveUuid, serviceUuid string) []string { + var paths []string + currentTime := pwf.time.Now() + + // scan for log files just beyond the retention period + for i := 0; i < retentionPeriodIntervals; i++ { + numWeeksToGoBack := retentionPeriodInWeeks + i + year, weekToRemove := currentTime.Add(time.Duration(-numWeeksToGoBack) * oneWeekDuration).ISOWeek() + filePathStr := getLogFilePath(year, weekToRemove, enclaveUuid, serviceUuid) + if _, err := fs.Stat(filePathStr); err != nil { + continue + } + paths = append(paths, filePathStr) + } + + return paths +} + +func DurationToWeeks(d time.Duration) int { + return int(math.Round(d.Hours() / float64(oneWeekInHours))) +} + +func getLogFilePath(year, week int, enclaveUuid, serviceUuid string) string { + formattedWeekNum := fmt.Sprintf("%02d", week) + return fmt.Sprintf(PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceUuid, volume_consts.Filetype) +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout_test.go new file mode 100644 index 0000000000..3c4731ed1b --- /dev/null +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout/per_week_file_layout_test.go @@ -0,0 +1,381 @@ +package file_layout + +import ( + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" + "github.com/stretchr/testify/require" + "testing" +) + +const ( + testEnclaveUuid = "test-enclave" + testUserService1Uuid = "test-user-service-1" + + retentionPeriodInWeeksForTesting = 5 + + defaultYear = 2023 + defaultDay = 0 // sunday +) + +func TestGetLogFilePaths(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 17 + currentTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(currentTime) + + week12filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 12, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week13filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 13, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week14filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 14, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week15filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 15, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week16filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 16, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week17filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 17, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week12filepath) + _, _ = filesystem.Create(week13filepath) + _, _ = filesystem.Create(week14filepath) + _, _ = filesystem.Create(week15filepath) + _, _ = filesystem.Create(week16filepath) + _, _ = filesystem.Create(week17filepath) + + expectedLogFilePaths := []string{ + week13filepath, + week14filepath, + week15filepath, + week16filepath, + week17filepath, + } + + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsAcrossNewYear(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 2 + currentTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(currentTime) + + // ../week/enclave uuid/service uuid.json + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear-1, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear-1, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear-1, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week51filepath) + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + expectedLogFilePaths := []string{ + week50filepath, + week51filepath, + week52filepath, + week1filepath, + week2filepath, + } + + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsAcrossNewYearWith53Weeks(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 3 + currentTime := logs_clock.NewMockLogsClock(2016, currentWeek, 1) + fileLayout := NewPerWeekFileLayout(currentTime) + + // According to ISOWeek, 2015 has 53 weeks + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2015, 51, 3).Now(), testEnclaveUuid, testUserService1Uuid) + week53filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2015, 52, 3).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2016, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2016, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week3filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2016, 3, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week53filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + _, _ = filesystem.Create(week3filepath) + + expectedLogFilePaths := []string{ + week52filepath, + week53filepath, + week1filepath, + week2filepath, + week3filepath, + } + + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsWithDiffRetentionPeriod(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 2 + mockTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(mockTime) + + // ../week/enclave uuid/service uuid.json + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear-1, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + expectedLogFilePaths := []string{ + week52filepath, + week1filepath, + week2filepath, + } + retentionPeriod := 3 * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsReturnsAllAvailableWeeks(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 2 + currentTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(currentTime) + + // ../week/enclave uuid/service uuid.json + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear-1, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + // should return existing file paths even though log files going all the back to retention period don't exist + expectedLogFilePaths := []string{ + week52filepath, + week1filepath, + week2filepath, + } + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Less(t, len(logFilePaths), retentionPeriodInWeeksForTesting) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsReturnsCorrectPathsIfWeeksMissingInBetween(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 3 + currentTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(currentTime) + + // ../week/enclave uuid/service uuid.json + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 0, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week3filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 3, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week3filepath) + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Len(t, logFilePaths, 1) + require.Equal(t, week3filepath, logFilePaths[0]) // should only return week 3 because week 2 is missing +} + +func TestGetLogFilePathsReturnsCorrectPathsIfCurrentWeekHasNoLogsYet(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + currentWeek := 3 + currentTime := logs_clock.NewMockLogsClock(defaultYear, currentWeek, defaultDay) + fileLayout := NewPerWeekFileLayout(currentTime) + + // ../week/enclave uuid/service uuid.json + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(defaultYear, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + // no logs for week current week exist yet + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + // should return week 1 and 2 logs, even though no logs for current week yet + expectedLogFilePaths := []string{ + week1filepath, + week2filepath, + } + + retentionPeriod := retentionPeriodInWeeksForTesting * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsOneIntervalBeyondRetentionPeriod(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + fileLayout := NewPerWeekFileLayout(mockTime) + + week49filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 49, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week49filepath) + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week51filepath) + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + retentionPeriod := 5 * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, 1, testEnclaveUuid, testUserService1Uuid) + require.NoError(t, err) + require.Len(t, logFilePaths, 1) + require.Equal(t, logFilePaths[0], week49filepath) +} + +func TestGetLogFilePathsTwoIntervalBeyondRetentionPeriod(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + fileLayout := NewPerWeekFileLayout(mockTime) + + week48filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 48, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week49filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 49, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week48filepath) + _, _ = filesystem.Create(week49filepath) + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week51filepath) + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + expectedLogFilePaths := []string{ + week49filepath, + week48filepath, + } + + retentionPeriod := 5 * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, 2, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Len(t, logFilePaths, 2) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} + +func TestGetLogFilePathsWithNoPathsBeyondRetentionPeriod(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + fileLayout := NewPerWeekFileLayout(mockTime) + + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week51filepath) + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + retentionPeriod := 5 * oneWeekDuration + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, 1, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Len(t, logFilePaths, 0) +} + +func TestGetLogFilePathsWithMissingPathBetweenIntervals(t *testing.T) { + filesystem := volume_filesystem.NewMockedVolumeFilesystem() + + mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + fileLayout := NewPerWeekFileLayout(mockTime) + + week47filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 48, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week49filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 49, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) + + _, _ = filesystem.Create(week47filepath) + // 48 is missing + _, _ = filesystem.Create(week49filepath) + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week50filepath) + _, _ = filesystem.Create(week51filepath) + _, _ = filesystem.Create(week52filepath) + _, _ = filesystem.Create(week1filepath) + _, _ = filesystem.Create(week2filepath) + + expectedLogFilePaths := []string{ + week49filepath, + week47filepath, + } + + retentionPeriod := 5 * oneWeekDuration + // the expected behavior is return all filepaths, even if some are missing + logFilePaths, err := fileLayout.GetLogFilePaths(filesystem, retentionPeriod, 3, testEnclaveUuid, testUserService1Uuid) + + require.NoError(t, err) + require.Len(t, logFilePaths, 2) + require.Equal(t, len(expectedLogFilePaths), len(logFilePaths)) + for i, filePath := range expectedLogFilePaths { + require.Equal(t, filePath, logFilePaths[i]) + } +} diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go index d9224ec674..9338df5d31 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager.go @@ -7,6 +7,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" @@ -28,6 +29,8 @@ type LogFileManager struct { filesystem volume_filesystem.VolumeFilesystem + fileLayout file_layout.LogFileLayout + time logs_clock.LogsClock logRetentionPeriodInWeeks int @@ -36,11 +39,13 @@ type LogFileManager struct { func NewLogFileManager( kurtosisBackend backend_interface.KurtosisBackend, filesystem volume_filesystem.VolumeFilesystem, + fileLayout file_layout.LogFileLayout, time logs_clock.LogsClock, logRetentionPeriodInWeeks int) *LogFileManager { return &LogFileManager{ kurtosisBackend: kurtosisBackend, filesystem: filesystem, + fileLayout: fileLayout, time: time, logRetentionPeriodInWeeks: logRetentionPeriodInWeeks, } @@ -51,12 +56,12 @@ func (manager *LogFileManager) StartLogFileManagement(ctx context.Context) { // Schedule thread for removing log files beyond retention period go func() { logrus.Debugf("Scheduling log removal for log retention every '%v' hours...", volume_consts.RemoveLogsWaitHours) - manager.RemoveLogsBeyondRetentionPeriod() + manager.RemoveLogsBeyondRetentionPeriod(ctx) logRemovalTicker := time.NewTicker(volume_consts.RemoveLogsWaitHours) for range logRemovalTicker.C { logrus.Debug("Attempting to remove old log file paths...") - manager.RemoveLogsBeyondRetentionPeriod() + manager.RemoveLogsBeyondRetentionPeriod(ctx) } }() @@ -90,8 +95,6 @@ func (manager *LogFileManager) StartLogFileManagement(ctx context.Context) { func (manager *LogFileManager) CreateLogFiles(ctx context.Context) error { var err error - year, week := manager.time.Now().ISOWeek() - enclaveToServicesMap, err := manager.getEnclaveAndServiceInfo(ctx) if err != nil { // already wrapped with propagate @@ -104,18 +107,18 @@ func (manager *LogFileManager) CreateLogFiles(ctx context.Context) error { serviceNameStr := string(serviceRegistration.GetName()) serviceShortUuidStr := uuid_generator.ShortenedUUIDString(serviceUuidStr) - serviceUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceUuidStr) + serviceUuidFilePathStr := manager.fileLayout.GetLogFilePath(manager.time.Now(), string(enclaveUuid), serviceUuidStr) if err = manager.createLogFileIdempotently(serviceUuidFilePathStr); err != nil { return err } - serviceNameFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceNameStr) + serviceNameFilePathStr := manager.fileLayout.GetLogFilePath(manager.time.Now(), string(enclaveUuid), serviceNameStr) if err = manager.createSymlinkLogFile(serviceUuidFilePathStr, serviceNameFilePathStr); err != nil { return err } logrus.Tracef("Created symlinked log file: '%v'", serviceNameFilePathStr) - serviceShortUuidFilePathStr := getFilepathStr(year, week, string(enclaveUuid), serviceShortUuidStr) + serviceShortUuidFilePathStr := manager.fileLayout.GetLogFilePath(manager.time.Now(), string(enclaveUuid), serviceShortUuidStr) if err = manager.createSymlinkLogFile(serviceUuidFilePathStr, serviceShortUuidFilePathStr); err != nil { return err } @@ -127,16 +130,56 @@ func (manager *LogFileManager) CreateLogFiles(ctx context.Context) error { } // RemoveLogsBeyondRetentionPeriod implements the Job cron interface. It removes logs a week older than the log retention period. -func (manager *LogFileManager) RemoveLogsBeyondRetentionPeriod() { - // compute the next oldest week - year, weekToRemove := manager.time.Now().Add(time.Duration(-manager.logRetentionPeriodInWeeks) * oneWeek).ISOWeek() - - // remove directory for that week - oldLogsDirPath := getLogsDirPathForWeek(year, weekToRemove) - if err := manager.filesystem.RemoveAll(oldLogsDirPath); err != nil { - logrus.Warnf("An error occurred removing old logs at the following path '%v': %v\n", oldLogsDirPath, err) +func (manager *LogFileManager) RemoveLogsBeyondRetentionPeriod(ctx context.Context) { + var pathsToRemove []string + enclaveToServicesMap, err := manager.getEnclaveAndServiceInfo(ctx) + if err != nil { + logrus.Errorf("An error occurred getting enclave and service info while removing logs beyond retention: %v", err) + return + } + for enclaveUuid, serviceRegistrations := range enclaveToServicesMap { + for _, serviceRegistration := range serviceRegistrations { + serviceUuidStr := string(serviceRegistration.GetUUID()) + serviceNameStr := string(serviceRegistration.GetName()) + serviceShortUuidStr := uuid_generator.ShortenedUUIDString(serviceUuidStr) + + retentionPeriod := time.Duration(manager.logRetentionPeriodInWeeks) * oneWeek + oldServiceLogFilesByUuid, err := manager.fileLayout.GetLogFilePaths(manager.filesystem, retentionPeriod, 1, string(enclaveUuid), serviceUuidStr) + if err != nil { + logrus.Errorf("An error occurred getting log file paths for service '%v' in enclave '%v' logs beyond retention: %v", serviceUuidStr, enclaveUuid, err) + } else { + pathsToRemove = append(pathsToRemove, oldServiceLogFilesByUuid...) + } + + oldServiceLogFilesByName, err := manager.fileLayout.GetLogFilePaths(manager.filesystem, retentionPeriod, 1, string(enclaveUuid), serviceNameStr) + if err != nil { + logrus.Errorf("An error occurred getting log file paths for service '%v' in enclave '%v' logs beyond retention: %v", serviceNameStr, enclaveUuid, err) + } else { + pathsToRemove = append(pathsToRemove, oldServiceLogFilesByName...) + } + + oldServiceLogFilesByShortUuid, err := manager.fileLayout.GetLogFilePaths(manager.filesystem, retentionPeriod, 1, string(enclaveUuid), serviceShortUuidStr) + if err != nil { + logrus.Errorf("An error occurred getting log file paths for service '%v' in enclave '%v' logs beyond retention: %v", serviceShortUuidStr, enclaveUuid, err) + } else { + pathsToRemove = append(pathsToRemove, oldServiceLogFilesByShortUuid...) + } + } + } + + successfullyRemovedLogFiles := []string{} + failedToRemoveLogFiles := []string{} + for _, pathToRemove := range pathsToRemove { + if err := manager.filesystem.Remove(pathToRemove); err != nil { + logrus.Warnf("An error occurred removing old log file at the following path '%v': %v\n", pathsToRemove, err) + failedToRemoveLogFiles = append(failedToRemoveLogFiles, pathToRemove) + } + successfullyRemovedLogFiles = append(successfullyRemovedLogFiles, pathToRemove) + } + logrus.Debugf("Successfully removed the following logs beyond retention period at the following path: '%v'", successfullyRemovedLogFiles) + if len(failedToRemoveLogFiles) > 0 { + logrus.Errorf("Failed to remove the following logs beyond retention period at the following path: '%v'", failedToRemoveLogFiles) } - logrus.Debugf("Removed logs beyond retention period at the following path: '%v'", oldLogsDirPath) } func (manager *LogFileManager) RemoveAllLogs() error { @@ -210,13 +253,6 @@ func (manager *LogFileManager) createSymlinkLogFile(targetLogFilePath, symlinkLo return nil } -// TODO: Implement a FilePath Builder to centralize log file path creation across the entire module -// creates a filepath of format //year/week//serviceIdentifier. -func getFilepathStr(year, week int, enclaveUuid, serviceIdentifier string) string { - formattedWeekNum := fmt.Sprintf("%02d", week) - return fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceIdentifier, volume_consts.Filetype) -} - // creates a directory path of format //year/week// func getEnclaveLogsDirPath(year, week int, enclaveUuid string) string { logsDirPathForYearAndWeek := getLogsDirPathForWeek(year, week) diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go index b5c96da757..5d502597c5 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager/log_file_manager_test.go @@ -8,6 +8,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/port_spec" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/uuid_generator" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" "github.com/stretchr/testify/require" @@ -25,17 +26,20 @@ const ( ) func TestRemoveLogsBeyondRetentionPeriod(t *testing.T) { - mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) + ctx := context.Background() mockTime := logs_clock.NewMockLogsClock(2023, 2, defaultDay) + fileLayout := file_layout.NewPerWeekFileLayout(mockTime) + + mockKurtosisBackend := getMockedKurtosisBackendWithEnclavesAndServices(ctx, t, mockTime) // setup filesystem mockFs := volume_filesystem.NewMockedVolumeFilesystem() - week49filepath := getFilepathStr(2022, 49, testEnclaveUuid, testUserService1Uuid) - week50filepath := getFilepathStr(2022, 50, testEnclaveUuid, testUserService1Uuid) - week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) - week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) - week1filepath := getFilepathStr(2023, 1, testEnclaveUuid, testUserService1Uuid) - week2filepath := getFilepathStr(2023, 2, testEnclaveUuid, testUserService1Uuid) + week49filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 49, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week50filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 50, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week1filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 1, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week2filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2023, 2, 0).Now(), testEnclaveUuid, testUserService1Uuid) _, _ = mockFs.Create(week49filepath) _, _ = mockFs.Create(week50filepath) @@ -44,8 +48,8 @@ func TestRemoveLogsBeyondRetentionPeriod(t *testing.T) { _, _ = mockFs.Create(week1filepath) _, _ = mockFs.Create(week2filepath) - logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime, 5) - logFileManager.RemoveLogsBeyondRetentionPeriod() // should remove week 49 logs + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, fileLayout, mockTime, 5) + logFileManager.RemoveLogsBeyondRetentionPeriod(ctx) // should remove week 49 logs _, err := mockFs.Stat(week49filepath) require.Error(t, err) @@ -55,21 +59,22 @@ func TestRemoveLogsBeyondRetentionPeriod(t *testing.T) { func TestRemoveEnclaveLogs(t *testing.T) { mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) + fileLayout := file_layout.NewPerWeekFileLayout(mockTime) // setup filesystem mockFs := volume_filesystem.NewMockedVolumeFilesystem() - week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) - week52filepathDiffEnclave := getFilepathStr(2022, 52, "enclaveOne", "serviceTwo") - week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) - week52filepathDiffService := getFilepathStr(2022, 52, testEnclaveUuid, "serviceThree") + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepathDiffEnclave := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), "enclaveOne", "serviceTwo") + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepathDiffService := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, "serviceThree") _, _ = mockFs.Create(week51filepath) _, _ = mockFs.Create(week52filepathDiffEnclave) _, _ = mockFs.Create(week52filepath) _, _ = mockFs.Create(week52filepathDiffService) - logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime, 5) + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, fileLayout, mockTime, 5) err := logFileManager.RemoveEnclaveLogs(testEnclaveUuid) // should remove only all log files for enclave one require.NoError(t, err) @@ -93,21 +98,22 @@ func TestRemoveEnclaveLogs(t *testing.T) { func TestRemoveAllLogs(t *testing.T) { mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) + fileLayout := file_layout.NewPerWeekFileLayout(mockTime) // setup filesystem mockFs := volume_filesystem.NewMockedVolumeFilesystem() - week51filepath := getFilepathStr(2022, 51, testEnclaveUuid, testUserService1Uuid) - week52filepathDiffEnclave := getFilepathStr(2022, 52, "enclaveOne", "serviceTwo") - week52filepath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) - week52filepathDiffService := getFilepathStr(2022, 52, testEnclaveUuid, "serviceThree") + week51filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 51, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepathDiffEnclave := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), "enclaveOne", "serviceTwo") + week52filepath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + week52filepathDiffService := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, "serviceThree") _, _ = mockFs.Create(week51filepath) _, _ = mockFs.Create(week52filepathDiffEnclave) _, _ = mockFs.Create(week52filepath) _, _ = mockFs.Create(week52filepathDiffService) - logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime, 5) + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, fileLayout, mockTime, 5) err := logFileManager.RemoveAllLogs() require.NoError(t, err) @@ -132,9 +138,31 @@ func TestRemoveAllLogs(t *testing.T) { func TestCreateLogFiles(t *testing.T) { mockTime := logs_clock.NewMockLogsClock(2022, 52, defaultDay) mockFs := volume_filesystem.NewMockedVolumeFilesystem() + fileLayout := file_layout.NewPerWeekFileLayout(mockTime) // setup kurtosis backend ctx := context.Background() + mockKurtosisBackend := getMockedKurtosisBackendWithEnclavesAndServices(ctx, t, mockTime) + + expectedServiceUuidFilePath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Uuid) + expectedServiceNameFilePath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, testUserService1Name) + expectedServiceShortUuidFilePath := fileLayout.GetLogFilePath(logs_clock.NewMockLogsClock(2022, 52, 0).Now(), testEnclaveUuid, uuid_generator.ShortenedUUIDString(testUserService1Uuid)) + + logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, fileLayout, mockTime, 5) + err := logFileManager.CreateLogFiles(ctx) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceUuidFilePath) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceNameFilePath) + require.NoError(t, err) + + _, err = mockFs.Stat(expectedServiceShortUuidFilePath) + require.NoError(t, err) +} + +func getMockedKurtosisBackendWithEnclavesAndServices(ctx context.Context, t *testing.T, mockTime logs_clock.LogsClock) *backend_interface.MockKurtosisBackend { mockKurtosisBackend := backend_interface.NewMockKurtosisBackend(t) // mock enclave @@ -160,21 +188,5 @@ func TestCreateLogFiles(t *testing.T) { EXPECT(). GetUserServices(ctx, enclaveUuid, &service.ServiceFilters{Names: nil, UUIDs: nil, Statuses: nil}). Return(servicesMap, nil) - - expectedServiceUuidFilePath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Uuid) - expectedServiceNameFilePath := getFilepathStr(2022, 52, testEnclaveUuid, testUserService1Name) - expectedServiceShortUuidFilePath := getFilepathStr(2022, 52, testEnclaveUuid, uuid_generator.ShortenedUUIDString(testUserService1Uuid)) - - logFileManager := NewLogFileManager(mockKurtosisBackend, mockFs, mockTime, 5) - err := logFileManager.CreateLogFiles(ctx) - require.NoError(t, err) - - _, err = mockFs.Stat(expectedServiceUuidFilePath) - require.NoError(t, err) - - _, err = mockFs.Stat(expectedServiceNameFilePath) - require.NoError(t, err) - - _, err = mockFs.Stat(expectedServiceShortUuidFilePath) - require.NoError(t, err) + return mockKurtosisBackend } diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go index 6880e8db3e..67bd58b6bb 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go @@ -5,6 +5,7 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" @@ -22,17 +23,21 @@ type persistentVolumeLogsDatabaseClient struct { filesystem volume_filesystem.VolumeFilesystem + logFileManager *log_file_manager.LogFileManager + streamStrategy stream_logs_strategy.StreamLogsStrategy } func NewPersistentVolumeLogsDatabaseClient( kurtosisBackend backend_interface.KurtosisBackend, filesystem volume_filesystem.VolumeFilesystem, + logFileManager *log_file_manager.LogFileManager, streamStrategy stream_logs_strategy.StreamLogsStrategy, ) *persistentVolumeLogsDatabaseClient { return &persistentVolumeLogsDatabaseClient{ kurtosisBackend: kurtosisBackend, filesystem: filesystem, + logFileManager: logFileManager, streamStrategy: streamStrategy, } } @@ -127,6 +132,18 @@ func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids( return filteredServiceUuidsSet, nil } +func (client *persistentVolumeLogsDatabaseClient) StartLogFileManagement(ctx context.Context) { + client.logFileManager.StartLogFileManagement(ctx) +} + +func (client *persistentVolumeLogsDatabaseClient) RemoveEnclaveLogs(enclaveUuid string) error { + return client.logFileManager.RemoveEnclaveLogs(enclaveUuid) +} + +func (client *persistentVolumeLogsDatabaseClient) RemoveAllLogs() error { + return client.logFileManager.RemoveAllLogs() +} + // ==================================================================================================== // // Private helper functions diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go index 82f14c00e9..d245581bde 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go @@ -6,6 +6,8 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts" @@ -694,7 +696,11 @@ func executeStreamCallAndGetReceivedServiceLogLines( kurtosisBackend := backend_interface.NewMockKurtosisBackend(t) - logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, underlyingFs, streamStrategy) + // no log file management is done in these tests so values for logFileManager aren't important + mockTime := logs_clock.NewMockLogsClock(0, 0, 0) + fileLayout := file_layout.NewPerWeekFileLayout(mockTime) + logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, underlyingFs, fileLayout, mockTime, 0) + logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, underlyingFs, logFileManager, streamStrategy) userServiceLogsByUuidChan, errChan, receivedCancelCtxFunc, err := logsDatabaseClient.StreamUserServiceLogs(ctx, enclaveUuid, userServiceUuids, logLinesFilters, shouldFollowLogs, defaultShouldReturnAllLogs, defaultNumLogLines) if err != nil { diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go index 1717721392..26d7be80ae 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy.go @@ -5,10 +5,8 @@ import ( "context" "encoding/json" "errors" - "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "io" - "os" - "strconv" "strings" "time" @@ -21,7 +19,6 @@ import ( "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" "github.com/kurtosis-tech/stacktrace" "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" ) const ( @@ -119,43 +116,10 @@ func (strategy *PerWeekStreamLogsStrategy) StreamLogs( // - The list of file paths is returned in order of oldest logs to most recent logs e.g. [ 03/80124/1234.json, /04/801234/1234.json, ...] // - If a file path does not exist, the function with exits and returns whatever file paths were found func (strategy *PerWeekStreamLogsStrategy) getLogFilePaths(filesystem volume_filesystem.VolumeFilesystem, retentionPeriodInWeeks int, enclaveUuid, serviceUuid string) ([]string, error) { - var paths []string - currentTime := strategy.time.Now() - - // scan for first existing log file - firstWeekWithLogs := 0 - for i := 0; i < retentionPeriodInWeeks; i++ { - year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek() - // %02d to format week num with leading zeros so 1-9 are converted to 01-09 for %V format - formattedWeekNum := fmt.Sprintf("%02d", week) - filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceUuid, volume_consts.Filetype) - if _, err := filesystem.Stat(filePathStr); err == nil { - paths = append(paths, filePathStr) - firstWeekWithLogs = i - break - } else { - // return if error is not due to nonexistent file path - if !os.IsNotExist(err) { - return paths, err - } - } - } - - // scan for remaining files as far back as they exist - for i := firstWeekWithLogs + 1; i < retentionPeriodInWeeks; i++ { - year, week := currentTime.Add(time.Duration(-i) * oneWeek).ISOWeek() - formattedWeekNum := fmt.Sprintf("%02d", week) - filePathStr := fmt.Sprintf(volume_consts.PerWeekFilePathFmtStr, volume_consts.LogsStorageDirpath, strconv.Itoa(year), formattedWeekNum, enclaveUuid, serviceUuid, volume_consts.Filetype) - if _, err := filesystem.Stat(filePathStr); err != nil { - break - } - paths = append(paths, filePathStr) - } - - // reverse for oldest to most recent - slices.Reverse(paths) - - return paths, nil + // TODO: embed FileLayout into StreamLogsStrategy interface + perWeekFileLayout := file_layout.NewPerWeekFileLayout(strategy.time) + retentionPeriod := time.Duration(retentionPeriodInWeeks) * oneWeek + return perWeekFileLayout.GetLogFilePaths(filesystem, retentionPeriod, -1, enclaveUuid, serviceUuid) } // Returns a Reader over all logs in [logFilePaths] and the open file descriptors of the associated [logFilePaths] diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go index 1157eca522..6339688daa 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy/per_week_stream_logs_strategy_test.go @@ -25,6 +25,8 @@ const ( defaultDay = 0 // sunday ) +// TODO: migrate GetLogFilePaths tests to FileLayout interface when it is fully merged +// for now, leave them duplicated so there's an extra layer of testing as the migration happens func TestGetLogFilePaths(t *testing.T) { filesystem := volume_filesystem.NewMockedVolumeFilesystem() diff --git a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go index ab959633a3..a2f53c054c 100644 --- a/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go +++ b/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts/consts.go @@ -27,6 +27,7 @@ const ( // basepath/enclave uuid/service uuid PerFileFmtStr = "%s%s/%s%s" + // TODO: remove these from consts once PerWeekFileLayout is fully merged // basepath /year/week PerWeekDirPathStr = "%s%s/%s/" diff --git a/engine/server/engine/centralized_logs/logs_database_client.go b/engine/server/engine/centralized_logs/logs_database_client.go index e9779426ba..c7075d0140 100644 --- a/engine/server/engine/centralized_logs/logs_database_client.go +++ b/engine/server/engine/centralized_logs/logs_database_client.go @@ -31,4 +31,10 @@ type LogsDatabaseClient interface { map[service.ServiceUUID]bool, error, ) + + StartLogFileManagement(ctx context.Context) + + RemoveEnclaveLogs(enclaveUuid string) error + + RemoveAllLogs() error } diff --git a/engine/server/engine/enclave_manager/enclave_manager.go b/engine/server/engine/enclave_manager/enclave_manager.go index fbb64957bd..f7d98bf0b0 100644 --- a/engine/server/engine/enclave_manager/enclave_manager.go +++ b/engine/server/engine/enclave_manager/enclave_manager.go @@ -3,12 +3,12 @@ package enclave_manager import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs" "sort" "strings" "sync" "time" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/metrics-library/golang/lib/metrics_client" dockerTypes "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types" @@ -64,10 +64,11 @@ type EnclaveManager struct { // this is an append only list allExistingAndHistoricalIdentifiers []*types.EnclaveIdentifiers - enclaveCreator *EnclaveCreator - enclavePool *EnclavePool - enclaveEnvVars string - enclaveLogFileManager *log_file_manager.LogFileManager + enclaveCreator *EnclaveCreator + enclavePool *EnclavePool + enclaveEnvVars string + + logsDbClient centralized_logs.LogsDatabaseClient metricsUserID string didUserAcceptSendingMetrics bool @@ -83,7 +84,7 @@ func CreateEnclaveManager( engineVersion string, poolSize uint8, enclaveEnvVars string, - enclaveLogFileManager *log_file_manager.LogFileManager, + logsDbClient centralized_logs.LogsDatabaseClient, metricsUserID string, didUserAcceptSendingMetrics bool, isCI bool, @@ -114,7 +115,7 @@ func CreateEnclaveManager( enclaveCreator: enclaveCreator, enclavePool: enclavePool, enclaveEnvVars: enclaveEnvVars, - enclaveLogFileManager: enclaveLogFileManager, + logsDbClient: logsDbClient, metricsUserID: metricsUserID, didUserAcceptSendingMetrics: didUserAcceptSendingMetrics, isCI: isCI, @@ -277,7 +278,7 @@ func (manager *EnclaveManager) DestroyEnclave(ctx context.Context, enclaveIdenti return stacktrace.Propagate(err, "An error occurred destroying the enclave") } if _, found := successfullyDestroyedEnclaves[enclaveUuid]; found { - if err = manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveUuid)); err != nil { + if err = manager.logsDbClient.RemoveEnclaveLogs(string(enclaveUuid)); err != nil { return stacktrace.Propagate(err, "An error occurred attempting to remove enclave '%v' logs after it was destroyed.", enclaveIdentifier) } return nil @@ -601,7 +602,7 @@ func (manager *EnclaveManager) cleanEnclaves( for enclaveId := range successfullyDestroyedEnclaves { successfullyDestroyedEnclaveIdStrs = append(successfullyDestroyedEnclaveIdStrs, string(enclaveId)) - if err := manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveId)); err != nil { + if err := manager.logsDbClient.RemoveEnclaveLogs(string(enclaveId)); err != nil { logRemovalErr := stacktrace.Propagate(err, "An error occurred removing enclave '%v' logs.", enclaveId) enclaveDestructionErrors = append(enclaveDestructionErrors, logRemovalErr) } diff --git a/engine/server/engine/main.go b/engine/server/engine/main.go index 5698808c01..e9945db5c0 100644 --- a/engine/server/engine/main.go +++ b/engine/server/engine/main.go @@ -8,6 +8,7 @@ package main import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout" "io/fs" "net" "net/http" @@ -174,14 +175,8 @@ func runMain() error { if err != nil { return stacktrace.Propagate(err, "An error occurred getting the Kurtosis backend for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig) } - logsDatabaseClient := getLogsDatabaseClient(serverArgs.KurtosisBackendType, kurtosisBackend) - - // TODO: Move log file management into LogsDatabaseClient - osFs := volume_filesystem.NewOsVolumeFilesystem() - realTime := logs_clock.NewRealClock() - logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, realTime, volume_consts.LogRetentionPeriodInWeeks) - logFileManager.StartLogFileManagement(ctx) + logsDatabaseClient.StartLogFileManagement(ctx) enclaveManager, err := getEnclaveManager( kurtosisBackend, @@ -189,7 +184,7 @@ func runMain() error { serverArgs.ImageVersionTag, serverArgs.PoolSize, serverArgs.EnclaveEnvVars, - logFileManager, + logsDatabaseClient, serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics, serverArgs.IsCI, @@ -290,7 +285,6 @@ func runMain() error { serverArgs, enclaveManager, logsDatabaseClient, - logFileManager, metricsClient, ) if err != nil { @@ -306,7 +300,6 @@ func runMain() error { serverArgs.MetricsUserID, serverArgs.DidUserAcceptSendingMetrics, logsDatabaseClient, - logFileManager, metricsClient) apiPath, handler := kurtosis_engine_rpc_api_bindingsconnect.NewEngineServiceHandler(engineConnectServer) defer func() { @@ -329,7 +322,7 @@ func getEnclaveManager( engineVersion string, poolSize uint8, enclaveEnvVars string, - enclaveLogFileManager *log_file_manager.LogFileManager, + logsDbClient centralized_logs.LogsDatabaseClient, metricsUserID string, didUserAcceptSendingMetrics bool, isCI bool, @@ -358,7 +351,7 @@ func getEnclaveManager( engineVersion, poolSize, enclaveEnvVars, - enclaveLogFileManager, + logsDbClient, metricsUserID, didUserAcceptSendingMetrics, isCI, @@ -411,10 +404,14 @@ func getLogsDatabaseClient(kurtosisBackendType args.KurtosisBackendType, kurtosi var logsDatabaseClient centralized_logs.LogsDatabaseClient switch kurtosisBackendType { case args.KurtosisBackendType_Docker: - osFs := volume_filesystem.NewOsVolumeFilesystem() realTime := logs_clock.NewRealClock() + + osFs := volume_filesystem.NewOsVolumeFilesystem() + perWeekFileLayout := file_layout.NewPerWeekFileLayout(realTime) + logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, perWeekFileLayout, realTime, volume_consts.LogRetentionPeriodInWeeks) perWeekStreamLogsStrategy := stream_logs_strategy.NewPerWeekStreamLogsStrategy(realTime, volume_consts.LogRetentionPeriodInWeeks) - logsDatabaseClient = persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, perWeekStreamLogsStrategy) + + logsDatabaseClient = persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, logFileManager, perWeekStreamLogsStrategy) case args.KurtosisBackendType_Kubernetes: logsDatabaseClient = kurtosis_backend.NewKurtosisBackendLogsDatabaseClient(kurtosisBackend) } @@ -436,7 +433,6 @@ func restApiServer( serverArgs *args.EngineServerArgs, enclave_manager *enclave_manager.EnclaveManager, logsDatabaseClient centralized_logs.LogsDatabaseClient, - logFileManager *log_file_manager.LogFileManager, metricsClient metrics_client.MetricsClient, ) error { @@ -464,7 +460,7 @@ func restApiServer( engineRuntime := restApi.EngineRuntime{ ImageVersionTag: serverArgs.ImageVersionTag, EnclaveManager: enclave_manager, - LogFileManager: logFileManager, + LogsDbClient: logsDatabaseClient, MetricsClient: metricsClient, } engineApi.RegisterHandlers(echoApiRouter, engineApi.NewStrictHandler(engineRuntime, nil)) @@ -481,7 +477,6 @@ func restApiServer( MetricsUserID: serverArgs.MetricsUserID, DidUserAcceptSendingMetrics: serverArgs.DidUserAcceptSendingMetrics, LogsDatabaseClient: logsDatabaseClient, - LogFileManager: logFileManager, MetricsClient: metricsClient, AsyncStarlarkLogs: asyncStarlarkLogs, CorsConfig: *corsConfig, diff --git a/engine/server/engine/server/engine_connect_server_service.go b/engine/server/engine/server/engine_connect_server_service.go index 3fc908e916..ea2d266e9e 100644 --- a/engine/server/engine/server/engine_connect_server_service.go +++ b/engine/server/engine/server/engine_connect_server_service.go @@ -10,7 +10,6 @@ import ( "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave" user_service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline" "github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/types" @@ -41,8 +40,6 @@ type EngineConnectServerService struct { // The client for consuming container logs from the logs database logsDatabaseClient centralized_logs.LogsDatabaseClient - logFileManager *log_file_manager.LogFileManager - metricsClient metrics_client.MetricsClient } @@ -52,7 +49,6 @@ func NewEngineConnectServerService( metricsUserId string, didUserAcceptSendingMetrics bool, logsDatabaseClient centralized_logs.LogsDatabaseClient, - logFileManager *log_file_manager.LogFileManager, metricsClient metrics_client.MetricsClient, ) *EngineConnectServerService { service := &EngineConnectServerService{ @@ -61,7 +57,6 @@ func NewEngineConnectServerService( metricsUserID: metricsUserId, didUserAcceptSendingMetrics: didUserAcceptSendingMetrics, logsDatabaseClient: logsDatabaseClient, - logFileManager: logFileManager, metricsClient: metricsClient, } return service @@ -265,7 +260,7 @@ func (service *EngineConnectServerService) Clean(ctx context.Context, connectArg return nil, stacktrace.Propagate(err, "An error occurred while cleaning enclaves") } if args.GetShouldCleanAll() { - if err = service.logFileManager.RemoveAllLogs(); err != nil { + if err = service.logsDatabaseClient.RemoveAllLogs(); err != nil { return nil, stacktrace.Propagate(err, "An error occurred removing all logs.") } } diff --git a/engine/server/engine/server/engine_rest_api_handler.go b/engine/server/engine/server/engine_rest_api_handler.go index e7f9665024..c3d74540b4 100644 --- a/engine/server/engine/server/engine_rest_api_handler.go +++ b/engine/server/engine/server/engine_rest_api_handler.go @@ -3,9 +3,9 @@ package server import ( "context" "fmt" + "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs" "net/http" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/mapping/to_http" "github.com/kurtosis-tech/kurtosis/engine/server/engine/types" @@ -23,7 +23,7 @@ type EngineRuntime struct { EnclaveManager *enclave_manager.EnclaveManager - LogFileManager *log_file_manager.LogFileManager + LogsDbClient centralized_logs.LogsDatabaseClient MetricsClient metrics_client.MetricsClient } @@ -41,7 +41,7 @@ func (engine EngineRuntime) DeleteEnclaves(ctx context.Context, request api.Dele }, nil } if removeAll { - if err = engine.LogFileManager.RemoveAllLogs(); err != nil { + if err = engine.LogsDbClient.RemoveAllLogs(); err != nil { response := internalErrorResponseInfof(err, "An error occurred removing all logs") return api.DeleteEnclavesdefaultJSONResponse{ Body: response, diff --git a/engine/server/engine/server/websocket_api_handler.go b/engine/server/engine/server/websocket_api_handler.go index 8cebce2683..7d5b57abe1 100644 --- a/engine/server/engine/server/websocket_api_handler.go +++ b/engine/server/engine/server/websocket_api_handler.go @@ -7,7 +7,6 @@ import ( user_service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service" "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs" - "github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager" "github.com/kurtosis-tech/kurtosis/engine/server/engine/mapping/to_http" "github.com/kurtosis-tech/kurtosis/engine/server/engine/streaming" @@ -37,8 +36,6 @@ type WebSocketRuntime struct { // The clients for consuming container logs from the logs' database server LogsDatabaseClient centralized_logs.LogsDatabaseClient - LogFileManager *log_file_manager.LogFileManager - MetricsClient metrics_client.MetricsClient // Pool of Starlark log streamers create by package/script runs