Test scripts

cloud/filestore/bin/initctl.sh

@@ -7,18 +7,26 @@ find_bin_dir() {
 }
 
 BIN_DIR=`find_bin_dir`
-LOGS_DIR=$BIN_DIR/data
+export LOGS_DIR=$BIN_DIR/data
 
-SERVER_PORT=${SERVER_PORT:-9021}
-VHOST_PORT=${VHOST_PORT:-9022}
+export SERVER_PORT=${SERVER_PORT:-9021}
+export VHOST_PORT=${VHOST_PORT:-9022}
 
-FS=${FS:-"nfs"}
-SHARD_COUNT=${SHARD_COUNT:-0}
-BLOCK_SIZE=${BLOCK_SIZE:-4096}
-MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"}
-VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock}
+export FS=${FS:-"nfs"}
+export SHARD_COUNT=${SHARD_COUNT:-0}
+export BLOCK_SIZE=${BLOCK_SIZE:-4096}
+export MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"}
+export VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock}
 
-PID_FILE=$BIN_DIR/pids.txt
+export PID_FILE=$BIN_DIR/pids.txt
+
+if [ -z "$FILESTORE_APPS_PATH" ]; then
+    REPO_ROOT=$BIN_DIR/../../..
+    FILESTORE_APPS_PATH=cloud/filestore/buildall/cloud/filestore/apps
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/client
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/server
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/vhost
+fi
 
 ################################################################################
 # STOP/KILL

cloud/filestore/tools/testing/loadtest/test/lib.sh

@@ -0,0 +1,83 @@
+
+if [ -z ${CONFIGURED} ]; then
+export CONFIGURED=1
+
+[ -z "${VERBOSE=${V}}" ] && set +x
+[ -n "${VERBOSE=${V}}" ] && set -x
+[ -z "${KEEP}" ]         && set -e
+
+#export ARCADIA_ROOT=${ARCADIA_ROOT:=`ya dump root || echo ~/arcadia/`}
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+
+# CUR_DIR=${CUR_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
+
+# YA_MAKE_ADD=--sanitize=address
+# YA_MAKE_ADD=--sanitize=memory
+# YA_MAKE_ADD=--sanitize=thread
+# YA_MAKE_ADD=--sanitize=undefined
+# YA_MAKE_ADD=--sanitize=leak
+# YA_MAKE_ADD=-r
+
+#YA_MAKE_ADD=${YA_MAKE_ADD=-r}
+#YA_MAKE=${YA_MAKE="ya make ${YA_MAKE_ADD}"} # "
+
+REBUILD=${REBUILD=1}
+
+# STARTER='ya tool gdb -ex run --args'
+# STARTER='ya tool gdb -ex run -ex backtrace --args'
+# STARTER='GRPC_VERBOSITY=info GRPC_TRACE=tcp,http,api'
+
+
+
+
+function re_create_mount {
+pushd ${ROOT_DIR}
+ya make -r cloud/filestore/buildall
+popd
+
+pushd ${ROOT_DIR}/cloud/filestore/bin
+
+. initctl.sh
+umount ${MOUNT_POINT} ||:
+. initctl.sh stop
+
+. initctl.sh format initialize
+. initctl.sh create startendpoint
+. initctl.sh start
+. initctl.sh mount
+
+sleep 3
+echo mountpoint = ${MOUNT_POINT}
+popd
+
+}
+
+
+# === Library section
+
+# Add pid of last runned program to "kill on exit" list
+function last_killer {
+    [ -n "${NO_KILL}" ] && NO_KILL_ECHO="echo "
+    TRAP+="${NO_KILL_ECHO} kill $! ||:;"
+    trap "$TRAP wait;" EXIT SIGINT SIGQUIT SIGTERM
+}
+
+# Run program as daemon. Kill at exit
+function daemon_killer {
+    bash -c "eval $* " &
+    last_killer
+}
+
+function on_exit {
+    TRAP+="$* ||:;"
+    trap "$TRAP" EXIT SIGINT SIGQUIT SIGTERM
+}
+
+function tmpl {
+    cp -v $1.tmpl $1
+    perl -p -i -E 's/\${(.+)}/$ENV{$1}/g' $1
+}
+
+fi

cloud/filestore/tools/testing/loadtest/test/load1.sh

@@ -0,0 +1,3 @@
+
+ffmpeg -f lavfi -i testsrc=duration=1:size=1080x1080:rate=30 -c:v rawvideo  testsrc.mpg
+ffmpeg -f rawvideo -pix_fmt rgb24 -video_size 1080x1080  -i testsrc.mpg -r 30  -c:v rawvideo -pix_fmt rgb24 -f caca -

cloud/filestore/tools/testing/loadtest/test/load3.sh

@@ -0,0 +1,6 @@
+
+git clone --depth 1 --recursive https://github.com/google/leveldb
+cd leveldb
+cmake -G Ninja .
+ninja
+ctest

cloud/filestore/tools/testing/loadtest/test/record.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export FS=record
+
+CUR_DIR=$(readlink -e $(dirname $0))
+. "${CUR_DIR}"/lib.sh
+
+export WORK_DIR=${CUR_DIR}/wrk
+mkdir -p ${WORK_DIR}
+
+re_create_mount
+
+df -h
+mount | grep ${HOME} ||:
+
+pushd ${MOUNT_POINT}
+
+# ${CUR_DIR}/load1.sh
+# ${CUR_DIR}/load2.sh
+ ${CUR_DIR}/load3.sh
+
+find .  -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/record_list.txt
+
+popd
+
+
+sleep 15
+
+ls -la ${LOGS_DIR}/filestore-server-profile-log.txt
+cp ${LOGS_DIR}/filestore-server-profile-log.txt ${WORK_DIR}
+
+
+umount ${MOUNT_POINT} ||:

cloud/filestore/tools/testing/loadtest/test/replay_fs.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Write to local fs:
+# REPLAY_ROOT=tmp/tmp NO_MOUNT=1 ./replay_fs.sh
+
+set -ex
+
+export FS=replay
+
+CUR_DIR=$(readlink -e $(dirname $0))
+. "${CUR_DIR}"/lib.sh
+
+export WORK_DIR=${CUR_DIR}/wrk
+mkdir -p ${WORK_DIR}
+
+pushd ${ROOT_DIR}
+ya make -r  cloud/filestore/tools/testing/loadtest/bin 
+popd
+
+[ -z "${NO_MOUNT}" ] && re_create_mount
+
+df -h ||:
+mount | grep ${HOME} ||:
+
+ls -la
+
+export REPLAY_ROOT=${REPLAY_ROOT=${MOUNT_POINT}/replayed}
+tmpl ${CUR_DIR}/replay_fs.txt
+
+#LOADTEST_VERBOSE=${LT_VERBOSE=error}
+LOADTEST_VERBOSE=${LT_VERBOSE=debug}
+env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_fs.txt 2>&1 | tee ${WORK_DIR}/log_fs.log
+
+#popd
+
+pushd ${REPLAY_ROOT}
+find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_fs_list.txt
+popd
+
+[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||:
+
+diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_fs_list.txt 

cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl

@@ -0,0 +1,18 @@
+Tests {
+    LoadTest {
+        Name: "smoke"
+        KeepFileStore: true
+        CreateFileStoreRequest: {
+            FileSystemId: "${FS}"
+            FolderId: "folder"
+            CloudId: "cloud"
+            BlocksCount: 10241024
+            BlockSize: 4096
+        }
+        ReplayFsSpec {
+             FileName: "${WORK_DIR}/filestore-server-profile-log.txt"
+             ReplayRoot: "${REPLAY_ROOT}"
+        }
+        IODepth: 2147483647
+    }
+}

cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -ex
+
+export FS=replay
+
+CUR_DIR=$(readlink -e $(dirname $0))
+. "${CUR_DIR}"/lib.sh
+
+#ROOT_DIR=$(git rev-parse --show-toplevel)
+export WORK_DIR=${CUR_DIR}/wrk
+mkdir -p ${WORK_DIR}
+
+pushd ${ROOT_DIR}
+ya make -r cloud/filestore/tools/testing/loadtest/bin
+popd
+
+re_create_mount
+
+# no need to mount
+
+df -h ||:
+mount | grep ${HOME} ||:
+
+tmpl ${CUR_DIR}/replay_grpc.txt
+
+#LOADTEST_VERBOSE=${VERBOSE=debug}
+LOADTEST_VERBOSE=${VERBOSE=info}
+env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_grpc.txt 2>&1 | tee ${WORK_DIR}/log_grpc.log
+
+pushd ${MOUNT_POINT}
+find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_grpc_list.txt
+popd
+
+[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||:
+
+diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_grpc_list.txt

cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl

@@ -0,0 +1,17 @@
+Tests {
+    LoadTest {
+        Name: "smoke"
+        KeepFileStore: true
+        CreateFileStoreRequest: {
+            FileSystemId: "${FS}"
+            FolderId: "folder"
+            CloudId: "cloud"
+            BlocksCount: 10241024
+            BlockSize: 4096
+        }
+        ReplayGrpcSpec {
+             FileName: "${WORK_DIR}/filestore-server-profile-log.txt"
+        }
+        IODepth: 64
+    }
+}