PBM-1114: check backup files before mark it as done #971
GitHub Actions / JUnit Test Report
failed
Sep 20, 2024 in 0s
29 tests run, 16 passed, 6 skipped, 7 failed.
Annotations
Check failure on line 86 in psmdb-testing/pbm-functional/pytest/test_PBM-1297.py
github-actions / JUnit Test Report
test_PBM-1297.test_logical_pitr_PBM_T253
AssertionError: Error: no base snapshot found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabbed0a90>
newcluster = <cluster.Cluster object at 0x7fbabd758250>
@pytest.mark.timeout(600, func_only=True)
def test_logical_pitr_PBM_T253(start_cluster,cluster,newcluster):
cluster.check_pbm_status()
cluster.make_backup("logical")
cluster.enable_pitr(pitr_extra_args="--set pitr.oplogSpanMin=0.5")
#Create the first database during oplog slicing
client=pymongo.MongoClient(cluster.connection)
client.admin.command("enableSharding", "test")
client.admin.command("shardCollection", "test.test", key={"_id": "hashed"})
for i in range(100):
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"doc":i})
time.sleep(30)
pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
backup="--time=" + pitr
Cluster.log("Time for PITR is: " + pitr)
time.sleep(30)
cluster.disable_pitr()
time.sleep(10)
cluster.destroy()
newcluster.create()
newcluster.setup_pbm()
time.sleep(10)
newcluster.check_pbm_status()
> newcluster.make_restore(backup,check_pbm_status=True)
test_PBM-1297.py:86:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabd758250>
name = '--time=2024-09-20T09:04:56', kwargs = {'check_pbm_status': True}
client = MongoClient(host=['newmongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabc5e1890>, exit_status=1, command=b'timeout 240 pbm restore --time=2024-09-20T09:04:56 --wait', _stdout=b'', _stderr=b'Error: no base snapshot found\n')
n = <testinfra.host.Host docker://newrscfg01>, timeout = 240, error = ''
host = 'newrscfg01', container = <Container: 1adad5e83f12>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: no base snapshot found
cluster.py:464: AssertionError
Check failure on line 108 in psmdb-testing/pbm-functional/pytest/test_PBM-773.py
github-actions / JUnit Test Report
test_PBM-773.test_logical_PBM_T221
AssertionError: Error: no base snapshot found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabc19f250>
@pytest.mark.timeout(300,func_only=True)
def test_logical_PBM_T221(start_cluster,cluster):
cluster.check_pbm_status()
cluster.make_backup("logical")
cluster.enable_pitr()
client = pymongo.MongoClient(cluster.connection)
db = client.test
collection = db.test
collection.insert_many(documents)
time.sleep(10)
with client.start_session() as session:
with session.start_transaction():
collection.insert_one({"e": 5}, session=session)
collection.insert_one({"f": 6}, session=session)
collection.insert_one({"g": 7}, session=session)
collection.insert_one({"h": 8}, session=session)
collection.insert_one({"i": 9}, session=session)
collection.insert_one({"j": 10}, session=session)
collection.insert_one({"k": 11}, session=session)
collection.insert_one({"l": 12}, session=session)
session.commit_transaction()
time.sleep(10)
pitr = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S")
backup="--time=" + pitr
Cluster.log("Time for PITR is: " + pitr)
time.sleep(10)
cluster.disable_pitr()
cluster.make_restore(backup,check_pbm_status=True)
assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == len(documents) + 8
folder="/backups/pbmPitr/rs1/" + datetime.utcnow().strftime("%Y%m%d") + "/"
for entry in os.scandir(folder):
file = entry.path
with open(file, "rb") as f:
data= f.read()
docs = bson.decode_all(data)
for doc in docs:
if "commitTransaction" in doc["o"]:
if doc["o"]["commitTransaction"] == 1:
Cluster.log("Oplog entry with commitTransaction: \n" + str(doc))
index = docs.index(doc)
Cluster.log("Index: " + str(index))
Cluster.log("Modifying oplog backup for shard rs1")
Cluster.log("Removing oplog entry with commitTransaction and all entries after it")
del docs[index:]
with open(file, "wb") as f:
f.truncate()
with open(file, "ab") as f:
for doc in docs:
f.write(bson.encode(doc))
> cluster.make_restore(backup,check_pbm_status=True)
test_PBM-773.py:108:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabc19f250>
name = '--time=2024-09-20T09:08:25', kwargs = {'check_pbm_status': True}
client = MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabbed2b10>, exit_status=1, command=b'timeout 240 pbm restore --time=2024-09-20T09:08:25 --wait', _stdout=b'', _stderr=b'Error: no base snapshot found\n')
n = <testinfra.host.Host docker://rscfg01>, timeout = 240, error = ''
host = 'rscfg03', container = <Container: 933c3ae9a7ea>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: no base snapshot found
cluster.py:464: AssertionError
Check failure on line 60 in psmdb-testing/pbm-functional/pytest/test_PBM-799.py
github-actions / JUnit Test Report
test_PBM-799.test_logical_PBM_T254
AssertionError: Error: backup '2024-09-20T09:09:56Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabc119ed0>
@pytest.mark.timeout(300,func_only=True)
def test_logical_PBM_T254(start_cluster,cluster):
cluster.check_pbm_status()
for i in range(100):
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i})
backup = cluster.make_backup("logical")
backup_status = cluster.exec_pbm_cli("describe-backup " + backup).stdout
Cluster.log("Backup status: \n" + backup_status)
cluster.make_resync()
cluster.check_pbm_status()
backup_status = cluster.exec_pbm_cli("describe-backup " + backup).stdout
Cluster.log("Backup status: \n" + backup_status)
> cluster.make_restore(backup,check_pbm_status=True)
test_PBM-799.py:60:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabc119ed0>, name = '2024-09-20T09:09:56Z'
kwargs = {'check_pbm_status': True}
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabb9aca50>, exit_status=1, command=b'time...240 pbm restore 2024-09-20T09:09:56Z --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:09:56Z' not found\n")
n = <testinfra.host.Host docker://rs101>, timeout = 240, error = ''
host = 'rs107', container = <Container: f291e8834c10>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:09:56Z' not found
cluster.py:464: AssertionError
Check failure on line 76 in psmdb-testing/pbm-functional/pytest/test_fresh_sharded.py
github-actions / JUnit Test Report
test_fresh_sharded.test_logical_PBM_T208
AssertionError: Error: backup '2024-09-20T09:24:25Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabb8e4fd0>
newcluster = <cluster.Cluster object at 0x7fbabb8e46d0>
@pytest.mark.timeout(600,func_only=True)
def test_logical_PBM_T208(start_cluster,cluster,newcluster):
cluster.check_pbm_status()
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
backup=cluster.make_backup("logical")
cluster.destroy()
newcluster.make_resync()
> newcluster.make_restore(backup,check_pbm_status=True)
test_fresh_sharded.py:76:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabb8e46d0>, name = '2024-09-20T09:24:25Z'
kwargs = {'check_pbm_status': True}
client = MongoClient(host=['newmongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabc5e1890>, exit_status=1, command=b'time...240 pbm restore 2024-09-20T09:24:25Z --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:24:25Z' not found\n")
n = <testinfra.host.Host docker://newrscfg01>, timeout = 240, error = ''
host = 'newrscfg03', container = <Container: 4bb6f06b980e>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:24:25Z' not found
cluster.py:464: AssertionError
Check failure on line 75 in psmdb-testing/pbm-functional/pytest/test_remap_sharded.py
github-actions / JUnit Test Report
test_remap_sharded.test_logical_PBM_T211
AssertionError: Error: backup '2024-09-20T09:29:16Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabc341810>
newcluster = <cluster.Cluster object at 0x7fbabc377d50>
@pytest.mark.timeout(600,func_only=True)
def test_logical_PBM_T211(start_cluster,cluster,newcluster):
cluster.check_pbm_status()
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
backup=cluster.make_backup("logical")
backup = backup + ' --replset-remapping="newrs1=rs1,newrs2=rs2,newrscfg=rscfg"'
cluster.destroy()
newcluster.make_resync()
> newcluster.make_restore(backup,check_pbm_status=True)
test_remap_sharded.py:75:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabc377d50>
name = '2024-09-20T09:29:16Z --replset-remapping="newrs1=rs1,newrs2=rs2,newrscfg=rscfg"'
kwargs = {'check_pbm_status': True}
client = MongoClient(host=['newmongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabc5e1890>, exit_status=1, command=b'time...ewrs1=rs1,newrs2=rs2,newrscfg=rscfg" --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:29:16Z' not found\n")
n = <testinfra.host.Host docker://newrscfg01>, timeout = 240, error = ''
host = 'newrscfg03', container = <Container: d0b187f5ee01>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:29:16Z' not found
cluster.py:464: AssertionError
Check failure on line 56 in psmdb-testing/pbm-functional/pytest/test_replicaset.py
github-actions / JUnit Test Report
test_replicaset.test_logical
AssertionError: Error: backup '2024-09-20T09:30:41Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbab591d250>
@pytest.mark.timeout(300,func_only=True)
def test_logical(start_cluster,cluster):
cluster.check_pbm_status()
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
pymongo.MongoClient(cluster.connection)["test"]["test1"].insert_many(documents)
backup_partial=cluster.make_backup("logical --ns=test.test")
backup_full=cluster.make_backup("logical")
pymongo.MongoClient(cluster.connection).drop_database('test')
cluster.make_restore(backup_partial,check_pbm_status=True)
assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == len(documents)
assert pymongo.MongoClient(cluster.connection)["test"]["test1"].count_documents({}) == 0
pymongo.MongoClient(cluster.connection).drop_database('test')
> cluster.make_restore(backup_full,check_pbm_status=True)
test_replicaset.py:56:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbab591d250>, name = '2024-09-20T09:30:41Z'
kwargs = {'check_pbm_status': True}
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabb9aca50>, exit_status=1, command=b'time...240 pbm restore 2024-09-20T09:30:41Z --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:30:41Z' not found\n")
n = <testinfra.host.Host docker://rs101>, timeout = 240, error = ''
host = 'rs103', container = <Container: 716e7a4fa07d>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:30:41Z' not found
cluster.py:464: AssertionError
Check failure on line 65 in psmdb-testing/pbm-functional/pytest/test_sharded.py
github-actions / JUnit Test Report
test_sharded.test_logical_PBM_T218
AssertionError: Error: backup '2024-09-20T09:34:35Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7fbabc35a050>
@pytest.mark.timeout(600,func_only=True)
def test_logical_PBM_T218(start_cluster,cluster):
cluster.check_pbm_status()
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
pymongo.MongoClient(cluster.connection)["test"]["test1"].insert_many(documents)
backup_partial_sharded=cluster.make_backup("logical --ns=test.test")
backup_partial_unsharded=cluster.make_backup("logical --ns=test.test1")
backup_full=cluster.make_backup("logical")
pymongo.MongoClient(cluster.connection).drop_database('test')
cluster.make_restore(backup_partial_sharded,check_pbm_status=True)
assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == len(documents)
assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False)
> cluster.make_restore(backup_partial_unsharded,check_pbm_status=True)
test_sharded.py:65:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7fbabc35a050>, name = '2024-09-20T09:34:35Z'
kwargs = {'check_pbm_status': True}
client = MongoClient(host=['mongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7fbabbed2b10>, exit_status=1, command=b'time...240 pbm restore 2024-09-20T09:34:35Z --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:34:35Z' not found\n")
n = <testinfra.host.Host docker://rscfg01>, timeout = 240, error = ''
host = 'rscfg03', container = <Container: d973dd567615>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:34:35Z' not found
cluster.py:464: AssertionError
Loading