PBM-1114: check backup files before mark it as done #971
GitHub Actions / JUnit Test Report
failed
Sep 20, 2024 in 0s
15 tests run, 12 passed, 1 skipped, 2 failed.
Annotations
Check failure on line 103 in psmdb-testing/pbm-functional/pytest/test_fresh_sharded.py
github-actions / JUnit Test Report
test_fresh_sharded.test_incremental_PBM_T209
AssertionError: Error: backup '2024-09-20T09:06:33Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7f0500acca10>
newcluster = <cluster.Cluster object at 0x7f0500cc37d0>
@pytest.mark.timeout(600,func_only=True)
def test_incremental_PBM_T209(start_cluster,cluster,newcluster):
cluster.check_pbm_status()
cluster.make_backup("incremental --base")
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
backup=cluster.make_backup("incremental")
cluster.destroy()
newcluster.make_resync()
> newcluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)
test_fresh_sharded.py:103:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7f0500cc37d0>, name = '2024-09-20T09:06:33Z'
kwargs = {'check_pbm_status': True, 'restart_cluster': True}
client = MongoClient(host=['newmongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7f0500dcff90>, exit_status=1, command=b'time...240 pbm restore 2024-09-20T09:06:33Z --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:06:33Z' not found\n")
n = <testinfra.host.Host docker://newrscfg01>, timeout = 240, error = ''
host = 'newrscfg03', container = <Container: ddf57f40e6e2>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:06:33Z' not found
cluster.py:464: AssertionError
Check failure on line 103 in psmdb-testing/pbm-functional/pytest/test_remap_sharded.py
github-actions / JUnit Test Report
test_remap_sharded.test_incremental_PBM_T243
AssertionError: Error: backup '2024-09-20T09:12:27Z' not found
Raw output
start_cluster = True, cluster = <cluster.Cluster object at 0x7f0500ce7990>
newcluster = <cluster.Cluster object at 0x7f0500ce4090>
@pytest.mark.timeout(600,func_only=True)
def test_incremental_PBM_T243(start_cluster,cluster,newcluster):
cluster.check_pbm_status()
cluster.make_backup("incremental --base")
pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
backup=cluster.make_backup("incremental")
backup = backup + ' --replset-remapping="newrs1=rs1,newrs2=rs2,newrscfg=rscfg"'
cluster.destroy()
newcluster.make_resync()
> newcluster.make_restore(backup,restart_cluster=True, check_pbm_status=True)
test_remap_sharded.py:103:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <cluster.Cluster object at 0x7f0500ce4090>
name = '2024-09-20T09:12:27Z --replset-remapping="newrs1=rs1,newrs2=rs2,newrscfg=rscfg"'
kwargs = {'check_pbm_status': True, 'restart_cluster': True}
client = MongoClient(host=['newmongos:27017'], document_class=dict, tz_aware=False, connect=True)
result = CommandResult(backend=<testinfra.backend.docker.DockerBackend object at 0x7f0500dcff90>, exit_status=1, command=b'time...ewrs1=rs1,newrs2=rs2,newrscfg=rscfg" --wait', _stdout=b'', _stderr=b"Error: backup '2024-09-20T09:12:27Z' not found\n")
n = <testinfra.host.Host docker://newrscfg01>, timeout = 240, error = ''
host = 'newrscfg03', container = <Container: cf36787d4064>
def make_restore(self, name, **kwargs):
if self.layout == "sharded":
client = pymongo.MongoClient(self.connection)
result = client.admin.command("balancerStop")
client.close()
Cluster.log("Stopping balancer: " + str(result))
self.stop_mongos()
self.stop_arbiters()
n = testinfra.get_host("docker://" + self.pbm_cli)
timeout = time.time() + 60
while True:
if not self.get_status()['running']:
break
if time.time() > timeout:
assert False, "Cannot start restore, another operation running"
time.sleep(1)
Cluster.log("Restore started")
timeout=kwargs.get('timeout', 240)
result = n.run('timeout ' + str(timeout) + ' pbm restore ' + name + ' --wait')
if result.rc == 0:
Cluster.log(result.stdout)
else:
# try to catch possible failures if timeout exceeded
error=''
for host in self.mongod_hosts:
try:
container = docker.from_env().containers.get(host)
get_logs = container.exec_run(
'cat /var/lib/mongo/pbm.restore.log', stderr=False)
if get_logs.exit_code == 0:
Cluster.log(
"!!!!Possible failure on {}, file pbm.restore.log was found:".format(host))
logs = get_logs.output.decode('utf-8')
Cluster.log(logs)
if '"s":"F"' in logs:
error = logs
except docker.errors.APIError:
pass
if error:
assert False, result.stdout + result.stderr + "\n" + error
else:
> assert False, result.stdout + result.stderr
E AssertionError: Error: backup '2024-09-20T09:12:27Z' not found
cluster.py:464: AssertionError
Loading