Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

OCPBUGS-41811: (agent-based installer) let the bootstrap wait for workers before the reboot #910

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

OCPBUGS-41811: (agent-based installer) let the bootstrap wait for workers before the reboot #910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/installer/installer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,13 @@ func (i *installer) InstallNode() error {
return err
}
i.log.Info("Setting bootstrap node new role to master")

// In the agent-based installer the assisted-service runs on the bootstrap node, so let's
// wait for the workers (if any) to kick-off the joining process before rebooting the
// bootstrap node.
if err = i.waitForWorkers(ctx); err != nil {
return err
}
}

//upload host logs and report log status before reboot
Expand All @@ -184,6 +191,42 @@ func (i *installer) InstallNode() error {
return i.finalize()
}

func (i *installer) waitForWorkers(ctx context.Context) error {
// This waiting condition is valid only for agent-based installer.
kc, err := i.kcBuilder(KubeconfigPath, i.log)
if err != nil {
i.log.Error(err)
return err
}
if invoker := common.GetInvoker(kc, i.log); invoker != common.InvokerAgent {
return nil
}

return utils.WaitForPredicate(waitForeverTimeout, 30*time.Second, func() bool {
workers, err := i.inventoryClient.ListsHostsForRole(ctx, string(models.HostRoleWorker))
if err != nil {
i.log.WithError(err).Error("failed to get workers")
return false
}
if len(workers) == 0 {
return true
}

i.log.Infof("Found %d workers, verifying their current installation stage", len(workers))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From just reading the log output, it is unclear what the wait is for. Perhaps change this log message to indicate that we are waiting for the worker to reach reboot.

Suggested change
i.log.Infof("Found %d workers, verifying their current installation stage", len(workers))
i.log.Infof("Found %d workers, waiting for workers to reach reboot stage", len(workers))

for _, worker := range workers {
// If any worker is still in the waiting for control plane stage, then
// it means it didn't reboot yet and still requires assisted-service
if worker.Progress.CurrentStage == models.HostStageWaitingForControlPlane {
i.log.Infof("Still waiting for worker %s to start the joining process (current stage: %s)", worker.ID, worker.Progress.CurrentStage)
return false
}
i.log.Infof("Worker %s current stage is %s", worker.ID, worker.Progress.CurrentStage)
}

return true
})
}

func (i *installer) finalize() error {
if i.DryRunEnabled {
i.UpdateHostInstallProgress(models.HostStageRebooting, "")
Expand Down
76 changes: 70 additions & 6 deletions src/installer/installer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,30 @@ var _ = Describe("installer HostRoleMaster role", func() {
mockbmclient.EXPECT().UploadLogs(gomock.Any(), clusterId, models.LogsTypeController, gomock.Any()).Return(nil).Times(1)
}

waitForWorkersSuccessfully := func() {
mockbmclient.EXPECT().ListsHostsForRole(gomock.Any(), gomock.Any()).Times(1).Return(
models.HostList{
{
Progress: &models.HostProgressInfo{
CurrentStage: models.HostStageRebooting,
},
},
}, nil)
}

setupInvoker := func(invoker ...string) {
i := "assisted-service"
if len(invoker) > 0 {
i = invoker[0]
}
mockk8sclient.EXPECT().GetConfigMap("openshift-config", "openshift-install-manifests").AnyTimes().Return(
&v1.ConfigMap{
Data: map[string]string{
"invoker": i,
},
}, nil)
}

resolvConfSuccess := func() {
mockops.EXPECT().ReloadHostFile("/etc/resolv.conf").Return(nil).Times(1)
}
Expand Down Expand Up @@ -253,7 +277,7 @@ var _ = Describe("installer HostRoleMaster role", func() {
mockops.EXPECT().SystemctlAction("start", services[i]).Return(nil).Times(1)
}
}
WaitMasterNodesSucccess := func() {
WaitMasterNodesSucccessWithCluster := func(cluster *models.Cluster) {
mockbmclient.EXPECT().GetEnabledHostsNamesHosts(gomock.Any(), gomock.Any()).Return(inventoryNamesHost, nil).AnyTimes()
mockk8sclient.EXPECT().ListMasterNodes().Return(GetKubeNodes(map[string]string{}), nil).Times(1)
kubeNamesIds = map[string]string{"node0": "7916fa89-ea7a-443e-a862-b3e930309f65"}
Expand All @@ -263,8 +287,10 @@ var _ = Describe("installer HostRoleMaster role", func() {
"node1": "eb82821f-bf21-4614-9a3b-ecb07929f238"}
mockk8sclient.EXPECT().ListMasterNodes().Return(GetKubeNodes(kubeNamesIds), nil).Times(1)
mockbmclient.EXPECT().UpdateHostInstallProgress(gomock.Any(), inventoryNamesHost["node1"].Host.InfraEnvID.String(), inventoryNamesHost["node1"].Host.ID.String(), models.HostStageJoined, "").Times(1)
mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(&models.Cluster{}, nil).Times(2)
mockk8sclient.EXPECT().GetConfigMap(gomock.Any(), gomock.Any()).AnyTimes()
mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(cluster, nil).Times(2)
}
WaitMasterNodesSucccess := func() {
WaitMasterNodesSucccessWithCluster(&models.Cluster{})
}
prepareControllerSuccess := func() {
mockops.EXPECT().PrepareController().Return(nil).Times(1)
Expand Down Expand Up @@ -296,7 +322,7 @@ var _ = Describe("installer HostRoleMaster role", func() {
mockops.EXPECT().CreateOpenshiftSshManifest(assistedInstallerSshManifest, sshManifestTmpl, sshPubKeyPath).Return(nil).Times(1)
}

bootstrapSetup := func() {
bootstrapSetup := func(invoker ...string) {
cleanInstallDevice()
mkdirSuccess(sshDir)
mkdirSuccess(InstallDir)
Expand All @@ -306,6 +332,7 @@ var _ = Describe("installer HostRoleMaster role", func() {
generateSshKeyPairSuccess()
createOpenshiftSshManifestSuccess()
daemonReload(nil)
setupInvoker(invoker...)
}
for _, version := range []string{"4.7", "4.7.1", "4.7-pre-release", "4.8"} {
Context(version, func() {
Expand Down Expand Up @@ -415,6 +442,45 @@ var _ = Describe("installer HostRoleMaster role", func() {
ret := installerObj.InstallNode()
Expect(ret).Should(BeNil())
})
It("bootstrap role happy flow (agent-based installer)", func() {
updateProgressSuccess([][]string{{string(models.HostStageStartingInstallation), conf.Role},
{string(models.HostStageWaitingForControlPlane), waitingForBootstrapToPrepare},
{string(models.HostStageWaitingForControlPlane), waitingForMastersStatusInfo},
{string(models.HostStageInstalling), string(models.HostRoleMaster)},
{string(models.HostStageWritingImageToDisk)},
{string(models.HostStageRebooting)},
})
bootstrapSetup("agent-installer")
checkLocalHostname("notlocalhost", nil)
restartNetworkManager(nil)
prepareControllerSuccess()
startServicesSuccess()
WaitMasterNodesSucccessWithCluster(&models.Cluster{
Platform: &models.Platform{
Type: models.PlatformTypeBaremetal.Pointer(),
},
})
waitForBootkubeSuccess()
bootkubeStatusSuccess()
waitForETCDBootstrapSuccess()
bootstrapETCDStatusSuccess()
resolvConfSuccess()
waitForControllerSuccessfully(conf.ClusterID)
waitForWorkersSuccessfully()
//HostRoleMaster flow:
downloadHostIgnitionSuccess(infraEnvId, hostId, "master-host-id.ign")
writeToDiskSuccess(gomock.Any())
reportLogProgressSuccess()
setBootOrderSuccess(gomock.Any())
uploadLogsSuccess(true)
ironicAgentDoesntExist()
rebootSuccess()
getEncapsulatedMcSuccess(nil)
overwriteImageSuccess()
ret := installerObj.InstallNode()
Expect(ret).Should(BeNil())
})

})
}
It("bootstrap role creating SSH manifest failed", func() {
Expand Down Expand Up @@ -577,7 +643,6 @@ var _ = Describe("installer HostRoleMaster role", func() {
// node not ready
mockk8sclient.EXPECT().ListMasterNodes().Return(GetNotReadyKubeNodes(kubeNamesIds), nil).Times(1)
mockbmclient.EXPECT().GetCluster(gomock.Any(), false).Return(cluster, nil).Times(2)
mockk8sclient.EXPECT().GetConfigMap(gomock.Any(), gomock.Any()).AnyTimes()
if expectedRemoveUninitializedTaint {
mockk8sclient.EXPECT().UntaintNode("node0").Return(nil).Times(1)
} else {
Expand All @@ -589,7 +654,6 @@ var _ = Describe("installer HostRoleMaster role", func() {
"node1": "eb82821f-bf21-4614-9a3b-ecb07929f238"}
mockk8sclient.EXPECT().ListMasterNodes().Return(GetKubeNodes(kubeNamesIds), nil).Times(1)
mockbmclient.EXPECT().UpdateHostInstallProgress(gomock.Any(), inventoryNamesHost["node1"].Host.InfraEnvID.String(), inventoryNamesHost["node1"].Host.ID.String(), models.HostStageJoined, "").Times(1)
// mockbmclient.EXPECT().UpdateHostInstallProgress(gomock.Any(), inventoryNamesHost["node0"].Host.InfraEnvID.String(), inventoryNamesHost["node0"].Host.ID.String(), models.HostStageJoined, "").Times(1)
waitForBootkubeSuccess()
bootkubeStatusSuccess()
waitForETCDBootstrapSuccess()
Expand Down