From 3cdbc7b5ca14d1cd0b8d498c8c990552c5ada2e5 Mon Sep 17 00:00:00 2001 From: elankath Date: Tue, 26 Dec 2023 09:19:31 +0530 Subject: [PATCH 1/3] Check, update node label on machine obj prior to drain --- .../machinecontroller/machine_util.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index 67919c5c8..846bfa1af 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -920,9 +920,19 @@ func (c *controller) getVMStatus(ctx context.Context, getMachineStatusRequest *d state v1alpha1.MachineState ) - _, err := c.driver.GetMachineStatus(ctx, getMachineStatusRequest) + statusResp, err := c.driver.GetMachineStatus(ctx, getMachineStatusRequest) if err == nil { // VM Found + + // If `node` label is missing on machine obj, then update this label on Machine object with nodeName from status response + nodeName := getMachineStatusRequest.Machine.Labels[v1alpha1.NodeLabelKey] + if nodeName == "" { + err = c.updateMachineNodeLabel(ctx, getMachineStatusRequest.Machine, statusResp.NodeName) + if err != nil { + return machineutils.ShortRetry, err + } + } + description = machineutils.InitiateDrain state = v1alpha1.MachineStateProcessing retry = machineutils.ShortRetry @@ -1666,3 +1676,19 @@ func getNodeName(machine *v1alpha1.Machine) string { func getMachineDeploymentName(machine *v1alpha1.Machine) string { return machine.Labels["name"] } + +func (c *controller) updateMachineNodeLabel(ctx context.Context, machine *v1alpha1.Machine, nodeName string) error { + klog.V(2).Infof("Updating %q label on machine %q to %q", v1alpha1.NodeLabelKey, machine.Name, nodeName) + clone := machine.DeepCopy() + if clone.Labels == nil { + clone.Labels = make(map[string]string) + } + clone.Labels[v1alpha1.NodeLabelKey] = nodeName + _, err := c.controlMachineClient.Machines(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) + if err != nil { + klog.Warningf("Failed to update %q label on machine %q to %q. Retrying, error: %s", v1alpha1.NodeLabelKey, machine.Name, nodeName, err) + return err + } + klog.V(2).Infof("Updated %q label on machine %q to %q", v1alpha1.NodeLabelKey, machine.Name, nodeName) + return nil +} From 8a3e2019988514d0e07504436b903d88036679ad Mon Sep 17 00:00:00 2001 From: elankath Date: Tue, 26 Dec 2023 10:50:39 +0530 Subject: [PATCH 2/3] added logs for node deletion --- pkg/util/provider/machinecontroller/machine_util.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index 846bfa1af..b12708cba 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -1354,20 +1354,24 @@ func (c *controller) deleteNodeObject(ctx context.Context, machine *v1alpha1.Mac if nodeName != "" { // Delete node object err = c.targetCoreClient.CoreV1().Nodes().Delete(ctx, nodeName, metav1.DeleteOptions{}) + klog.V(3).Infof("Deleting node %q associated with machine %q", nodeName, machine.Name) if err != nil && !apierrors.IsNotFound(err) { // If its an error, and any other error than object not found description = fmt.Sprintf("Deletion of Node Object %q failed due to error: %s. %s", nodeName, err, machineutils.InitiateNodeDeletion) + klog.Error(description) state = v1alpha1.MachineStateFailed } else if err == nil { description = fmt.Sprintf("Deletion of Node Object %q is successful. %s", nodeName, machineutils.InitiateFinalizerRemoval) + klog.V(3).Info(description) state = v1alpha1.MachineStateProcessing - err = fmt.Errorf("Machine deletion in process. Deletion of node object was successful") } else { description = fmt.Sprintf("No node object found for %q, continuing deletion flow. %s", nodeName, machineutils.InitiateFinalizerRemoval) + klog.Warning(description) state = v1alpha1.MachineStateProcessing } } else { + klog.Errorf("Label %q not present on machine %q. Cannot delete associated Node", v1alpha1.NodeLabelKey, machine.Name) description = fmt.Sprintf("No node object found for machine, continuing deletion flow. %s", machineutils.InitiateFinalizerRemoval) state = v1alpha1.MachineStateProcessing From 35c0b0bcd588afdd67dc0973759503581e0b9298 Mon Sep 17 00:00:00 2001 From: elankath Date: Wed, 27 Dec 2023 11:51:49 +0530 Subject: [PATCH 3/3] modified log statement for clarity --- pkg/util/provider/machinecontroller/machine_util.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index b12708cba..e73619299 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -1371,10 +1371,9 @@ func (c *controller) deleteNodeObject(ctx context.Context, machine *v1alpha1.Mac state = v1alpha1.MachineStateProcessing } } else { - klog.Errorf("Label %q not present on machine %q. Cannot delete associated Node", v1alpha1.NodeLabelKey, machine.Name) - description = fmt.Sprintf("No node object found for machine, continuing deletion flow. %s", machineutils.InitiateFinalizerRemoval) + description = fmt.Sprintf("Label %q not present on machine %q or no associated node object found, continuing deletion flow. %s", v1alpha1.NodeLabelKey, machine.Name, machineutils.InitiateFinalizerRemoval) + klog.Error(description) state = v1alpha1.MachineStateProcessing - err = fmt.Errorf("Machine deletion in process. No node object found") }