fix: record failed lifecycle operations

changes/unreleased/Fixed-20260119-095716.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: Fixed a bug that prevented database deletion when we failed to create the Swarm service.
+time: 2026-01-19T09:57:16.746487-05:00

server/internal/resource/event.go

@@ -0,0 +1,149 @@
+package resource
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/wI2L/jsondiff"
+)
+
+type EventType string
+
+const (
+	EventTypeRefresh EventType = "refresh"
+	EventTypeCreate  EventType = "create"
+	EventTypeUpdate  EventType = "update"
+	EventTypeDelete  EventType = "delete"
+)
+
+type EventReason string
+
+const (
+	EventReasonDoesNotExist      EventReason = "does_not_exist"
+	EventReasonNeedsRecreate     EventReason = "needs_recreate"
+	EventReasonHasDiff           EventReason = "has_diff"
+	EventReasonForceUpdate       EventReason = "force_update"
+	EventReasonDependencyUpdated EventReason = "dependency_updated"
+	EventReasonHasError          EventReason = "has_error"
+)
+
+type Event struct {
+	Type     EventType      `json:"type"`
+	Resource *ResourceData  `json:"resource"`
+	Reason   EventReason    `json:"reason,omitempty"`
+	Diff     jsondiff.Patch `json:"diff,omitempty"`
+}
+
+func (e *Event) ResourceError() error {
+	if e.Resource != nil && e.Resource.Error != "" {
+		return errors.New(e.Resource.Error)
+	}
+	return nil
+}
+
+// Apply applies this event to its resource. It does not modify the state in the
+// given Context.
+func (e *Event) Apply(ctx context.Context, rc *Context) error {
+	resource, err := rc.Registry.Resource(e.Resource)
+	if err != nil {
+		return err
+	}
+
+	switch e.Type {
+	case EventTypeRefresh:
+		return e.refresh(ctx, rc, resource)
+	case EventTypeCreate:
+		return e.create(ctx, rc, resource)
+	case EventTypeUpdate:
+		return e.update(ctx, rc, resource)
+	case EventTypeDelete:
+		return e.delete(ctx, rc, resource)
+	default:
+		return fmt.Errorf("unknown event type: %s", e.Type)
+	}
+}
+
+func (e *Event) refresh(ctx context.Context, rc *Context, resource Resource) error {
+	// Retain the original Error and NeedsRecreate fields so that they're
+	// available for planCreates.
+	needsRecreate := e.Resource.NeedsRecreate
+	applyErr := e.Resource.Error
+
+	err := resource.Refresh(ctx, rc)
+	if errors.Is(err, ErrNotFound) {
+		needsRecreate = true
+	} else if err != nil {
+		return fmt.Errorf("failed to refresh resource %s: %w", resource.Identifier(), err)
+	}
+
+	updated, err := ToResourceData(resource)
+	if err != nil {
+		return err
+	}
+
+	updated.NeedsRecreate = needsRecreate
+	updated.Error = applyErr
+
+	e.Resource = updated
+
+	return nil
+}
+
+func (e *Event) create(ctx context.Context, rc *Context, resource Resource) error {
+	var needsRecreate bool
+	var applyErr string
+
+	if err := resource.Create(ctx, rc); err != nil {
+		needsRecreate = true
+		applyErr = fmt.Sprintf("failed to create resource %s: %s", resource.Identifier(), err.Error())
+	}
+
+	updated, err := ToResourceData(resource)
+	if err != nil {
+		return err
+	}
+	updated.NeedsRecreate = needsRecreate
+	updated.Error = applyErr
+
+	e.Resource = updated
+
+	return nil
+}
+
+func (e *Event) update(ctx context.Context, rc *Context, resource Resource) error {
+	var applyErr string
+
+	if err := resource.Update(ctx, rc); err != nil {
+		applyErr = fmt.Sprintf("failed to update resource %s: %s", resource.Identifier(), err.Error())
+	}
+
+	updated, err := ToResourceData(resource)
+	if err != nil {
+		return err
+	}
+	updated.Error = applyErr
+
+	e.Resource = updated
+
+	return nil
+}
+
+func (e *Event) delete(ctx context.Context, rc *Context, resource Resource) error {
+	if err := resource.Delete(ctx, rc); err != nil {
+		// We need to return an error here to indicate that this event should
+		// not be applied to the state. Applying a delete event to the state
+		// removes the resource, so if we didn't return the error it would be
+		// impossible to retry this operation.
+		return fmt.Errorf("failed to delete resource %s: %w", resource.Identifier(), err)
+	}
+
+	updated, err := ToResourceData(resource)
+	if err != nil {
+		return err
+	}
+
+	e.Resource = updated
+
+	return nil
+}

server/internal/resource/event_test.go

@@ -0,0 +1,138 @@
+package resource_test
+
+import (
+	"testing"
+
+	"github.com/pgEdge/control-plane/server/internal/resource"
+	"github.com/samber/do"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEvent(t *testing.T) {
+	t.Run("Apply", func(t *testing.T) {
+		registry := resource.NewRegistry()
+		resource.RegisterResourceType[*testResource](registry, testResourceType)
+
+		rc := &resource.Context{
+			State:    resource.NewState(),
+			Registry: registry,
+			Injector: do.New(),
+		}
+
+		for _, tc := range []struct {
+			name                          string
+			eventType                     resource.EventType
+			notFound                      bool
+			lifecycleError                string
+			originalResourceError         string
+			originalResourceNeedsRecreate bool
+			expectedErr                   string
+			expectedResourceError         string
+			expectedResourceNeedsRecreate bool
+		}{
+			{
+				name:      "refresh success",
+				eventType: resource.EventTypeRefresh,
+			},
+			{
+				name:                          "refresh success retains Error and NeedsRecreate",
+				eventType:                     resource.EventTypeRefresh,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+				expectedResourceError:         "some error",
+				expectedResourceNeedsRecreate: true,
+			},
+			{
+				name:                          "refresh not found",
+				eventType:                     resource.EventTypeRefresh,
+				notFound:                      true,
+				expectedResourceNeedsRecreate: true,
+			},
+			{
+				name:           "refresh failed",
+				eventType:      resource.EventTypeRefresh,
+				lifecycleError: "some error",
+				expectedErr:    "failed to refresh resource test_resource::test: some error",
+			},
+			{
+				name:      "create success",
+				eventType: resource.EventTypeCreate,
+			},
+			{
+				name:                          "create success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeCreate,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
+			{
+				name:                          "create failed",
+				eventType:                     resource.EventTypeCreate,
+				lifecycleError:                "some error",
+				expectedResourceError:         "failed to create resource test_resource::test: some error",
+				expectedResourceNeedsRecreate: true,
+			},
+			{
+				name:      "update success",
+				eventType: resource.EventTypeUpdate,
+			},
+			{
+				name:                          "update success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeUpdate,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
+			{
+				name:                  "update failed",
+				eventType:             resource.EventTypeUpdate,
+				lifecycleError:        "some error",
+				expectedResourceError: "failed to update resource test_resource::test: some error",
+			},
+			{
+				name:      "delete success",
+				eventType: resource.EventTypeDelete,
+			},
+			{
+				name:                          "delete success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeDelete,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
+			{
+				name:           "delete failed",
+				eventType:      resource.EventTypeDelete,
+				lifecycleError: "some error",
+				expectedErr:    "failed to delete resource test_resource::test: some error",
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				r := &testResource{
+					ID:       "test",
+					NotFound: tc.notFound,
+					Error:    tc.lifecycleError,
+				}
+
+				original := r.data(t)
+				original.Error = tc.originalResourceError
+				original.NeedsRecreate = tc.originalResourceNeedsRecreate
+
+				expected := r.data(t)
+				expected.Error = tc.expectedResourceError
+				expected.NeedsRecreate = tc.expectedResourceNeedsRecreate
+
+				event := &resource.Event{
+					Type:     tc.eventType,
+					Resource: original,
+				}
+
+				err := event.Apply(t.Context(), rc)
+
+				if tc.expectedErr != "" {
+					assert.ErrorContains(t, err, tc.expectedErr)
+				} else {
+					assert.NoError(t, err)
+					assert.Equal(t, expected, event.Resource)
+				}
+			})
+		}
+	})
+}

server/internal/resource/resource.go

@@ -43,6 +43,7 @@ type ResourceData struct {
 	DiffIgnore      []string        `json:"diff_ignore"`
 	ResourceVersion string          `json:"resource_version"`
 	PendingDeletion bool            `json:"pending_deletion"`
+	Error           string          `json:"error"`
 }
 
 func (r *ResourceData) Diff(other *ResourceData) (jsondiff.Patch, error) {
@@ -70,6 +71,7 @@ func (r *ResourceData) Clone() *ResourceData {
 		DiffIgnore:      slices.Clone(r.DiffIgnore),
 		ResourceVersion: r.ResourceVersion,
 		PendingDeletion: r.PendingDeletion,
+		Error:           r.Error,
 	}
 }
 

server/internal/resource/state.go

@@ -6,48 +6,11 @@ import (
 	"maps"
 	"slices"
 
-	"github.com/wI2L/jsondiff"
 	"gonum.org/v1/gonum/graph/simple"
 
 	"github.com/pgEdge/control-plane/server/internal/ds"
 )
 
-type EventType string
-
-const (
-	EventTypeRefresh EventType = "refresh"
-	EventTypeCreate  EventType = "create"
-	EventTypeUpdate  EventType = "update"
-	EventTypeDelete  EventType = "delete"
-)
-
-type EventReason string
-
-const (
-	EventReasonDoesNotExist      EventReason = "does_not_exist"
-	EventReasonNeedsRecreate     EventReason = "needs_recreate"
-	EventReasonHasDiff           EventReason = "has_diff"
-	EventReasonForceUpdate       EventReason = "force_update"
-	EventReasonDependencyUpdated EventReason = "dependency_updated"
-)
-
-type Event struct {
-	Type     EventType      `json:"type"`
-	Resource *ResourceData  `json:"resource"`
-	Reason   EventReason    `json:"reason,omitempty"`
-	Diff     jsondiff.Patch `json:"diff,omitempty"`
-}
-
-// WithData returns a clone of this event with the given data.
-func (e *Event) WithData(data *ResourceData) *Event {
-	return &Event{
-		Type:     e.Type,
-		Resource: data,
-		Reason:   e.Reason,
-		Diff:     e.Diff,
-	}
-}
-
 type State struct {
 	Resources map[Type]map[string]*ResourceData `json:"resources"`
 }
@@ -334,6 +297,12 @@ func (s *State) planCreates(options PlanOptions, desired *State) (Plan, error) {
 					Resource: resource,
 					Reason:   EventReasonNeedsRecreate,
 				}
+			case currentResource.Error != "":
+				event = &Event{
+					Type:     EventTypeUpdate,
+					Resource: resource,
+					Reason:   EventReasonHasError,
+				}
 			case options.ForceUpdate:
 				event = &Event{
 					Type:     EventTypeUpdate,